diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index f049a16e..ae96209f 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -9,9 +9,9 @@ on: workflow_dispatch: inputs: version_tag: - description: 'Version tag (e.g. 2.5.8)' + description: 'Version tag (e.g. 2.5.9)' required: true - default: '2.5.8' + default: '2.5.9' jobs: build-and-push: diff --git a/config/settings.py b/config/settings.py index aa989f41..78b019fc 100644 --- a/config/settings.py +++ b/config/settings.py @@ -480,11 +480,19 @@ class ConfigManager: "search_min_delay_seconds": 0, }, "download_source": { - "mode": "soulseek", # Options: "soulseek", "youtube", "tidal", "qobuz", "hifi", "hybrid" + "mode": "soulseek", # Options: "soulseek", "youtube", "tidal", "qobuz", "hifi", "hybrid", "torrent", "usenet" "hybrid_primary": "soulseek", # Legacy: primary source for hybrid mode "hybrid_secondary": "youtube", # Legacy: fallback source for hybrid mode "hybrid_order": [], # Ordered list of sources for hybrid mode (overrides primary/secondary) "stream_source": "youtube", # Options: "youtube" (instant, default), "active" (use download source; falls back to youtube if soulseek) + # Album-bundle (torrent / usenet single-source) poll tuning. + # Downloader is polled every N seconds until the release + # lands; whole job aborts at the timeout. Defaults match + # the previous hard-coded constants. Users on slow private + # trackers / large box sets can extend the timeout without + # editing source. + "album_bundle_poll_interval_seconds": 2.0, + "album_bundle_timeout_seconds": 6 * 60 * 60, # 6 hours }, "tidal_download": { "quality": "lossless", # Options: "low", "high", "lossless", "hires" diff --git a/core/archive_pipeline.py b/core/archive_pipeline.py new file mode 100644 index 00000000..138e520c --- /dev/null +++ b/core/archive_pipeline.py @@ -0,0 +1,238 @@ +"""Archive extraction + audio-file discovery for torrent / usenet downloads. + +The torrent and usenet download plugins need a uniform way to: + +1. Walk the downloader's save directory and find every audio file in it. +2. If the directory contains an archive (``.zip`` / ``.rar`` / ``.tar`` / + ``.7z``), extract it first so the audio files inside become walkable. + +This module is intentionally narrow — no matching, no tagging, no +import. The download plugin layer composes this with the existing +post-processing / matching pipeline. Lidarr does NOT use this module: +Lidarr extracts archives in its own import step before SoulSync sees +the files at all. Usenet downloaders (SABnzbd, NZBGet) also auto- +extract by default. Torrents are the main case where SoulSync may +need to do the extract step itself — most music torrents ship loose, +but some bundle the album in a ``.rar`` archive. + +``rarfile`` is an optional dependency. If it isn't installed, archives +with ``.rar`` content are skipped with a single warning rather than +crashing the download. +""" + +from __future__ import annotations + +import tarfile +import zipfile +from pathlib import Path +from typing import List, Optional + +from utils.logging_config import get_logger + +logger = get_logger("archive_pipeline") + + +# Same audio-extension set as ``core/imports/file_ops.py`` ``quality_tiers``. +# Keep them in sync — if a new format is added to file_ops, add it here too +# or the walker will skip it and the download plugin will mark the download +# failed even when files arrived. +AUDIO_EXTENSIONS = frozenset([ + # lossless + '.flac', '.ape', '.wav', '.alac', '.dsf', '.dff', '.aiff', '.aif', + # high lossy + '.opus', '.ogg', + # standard lossy + '.m4a', '.aac', + # low lossy + '.mp3', '.wma', +]) + +ARCHIVE_EXTENSIONS = frozenset(['.zip', '.rar', '.tar', '.tar.gz', '.tgz', '.7z']) + + +def is_archive(path: Path) -> bool: + """True if the file extension looks like a supported archive. + + Compound extensions (``.tar.gz``, ``.tar.bz2``) are detected by + checking the last two suffixes joined together — Path.suffix + only returns the final suffix. + """ + if not path.is_file(): + return False + name = path.name.lower() + if name.endswith(('.tar.gz', '.tar.bz2', '.tar.xz')): + return True + return path.suffix.lower() in ARCHIVE_EXTENSIONS + + +def walk_audio_files(directory: Path) -> List[Path]: + """Recursively scan ``directory`` for audio files. Returns + a sorted list of absolute paths. Empty list if the directory + doesn't exist or contains no audio. + """ + if not directory or not directory.exists() or not directory.is_dir(): + return [] + out: List[Path] = [] + for child in directory.rglob('*'): + if not child.is_file(): + continue + if child.suffix.lower() in AUDIO_EXTENSIONS: + out.append(child.resolve()) + out.sort() + return out + + +def find_archives_in_dir(directory: Path) -> List[Path]: + """Find every archive file directly inside ``directory`` (one + level deep — torrents normally put the archive at the root of + their folder; we don't search nested dirs to avoid extracting + something we shouldn't). + """ + if not directory or not directory.exists() or not directory.is_dir(): + return [] + return sorted(p for p in directory.iterdir() if is_archive(p)) + + +def extract_archive(archive_path: Path, extract_to: Optional[Path] = None) -> Optional[Path]: + """Extract a single archive in-place (or to ``extract_to`` if + given). Returns the directory the archive was extracted into, + or ``None`` on failure. + + Supports ``.zip``, ``.tar``/``.tar.gz``/``.tar.bz2``/``.tar.xz``, + and ``.rar`` (only when the optional ``rarfile`` library is + installed). ``.7z`` is recognised but extraction requires + ``py7zr``; without it, the call logs and returns None. + """ + if not archive_path or not archive_path.exists(): + logger.warning("archive_pipeline: %s does not exist", archive_path) + return None + dest = extract_to or archive_path.parent + dest.mkdir(parents=True, exist_ok=True) + + name = archive_path.name.lower() + try: + if name.endswith('.zip'): + with zipfile.ZipFile(archive_path) as zf: + _safe_extract_zip(zf, dest) + return dest + if name.endswith(('.tar', '.tar.gz', '.tgz', '.tar.bz2', '.tar.xz')): + with tarfile.open(archive_path) as tf: + _safe_extract_tar(tf, dest) + return dest + if name.endswith('.rar'): + return _extract_rar(archive_path, dest) + if name.endswith('.7z'): + return _extract_7z(archive_path, dest) + except (zipfile.BadZipFile, tarfile.TarError, OSError) as e: + logger.error("archive_pipeline: failed to extract %s: %s", archive_path, e) + return None + logger.warning("archive_pipeline: unknown archive type for %s", archive_path) + return None + + +def extract_all_in_dir(directory: Path) -> List[Path]: + """Find every archive in ``directory`` and extract each in place. + Returns the list of directories archives were extracted into + (usually all the same — ``directory`` itself). Archives that + failed to extract are skipped silently after a warning. + """ + out: List[Path] = [] + for archive in find_archives_in_dir(directory): + result = extract_archive(archive) + if result is not None: + out.append(result) + return out + + +def collect_audio_after_extraction(directory: Path) -> List[Path]: + """One-shot helper for the download plugins: extract any archives + in the directory, then return the walked audio file list. This is + the common pattern — torrent / usenet plugin gets a save_path, + calls this, hands the resulting files to the matching pipeline. + """ + extract_all_in_dir(directory) + return walk_audio_files(directory) + + +# --------------------------------------------------------------------------- +# Safety helpers +# --------------------------------------------------------------------------- + + +def _safe_extract_zip(zf: zipfile.ZipFile, dest: Path) -> None: + """Extract a zipfile after rejecting any member whose resolved + path escapes ``dest`` (path traversal protection). + """ + dest = dest.resolve() + for member in zf.namelist(): + target = (dest / member).resolve() + if dest not in target.parents and target != dest: + logger.error("archive_pipeline: refusing path-traversal member %r", member) + return + zf.extractall(dest) + + +def _safe_extract_tar(tf: tarfile.TarFile, dest: Path) -> None: + """Same path-traversal protection for tarfiles.""" + dest = dest.resolve() + for member in tf.getmembers(): + target = (dest / member.name).resolve() + if dest not in target.parents and target != dest: + logger.error("archive_pipeline: refusing path-traversal member %r", member.name) + return + # ``filter='data'`` is the Python 3.12+ safe extractor; fall back + # to the legacy call on older runtimes. + try: + tf.extractall(dest, filter='data') # type: ignore[call-arg] + except TypeError: + tf.extractall(dest) + + +def _extract_rar(archive_path: Path, dest: Path) -> Optional[Path]: + try: + import rarfile # type: ignore[import-untyped] + except ImportError: + logger.warning( + "archive_pipeline: cannot extract %s — rarfile library not installed. " + "Install with: pip install rarfile (and ensure unrar is on PATH).", + archive_path, + ) + return None + try: + with rarfile.RarFile(archive_path) as rf: + dest_resolved = dest.resolve() + for name in rf.namelist(): + target = (dest_resolved / name).resolve() + if dest_resolved not in target.parents and target != dest_resolved: + logger.error("archive_pipeline: refusing path-traversal rar member %r", name) + return None + rf.extractall(dest) + return dest + except Exception as e: + logger.error("archive_pipeline: rar extract failed for %s: %s", archive_path, e) + return None + + +def _extract_7z(archive_path: Path, dest: Path) -> Optional[Path]: + try: + import py7zr # type: ignore[import-untyped] + except ImportError: + logger.warning( + "archive_pipeline: cannot extract %s — py7zr library not installed. " + "Install with: pip install py7zr.", + archive_path, + ) + return None + try: + with py7zr.SevenZipFile(archive_path, 'r') as sz: + dest_resolved = dest.resolve() + for name in sz.getnames(): + target = (dest_resolved / name).resolve() + if dest_resolved not in target.parents and target != dest_resolved: + logger.error("archive_pipeline: refusing path-traversal 7z member %r", name) + return None + sz.extractall(path=dest) + return dest + except Exception as e: + logger.error("archive_pipeline: 7z extract failed for %s: %s", archive_path, e) + return None diff --git a/core/connection_test.py b/core/connection_test.py index c470cfec..745caa2a 100644 --- a/core/connection_test.py +++ b/core/connection_test.py @@ -191,6 +191,12 @@ def run_service_test(service, test_config): 'tidal': "Tidal download source ready.", 'qobuz': "Qobuz download source ready.", 'hifi': "HiFi download source ready.", + 'deezer_dl': "Deezer download source ready.", + 'amazon': "Amazon download source ready.", + 'lidarr': "Lidarr download source ready.", + 'soundcloud': "SoundCloud download source ready.", + 'torrent': "Torrent download source ready.", + 'usenet': "Usenet download source ready.", 'hybrid': "Download sources ready (Hybrid mode)." } message = mode_messages.get(download_mode, "Download source connected.") @@ -203,6 +209,12 @@ def run_service_test(service, test_config): 'tidal': "Tidal download source not available. Check authentication.", 'qobuz': "Qobuz download source not available. Check authentication.", 'hifi': "HiFi download source not available. Public API instances may be down.", + 'deezer_dl': "Deezer download source not available. Check authentication.", + 'amazon': "Amazon download source not available.", + 'lidarr': "Lidarr download source not available. Check Lidarr URL and API key.", + 'soundcloud': "SoundCloud download source not available.", + 'torrent': "Torrent download source not available. Check Prowlarr and torrent client settings.", + 'usenet': "Usenet download source not available. Check Prowlarr and usenet client settings.", 'hybrid': "Could not connect to download sources. Check configuration." } error = mode_errors.get(download_mode, "Download source connection failed.") diff --git a/core/download_orchestrator.py b/core/download_orchestrator.py index 480ce7d6..879d0f52 100644 --- a/core/download_orchestrator.py +++ b/core/download_orchestrator.py @@ -286,11 +286,25 @@ class DownloadOrchestrator: chain = ['soulseek'] return chain - async def search(self, query: str, timeout: int = None, progress_callback=None) -> Tuple[List[TrackResult], List[AlbumResult]]: + async def search(self, query: str, timeout: int = None, progress_callback=None, + exclude_sources=None) -> Tuple[List[TrackResult], List[AlbumResult]]: """Search for tracks using configured source(s). Single-source modes route directly; hybrid mode delegates to - ``engine.search_with_fallback`` which tries the chain in order.""" + ``engine.search_with_fallback`` which tries the chain in order. + + ``exclude_sources`` (optional) is an iterable of source names + the caller wants filtered out of the hybrid chain. Used by the + per-track download worker to skip torrent / usenet for album- + context batches — those sources are release-level and don't + score meaningfully on per-track titles; the album-bundle flow + on the master worker handles them separately when they're the + single active source.""" if self.mode != 'hybrid': + # Single-source mode is opt-in; honour the user's choice even + # if it's torrent/usenet on an album batch (the master worker + # routes those through the album-bundle flow before per-track + # tasks ever fire, so search() being called here would be a + # non-album / wishlist / basic-search use case). client = self._client(self.mode) if not client: logger.error(f"{self.registry.display_name(self.mode)} client not available (failed to initialize)") @@ -299,6 +313,19 @@ class DownloadOrchestrator: return await client.search(query, timeout, progress_callback) chain = self._resolve_source_chain() + if exclude_sources: + blocked = {s.lower() for s in exclude_sources if s} + filtered = [s for s in chain if s.lower() not in blocked] + if filtered != chain: + logger.info( + "Hybrid search: excluding %s for this query (chain %s -> %s)", + sorted(blocked & {s.lower() for s in chain}), + " → ".join(chain), " → ".join(filtered) if filtered else "(empty)", + ) + chain = filtered + if not chain: + logger.warning("Hybrid search exhausted: no eligible sources after exclusion filter") + return [], [] logger.info(f"Hybrid search ({' → '.join(chain)}): {query}") return await self.engine.search_with_fallback(query, chain, timeout, progress_callback) diff --git a/core/download_plugins/album_bundle.py b/core/download_plugins/album_bundle.py new file mode 100644 index 00000000..6a0fb246 --- /dev/null +++ b/core/download_plugins/album_bundle.py @@ -0,0 +1,217 @@ +"""Shared helpers for the album-bundle download flow. + +The torrent and usenet download plugins both implement a +``download_album_to_staging`` method that searches Prowlarr for a +whole release, hands it to the active downloader, walks the +resulting audio files, and copies them into the staging folder. The +two implementations share the same release-picker heuristic and the +same staging-path collision logic. + +Pulled out of ``core/download_plugins/torrent.py`` so the usenet +plugin doesn't have to import private helpers from a sibling +plugin (Cin's "no leaky module boundaries" standard). + +Also exposes ``atomic_copy_to_staging`` — the audio file is copied +to a ``.tmp.`` sidecar first and atomically renamed onto its +final extension. The Auto-Import worker filters by audio extension +so the in-flight ``.tmp`` file is never picked up mid-copy, closing +the race between the album-bundle copy loop and Auto-Import's +folder scan. +""" + +from __future__ import annotations + +import shutil +import time +import uuid +from pathlib import Path +from typing import Iterable, Optional + +from config.settings import config_manager +from utils.logging_config import get_logger + +logger = get_logger("download_plugins.album_bundle") + + +# Album-pick size floor / ceiling. Single-track torrents (~10 MB) +# are rejected when bigger candidates exist; anything past 3 GB is +# treated as suspicious (multi-disc box-set + scans + extras). +ALBUM_PICK_MIN_BYTES = 40 * 1024 * 1024 +ALBUM_PICK_MAX_BYTES = 3 * 1024 * 1024 * 1024 + + +# Quality-score weights for the album-pick heuristic. Mirrors the +# tier order in ``core/imports/file_ops.py``'s ``quality_tiers`` — +# higher number = preferred. +_QUALITY_SCORE = {'flac': 4, 'ogg': 3, 'aac': 2, 'mp3': 1} + + +# Default poll cadence + timeout for the album-download poll loop. +# Both are overridable through config so users with slow trackers +# / large box-sets can extend the deadline without editing code. +DEFAULT_POLL_INTERVAL_SECONDS = 2.0 +DEFAULT_POLL_TIMEOUT_SECONDS = 6 * 60 * 60 + + +def get_poll_interval() -> float: + """Return the per-poll sleep duration (seconds). Configurable via + ``download_source.album_bundle_poll_interval_seconds``.""" + raw = config_manager.get('download_source.album_bundle_poll_interval_seconds', + DEFAULT_POLL_INTERVAL_SECONDS) + try: + value = float(raw) + if value > 0: + return value + except (TypeError, ValueError): + pass + return DEFAULT_POLL_INTERVAL_SECONDS + + +def get_poll_timeout() -> float: + """Return the total deadline for an album-bundle download + (seconds). Configurable via + ``download_source.album_bundle_timeout_seconds``.""" + raw = config_manager.get('download_source.album_bundle_timeout_seconds', + DEFAULT_POLL_TIMEOUT_SECONDS) + try: + value = float(raw) + if value > 0: + return value + except (TypeError, ValueError): + pass + return DEFAULT_POLL_TIMEOUT_SECONDS + + +def quality_score(title: str, quality_guess) -> int: + """Map a release title's inferred quality to a sortable integer. + + ``quality_guess`` is the function from each plugin that maps a + title string to a quality string ('flac' / 'mp3' / etc.) — passed + in so this module doesn't have to import either plugin and risk + a circular import.""" + return _QUALITY_SCORE.get(quality_guess(title) or '', 0) + + +def pick_best_album_release(candidates, quality_guess) -> Optional[object]: + """Pick the single best torrent / NZB for an album-bundle download. + + Heuristic, in priority order: + 1. Reasonable album-ish size (40 MB – 3 GB) — drops single-track + releases that snuck in and quarantines suspicious giants. + 2. Higher seeders > lower (dead torrents = dead downloads). + Usenet releases use ``grabs`` as a popularity proxy when + seeders is None. + 3. Higher quality (FLAC > AAC > MP3) inferred from title. + 4. Larger size as tiebreaker (often = higher bitrate). + """ + if not candidates: + return None + sized = [c for c in candidates + if ALBUM_PICK_MIN_BYTES <= (c.size or 0) <= ALBUM_PICK_MAX_BYTES] + pool = sized or list(candidates) + if not pool: + return None + + def _score(c) -> tuple: + seeders = c.seeders if c.seeders is not None else (c.grabs or 0) + return (seeders, quality_score(c.title or '', quality_guess), c.size or 0) + + return max(pool, key=_score) + + +def unique_staging_path(staging_dir: Path, src: Path) -> Path: + """Return a destination path inside ``staging_dir`` that doesn't + collide with an existing file. Appends ``_1``, ``_2``, ... before + the extension when needed; gives up after 1000 candidates and + returns the unsuffixed path so the caller will overwrite (better + than infinite loop or crash).""" + dest = staging_dir / src.name + if not dest.exists(): + return dest + stem = dest.stem + suffix = dest.suffix + for i in range(1, 1000): + candidate = staging_dir / f"{stem}_{i}{suffix}" + if not candidate.exists(): + return candidate + return dest + + +def atomic_copy_to_staging(src: Path, dest: Path) -> bool: + """Copy ``src`` to ``dest`` without exposing a partial file to + folder scanners. + + The Auto-Import worker filters by audio extension when scanning + Staging — see ``AUDIO_EXTENSIONS`` in ``core/auto_import_worker.py``. + Naming the in-flight file ``.tmp.`` keeps it + invisible until the rename atomically swings it to its final + extension. ``os.replace`` (used by ``Path.rename`` on Python 3.x) + is atomic on the same filesystem, so Auto-Import either sees the + file at its final name (complete) or doesn't see it at all + (in flight). + + Returns True on success, False on copy / rename failure. Caller + is expected to log the failure case so we don't double-log here. + """ + tmp = dest.with_name(f"{dest.name}.tmp.{uuid.uuid4().hex[:8]}") + try: + shutil.copy2(src, tmp) + except Exception: + # Best-effort cleanup of the partial file. If unlink fails + # (locked, permissions) we leave it — Auto-Import ignores it + # anyway because of the .tmp extension. + try: + if tmp.exists(): + tmp.unlink() + except Exception as cleanup_exc: + logger.debug("album_bundle tmp cleanup failed: %s", cleanup_exc) + raise + try: + tmp.replace(dest) + return True + except Exception: + try: + tmp.unlink(missing_ok=True) + except Exception as cleanup_exc: + logger.debug("album_bundle tmp cleanup failed: %s", cleanup_exc) + raise + + +def copy_audio_files_atomically( + sources: Iterable[Path], staging_dir: Path, +) -> list: + """Convenience wrapper: pick a non-colliding staging path for + each source, copy via ``atomic_copy_to_staging``. Returns the + list of final destination paths (as strings). Files that fail + to copy are logged and skipped; the caller decides what to do + with a partial result.""" + staging_dir.mkdir(parents=True, exist_ok=True) + out: list = [] + for src in sources: + dest = unique_staging_path(staging_dir, src) + try: + atomic_copy_to_staging(src, dest) + out.append(str(dest)) + except Exception as e: + logger.warning("[album_bundle] Failed to stage %s -> %s: %s", src, dest, e) + return out + + +# Re-export so callers don't have to remember which module owns +# what. The ``time`` import is kept so plugins can ``from +# core.download_plugins.album_bundle import time`` if they want to, +# avoiding a second std-lib import line for a single use. +__all__ = [ + "ALBUM_PICK_MIN_BYTES", + "ALBUM_PICK_MAX_BYTES", + "DEFAULT_POLL_INTERVAL_SECONDS", + "DEFAULT_POLL_TIMEOUT_SECONDS", + "atomic_copy_to_staging", + "copy_audio_files_atomically", + "get_poll_interval", + "get_poll_timeout", + "pick_best_album_release", + "quality_score", + "time", + "unique_staging_path", +] diff --git a/core/download_plugins/registry.py b/core/download_plugins/registry.py index ad01c2e9..2ca5ebb8 100644 --- a/core/download_plugins/registry.py +++ b/core/download_plugins/registry.py @@ -40,6 +40,8 @@ from core.download_plugins.base import DownloadSourcePlugin # orchestrator did. from core.amazon_download_client import AmazonDownloadClient from core.deezer_download_client import DeezerDownloadClient +from core.download_plugins.torrent import TorrentDownloadPlugin +from core.download_plugins.usenet import UsenetDownloadPlugin from core.hifi_client import HiFiClient from core.lidarr_download_client import LidarrDownloadClient from core.qobuz_client import QobuzClient @@ -190,5 +192,7 @@ def build_default_registry() -> DownloadPluginRegistry: aliases=('deezer_dl',))) registry.register(PluginSpec(name='lidarr', factory=LidarrDownloadClient, display_name='Lidarr')) registry.register(PluginSpec(name='soundcloud',factory=SoundcloudClient, display_name='SoundCloud')) + registry.register(PluginSpec(name='torrent', factory=TorrentDownloadPlugin, display_name='Torrent (Prowlarr)')) + registry.register(PluginSpec(name='usenet', factory=UsenetDownloadPlugin, display_name='Usenet (Prowlarr)')) return registry diff --git a/core/download_plugins/torrent.py b/core/download_plugins/torrent.py new file mode 100644 index 00000000..388cbe98 --- /dev/null +++ b/core/download_plugins/torrent.py @@ -0,0 +1,666 @@ +"""TorrentDownloadPlugin — composes Prowlarr search + torrent client +adapter + archive_pipeline into a uniform download source. + +Two flows: + +**Per-track flow** (basic search, single-track wishlist) — +1. ``search(query)`` calls ``ProwlarrClient.search`` filtered to + ``protocol='torrent'`` results, projects releases into + ``TrackResult`` / ``AlbumResult`` shaped objects the existing + search UI already understands. Encodes the indexer's + ``downloadUrl`` (or magnet URI) into the filename so + ``download()`` can recover it. +2. ``download(username, filename, ...)`` decodes the URL, asks the + active torrent adapter (qBittorrent, Transmission, or Deluge per + user's settings) to add it, spawns a background thread that + polls the adapter for completion. +3. On completion the thread walks the adapter-reported save path + via ``archive_pipeline.collect_audio_after_extraction`` and + exposes the full audio-file list. Post-processing can then pick + the requested track from a completed release instead of importing + the first file blindly. + +**Album-bundle flow** (album-context batch downloads — wired in +``core/downloads/master.py``) — +4. ``download_album_to_staging(album, artist, staging_dir)`` does + ONE Prowlarr search for the whole release, picks the best + torrent (prefers FLAC, decent seeders, reasonable size), + downloads it, extracts archives if needed, copies every audio + file into the staging directory. The existing per-track + ``try_staging_match`` flow then finds + imports each track by + fuzzy title match against the staged files. Per-track Prowlarr + queries never fire — track titles like "Luther (with SZA)" + would match album torrents like "GNX (2024) [FLAC]" at near- + zero confidence and break the per-track dispatch. + +Limitations: +- ``save_path`` is the torrent client's view of the disk. If + SoulSync runs on a different host than qBit / Trans / Deluge, + the post-processing pipeline can't see those files. The plugin + works fine for the all-on-one-box case (most users); remote + setups will need a future sync step (rclone / SMB / Docker + bind mount). +- Track-level metadata isn't available until after download. + Search results carry only the release title + indexer metadata; + individual track names are populated when the matching pipeline + walks the extracted audio files. +""" + +from __future__ import annotations + +import asyncio +import re +import threading +import time +import uuid +from pathlib import Path +from typing import Any, Dict, List, Optional, Tuple + +from config.settings import config_manager +from core.archive_pipeline import collect_audio_after_extraction +from core.download_plugins.album_bundle import ( + copy_audio_files_atomically, + get_poll_interval, + get_poll_timeout, + pick_best_album_release, +) +from core.download_plugins.base import DownloadSourcePlugin +from core.download_plugins.types import AlbumResult, DownloadStatus, TrackResult +from core.prowlarr_client import ( + DEFAULT_MUSIC_CATEGORIES, + ProwlarrClient, + ProwlarrSearchResult, +) +from core.torrent_clients import get_active_adapter as get_active_torrent_adapter +from utils.async_helpers import run_async +from utils.logging_config import get_logger + +logger = get_logger("download_plugins.torrent") + + +# Separator used to encode the download URL inside the filename +# field. Same convention Lidarr / YouTube use for embedding their +# own opaque identifiers — ``||``. +_FILENAME_SEP = '||' + +# Adapter states that count as the download being on-disk and +# safe to walk. ``seeding`` and ``completed`` both mean the +# bits are there; the user can pause seeding manually if they +# don't want to keep sharing. +_COMPLETE_STATES = frozenset(['seeding', 'completed']) + +# Poll cadence / timeout — both pull from config via the shared +# album_bundle helpers so users can extend the deadline for slow +# trackers without editing source. Kept as module aliases so the +# per-track flow at the bottom of this file can still import them +# under the legacy names without re-reading config every loop. +_POLL_TIMEOUT_SECONDS = get_poll_timeout() +_POLL_INTERVAL_SECONDS = get_poll_interval() + + +class TorrentDownloadPlugin(DownloadSourcePlugin): + """Torrent download source backed by Prowlarr + an active + torrent client adapter.""" + + def __init__(self) -> None: + self._prowlarr = ProwlarrClient() + # Track every download we've kicked off. Keyed by our own + # uuid — NOT the adapter's hash — because the orchestrator + # owns the lifecycle and we need a stable id even before + # the adapter has assigned one. + self.active_downloads: Dict[str, Dict[str, Any]] = {} + self._lock = threading.Lock() + self.shutdown_check = None + + def set_shutdown_check(self, check_callable): + self.shutdown_check = check_callable + + def reload_settings(self) -> None: + self._prowlarr.reload_settings() + + # ------------------------------------------------------------------ + # Lifecycle + # ------------------------------------------------------------------ + + def is_configured(self) -> bool: + if not self._prowlarr.is_configured(): + return False + adapter = get_active_torrent_adapter() + return bool(adapter and adapter.is_configured()) + + async def check_connection(self) -> bool: + if not self._prowlarr.is_configured(): + return False + adapter = get_active_torrent_adapter() + if not adapter or not adapter.is_configured(): + return False + # Probe both sides. A torrent download is useless if either + # the indexer or the downloader is unreachable. + prowlarr_ok = await self._prowlarr.check_connection() + if not prowlarr_ok: + return False + return await adapter.check_connection() + + # ------------------------------------------------------------------ + # Search + # ------------------------------------------------------------------ + + async def search( + self, + query: str, + timeout: Optional[int] = None, + progress_callback=None, + ) -> Tuple[List[TrackResult], List[AlbumResult]]: + if not self._prowlarr.is_configured(): + return ([], []) + try: + indexer_ids = _parse_indexer_id_filter() + results = await self._prowlarr.search( + query, + categories=DEFAULT_MUSIC_CATEGORIES, + indexer_ids=indexer_ids, + ) + except Exception as e: + logger.error("Torrent plugin search failed: %s", e) + return ([], []) + return self._project_results(results) + + def _project_results( + self, results: List[ProwlarrSearchResult] + ) -> Tuple[List[TrackResult], List[AlbumResult]]: + """Turn Prowlarr releases into TrackResult / AlbumResult + shaped objects. One TrackResult + one AlbumResult per + release — Prowlarr search hits are at the release level, + not the track level, so we can't synthesise track listings + without downloading the actual torrent.""" + tracks: List[TrackResult] = [] + albums: List[AlbumResult] = [] + for result in results: + if result.protocol != 'torrent': + continue + download_url = result.magnet_uri or result.download_url + if not download_url: + continue + filename = f"{download_url}{_FILENAME_SEP}{result.title}" + quality = _guess_quality_from_title(result.title) + parsed_artist, parsed_title = _parse_release_title(result.title) + tr = TrackResult( + username='torrent', + filename=filename, + size=result.size, + bitrate=None, + duration=None, + quality=quality, + # Torrent results don't have per-uploader slot / queue + # data the way Soulseek does. Fill with neutral values + # so the quality_score doesn't punish them artificially. + free_upload_slots=max(1, result.seeders or 0), + upload_speed=0, + queue_length=0, + # Pre-fill artist + title so TrackResult.__post_init__ + # doesn't auto-parse the filename — our filename starts + # with the indexer download URL, which would otherwise + # show up as "by download?apikey=..." in the UI. + artist=parsed_artist or result.indexer_name or 'Torrent', + title=parsed_title or result.title, + album=parsed_title or None, + track_number=None, + _source_metadata={ + 'indexer': result.indexer_name, + 'indexer_id': result.indexer_id, + 'seeders': result.seeders, + 'leechers': result.leechers, + 'grabs': result.grabs, + 'protocol': 'torrent', + }, + ) + tracks.append(tr) + albums.append(AlbumResult( + username='torrent', + album_path=f"torrent/{result.guid}", + album_title=parsed_title or result.title, + artist=parsed_artist or None, + track_count=1, # unknown until download finishes + total_size=result.size, + tracks=[tr], + dominant_quality=quality, + year=None, + )) + return tracks, albums + + # ------------------------------------------------------------------ + # Download + # ------------------------------------------------------------------ + + async def download( + self, + username: str, + filename: str, + file_size: int = 0, + ) -> Optional[str]: + if not self.is_configured(): + return None + download_url, display_name = _decode_filename(filename) + if not download_url: + logger.error("Torrent download missing URL in filename: %r", filename) + return None + + download_id = str(uuid.uuid4()) + with self._lock: + self.active_downloads[download_id] = { + 'id': download_id, + 'filename': filename, + 'username': 'torrent', + 'display_name': display_name, + 'state': 'Initializing', + 'progress': 0.0, + 'size': file_size, + 'transferred': 0, + 'speed': 0, + 'file_path': None, + 'audio_files': [], + 'torrent_hash': None, + 'error': None, + } + + thread = threading.Thread( + target=self._download_thread, + args=(download_id, download_url, display_name), + daemon=True, + name=f'torrent-dl-{download_id[:8]}', + ) + thread.start() + return download_id + + def _download_thread(self, download_id: str, download_url: str, display_name: str) -> None: + """Background worker: hand the URL to the active adapter, + poll until done, then walk the resulting directory.""" + adapter = get_active_torrent_adapter() + if adapter is None or not adapter.is_configured(): + self._mark_error(download_id, "No torrent client configured") + return + + try: + torrent_hash = run_async(adapter.add_torrent(download_url)) + except Exception as e: + self._mark_error(download_id, f"add_torrent failed: {e}") + return + if not torrent_hash: + self._mark_error(download_id, "Torrent client refused the URL") + return + + with self._lock: + row = self.active_downloads.get(download_id) + if row is not None: + row['torrent_hash'] = torrent_hash + row['state'] = 'InProgress, Downloading' + + deadline = time.monotonic() + _POLL_TIMEOUT_SECONDS + last_save_path: Optional[str] = None + while time.monotonic() < deadline: + if self.shutdown_check and self.shutdown_check(): + return + try: + status = run_async(adapter.get_status(torrent_hash)) + except Exception as e: + logger.warning("Torrent poll error for %s: %s", torrent_hash, e) + status = None + + if status is None: + # Adapter forgot about the torrent — probably user-removed. + self._mark_error(download_id, "Torrent disappeared from client") + return + + with self._lock: + row = self.active_downloads.get(download_id) + if row is not None: + row['progress'] = status.progress * 100.0 + row['transferred'] = status.downloaded + row['speed'] = status.download_speed + row['size'] = status.size or row.get('size', 0) + row['state'] = _adapter_state_to_display(status.state) + row['error'] = status.error + if status.save_path: + last_save_path = status.save_path + + if status.state in _COMPLETE_STATES: + self._finalize_download(download_id, last_save_path) + return + if status.state == 'error': + self._mark_error(download_id, status.error or "Torrent client reported error") + return + + time.sleep(_POLL_INTERVAL_SECONDS) + + self._mark_error(download_id, "Torrent download timed out") + + def _finalize_download(self, download_id: str, save_path: Optional[str]) -> None: + """Adapter said complete. Walk the directory + pick the + first audio file as the canonical ``file_path``.""" + if not save_path: + self._mark_error(download_id, "Torrent completed but no save_path reported") + return + try: + audio_files = collect_audio_after_extraction(Path(save_path)) + except Exception as e: + self._mark_error(download_id, f"Post-extract walk failed: {e}") + return + if not audio_files: + self._mark_error(download_id, f"No audio files found in {save_path}") + return + primary = audio_files[0] + with self._lock: + row = self.active_downloads.get(download_id) + if row is not None: + row['state'] = 'Completed, Succeeded' + row['progress'] = 100.0 + row['file_path'] = str(primary) + row['audio_files'] = [str(path) for path in audio_files] + logger.info("Torrent download complete: %s -> %s (%d audio files)", + download_id[:8], primary.name, len(audio_files)) + + def _mark_error(self, download_id: str, message: str) -> None: + logger.error("Torrent download %s failed: %s", download_id[:8], message) + with self._lock: + row = self.active_downloads.get(download_id) + if row is not None: + row['state'] = 'Completed, Errored' + row['error'] = message + + # ------------------------------------------------------------------ + # Status / lifecycle + # ------------------------------------------------------------------ + + async def get_all_downloads(self) -> List[DownloadStatus]: + with self._lock: + rows = list(self.active_downloads.values()) + return [_row_to_status(r) for r in rows] + + async def get_download_status(self, download_id: str) -> Optional[DownloadStatus]: + with self._lock: + row = self.active_downloads.get(download_id) + if row is None: + return None + return _row_to_status(row) + + async def cancel_download( + self, + download_id: str, + username: Optional[str] = None, + remove: bool = False, + ) -> bool: + adapter = get_active_torrent_adapter() + with self._lock: + row = self.active_downloads.get(download_id) + torrent_hash = row.get('torrent_hash') if row else None + if adapter and torrent_hash: + try: + await adapter.remove(torrent_hash, delete_files=remove) + except Exception as e: + logger.warning("Torrent cancel via adapter failed: %s", e) + with self._lock: + if remove: + self.active_downloads.pop(download_id, None) + else: + row = self.active_downloads.get(download_id) + if row is not None: + row['state'] = 'Cancelled' + return True + + async def clear_all_completed_downloads(self) -> bool: + with self._lock: + for did in list(self.active_downloads.keys()): + state = self.active_downloads[did].get('state', '') + if state.startswith('Completed') or state == 'Cancelled': + self.active_downloads.pop(did, None) + return True + + # ------------------------------------------------------------------ + # Album-bundle flow + # ------------------------------------------------------------------ + + def download_album_to_staging( + self, + album_name: str, + artist_name: str, + staging_dir: str, + progress_callback=None, + ) -> Dict[str, Any]: + """One-shot album download: search Prowlarr for the whole + release, pick the best torrent, fetch it, extract if needed, + copy every audio file into ``staging_dir`` so the existing + ``try_staging_match`` flow can hand each track off to the + post-processing pipeline. + + ``progress_callback`` is called with a dict on each state + change so the batch UI can show download progress without + waiting for the whole thing. + + Returns ``{'success': bool, 'files': [paths], 'error': str|None}``. + """ + result: Dict[str, Any] = {'success': False, 'files': [], 'error': None} + if not self.is_configured(): + result['error'] = 'Torrent source not configured' + return result + + adapter = get_active_torrent_adapter() + if adapter is None or not adapter.is_configured(): + result['error'] = 'No active torrent client' + return result + + def _emit(state: str, **extra) -> None: + if progress_callback: + payload = {'state': state, **extra} + try: + progress_callback(payload) + except Exception as cb_exc: + logger.debug("[Torrent album] progress callback failed: %s", cb_exc) + + # Phase 1: search Prowlarr for the album. + query = f"{artist_name} {album_name}".strip() + _emit('searching', query=query) + try: + search_results = run_async(self._prowlarr.search( + query, categories=DEFAULT_MUSIC_CATEGORIES, + indexer_ids=_parse_indexer_id_filter(), + )) + except Exception as e: + result['error'] = f'Prowlarr search failed: {e}' + return result + + candidates = [r for r in search_results + if r.protocol == 'torrent' and (r.magnet_uri or r.download_url)] + if not candidates: + result['error'] = f'No torrent results found for "{query}"' + return result + + picked = pick_best_album_release(candidates, _guess_quality_from_title) + if picked is None: + result['error'] = 'No suitable torrent candidate after filtering' + return result + + download_url = picked.magnet_uri or picked.download_url + logger.info("[Torrent album] Picked '%s' (size=%.1fMB seeders=%s indexer=%s)", + picked.title, picked.size / 1_048_576, picked.seeders, picked.indexer_name) + _emit('queued', release=picked.title, size=picked.size, seeders=picked.seeders) + + # Phase 2: hand to adapter. + try: + torrent_id = run_async(adapter.add_torrent(download_url)) + except Exception as e: + result['error'] = f'Torrent client refused the release: {e}' + return result + if not torrent_id: + result['error'] = 'Torrent client refused the release' + return result + + # Phase 3: poll until complete. + _emit('downloading', release=picked.title) + save_path = self._poll_album_download(adapter, torrent_id, picked.title, _emit) + if save_path is None: + result['error'] = 'Torrent download failed or timed out' + return result + + # Phase 4: extract + walk + copy to staging. + _emit('staging', release=picked.title) + try: + audio_files = collect_audio_after_extraction(Path(save_path)) + except Exception as e: + result['error'] = f'Failed to walk audio files: {e}' + return result + if not audio_files: + result['error'] = f'No audio files found in {save_path}' + return result + + copied = copy_audio_files_atomically(audio_files, Path(staging_dir)) + if not copied: + result['error'] = 'No audio files copied to staging' + return result + logger.info("[Torrent album] Staged %d audio files for '%s'", len(copied), album_name) + _emit('staged', count=len(copied)) + result['success'] = True + result['files'] = copied + return result + + def _poll_album_download(self, adapter, torrent_id, title, emit) -> Optional[str]: + """Poll the adapter until the torrent is complete. Returns + the save path or ``None`` on timeout / failure.""" + deadline = time.monotonic() + _POLL_TIMEOUT_SECONDS + last_save_path: Optional[str] = None + while time.monotonic() < deadline: + if self.shutdown_check and self.shutdown_check(): + return None + try: + status = run_async(adapter.get_status(torrent_id)) + except Exception as e: + logger.warning("[Torrent album] Poll error: %s", e) + status = None + if status is None: + logger.error("[Torrent album] '%s' disappeared from client", title) + return None + emit('downloading', progress=status.progress, downloaded=status.downloaded, + speed=status.download_speed) + if status.save_path: + last_save_path = status.save_path + if status.state in _COMPLETE_STATES: + return last_save_path + if status.state == 'error': + logger.error("[Torrent album] '%s' errored: %s", title, status.error) + return None + time.sleep(_POLL_INTERVAL_SECONDS) + logger.error("[Torrent album] '%s' timed out", title) + return None + + +# --------------------------------------------------------------------------- +# Module-level helpers (pure functions — easy to unit-test) +# --------------------------------------------------------------------------- + + +def _decode_filename(filename: str) -> Tuple[Optional[str], str]: + """Pull the encoded download URL out of the ``filename`` string. + Returns ``(url, display_name)``. ``url`` is None when the string + has no separator.""" + if not filename or _FILENAME_SEP not in filename: + return (None, filename or '') + url, display = filename.split(_FILENAME_SEP, 1) + return (url, display) + + +def _parse_release_title(title: str) -> Tuple[str, str]: + """Split a release title into ``(artist, title)`` using the + ``Artist - Title`` / ``Artist - Album`` convention almost every + indexer follows. Returns ``('', title)`` when no dash is found. + + Without this, ``TrackResult.__post_init__`` runs the bare + filename through ``parse_filename_metadata`` — and our filename + starts with the indexer's download URL, so the auto-parser + extracts garbage like ``download?apikey=...`` as the artist + and shows it in the search-result UI's "by" line. Pre-filling + the artist field short-circuits the auto-parse. + """ + if not title: + return ('', '') + # Strip common quality / format tags so the dash split doesn't + # eat them — "Artist - Album [FLAC] (2020)" → "Artist", "Album". + cleaned = re.sub(r'\s*[\[\(][^\]\)]*[\]\)]\s*$', '', title.strip()) + # Look for the FIRST " - " (or "-" surrounded by content). Some + # release titles have multiple dashes (subtitle dashes); the + # first split is the artist/work boundary. + parts = re.split(r'\s+-\s+|\s+-(?=\S)|(?<=\S)-\s+', cleaned, maxsplit=1) + if len(parts) == 2: + artist = parts[0].strip() + rest = parts[1].strip() + # Reject obvious non-artist prefixes (URLs, hashes, single + # punctuation) so we don't propagate garbage. + if artist and not re.match(r'^https?:|^[a-f0-9]{32,}$', artist): + return (artist, rest or cleaned) + return ('', cleaned) + + +def _guess_quality_from_title(title: str) -> str: + """Read the quality hint from a release title — most music + torrents put the encoding right in the name (FLAC, MP3 320, + etc.). Falls back to ``'mp3'`` so quality_score doesn't crash.""" + if not title: + return 'mp3' + lower = title.lower() + if 'flac' in lower: + return 'flac' + if re.search(r'\b24[\s-]?bit\b', lower) or 'hi-?res' in lower: + return 'flac' + if 'aac' in lower: + return 'aac' + if 'ogg' in lower: + return 'ogg' + return 'mp3' + + +def _parse_indexer_id_filter() -> List[int]: + """Read the comma-separated indexer-ID allowlist from config. + Empty list = search every enabled indexer.""" + raw = (config_manager.get('prowlarr.indexer_ids', '') or '').strip() + if not raw: + return [] + out: List[int] = [] + for chunk in raw.split(','): + chunk = chunk.strip() + if not chunk: + continue + try: + out.append(int(chunk)) + except ValueError: + continue + return out + + +def _adapter_state_to_display(state: str) -> str: + """Translate the adapter-uniform state strings into the + ``'InProgress, Downloading'`` / ``'Completed, Succeeded'`` + style the existing UI expects (matches Soulseek + Lidarr).""" + mapping = { + 'queued': 'Queued', + 'downloading': 'InProgress, Downloading', + 'stalled': 'InProgress, Stalled', + 'seeding': 'Completed, Succeeded', + 'completed': 'Completed, Succeeded', + 'paused': 'Paused', + 'error': 'Completed, Errored', + } + return mapping.get(state, state.title()) + + +def _row_to_status(row: Dict[str, Any]) -> DownloadStatus: + return DownloadStatus( + id=row['id'], + filename=row['filename'], + username=row['username'], + state=row.get('state', 'Unknown'), + progress=float(row.get('progress', 0.0)), + size=int(row.get('size', 0)), + transferred=int(row.get('transferred', 0)), + speed=int(row.get('speed', 0)), + time_remaining=None, + file_path=row.get('file_path'), + audio_files=row.get('audio_files') or None, + ) diff --git a/core/download_plugins/types.py b/core/download_plugins/types.py index 46e250c3..59d52913 100644 --- a/core/download_plugins/types.py +++ b/core/download_plugins/types.py @@ -197,3 +197,4 @@ class DownloadStatus: speed: int time_remaining: Optional[int] = None file_path: Optional[str] = None + audio_files: Optional[List[str]] = None diff --git a/core/download_plugins/usenet.py b/core/download_plugins/usenet.py new file mode 100644 index 00000000..419bc380 --- /dev/null +++ b/core/download_plugins/usenet.py @@ -0,0 +1,461 @@ +"""UsenetDownloadPlugin — composes Prowlarr search + usenet client +adapter + archive_pipeline into a uniform download source. + +Mirrors ``TorrentDownloadPlugin`` in shape and lifecycle (see that +module's docstring for the full pipeline rationale). Differences: + +- Search filters Prowlarr results to ``protocol='usenet'``. +- ``add_nzb`` replaces ``add_torrent``; for NZBs we usually have + a direct HTTP URL the indexer exposes via Prowlarr. +- Usenet clients (SABnzbd, NZBGet) typically auto-extract during + post-processing, so ``archive_pipeline.collect_audio_after_extraction`` + usually has nothing to extract and just walks loose files. +""" + +from __future__ import annotations + +import threading +import time +import uuid +from pathlib import Path +from typing import Any, Dict, List, Optional, Tuple + +from core.archive_pipeline import collect_audio_after_extraction +from core.download_plugins.album_bundle import ( + copy_audio_files_atomically, + pick_best_album_release, +) +from core.download_plugins.base import DownloadSourcePlugin +from core.download_plugins.torrent import ( + _adapter_state_to_display, + _decode_filename, + _guess_quality_from_title, + _parse_indexer_id_filter, + _parse_release_title, + _row_to_status, + _COMPLETE_STATES, + _FILENAME_SEP, + _POLL_INTERVAL_SECONDS, + _POLL_TIMEOUT_SECONDS, +) +from core.download_plugins.types import AlbumResult, DownloadStatus, TrackResult +from core.prowlarr_client import ( + DEFAULT_MUSIC_CATEGORIES, + ProwlarrClient, + ProwlarrSearchResult, +) +from core.usenet_clients import get_active_adapter as get_active_usenet_adapter +from utils.async_helpers import run_async +from utils.logging_config import get_logger + +logger = get_logger("download_plugins.usenet") + + +class UsenetDownloadPlugin(DownloadSourcePlugin): + """Usenet download source backed by Prowlarr + an active usenet + client adapter (SABnzbd or NZBGet).""" + + def __init__(self) -> None: + self._prowlarr = ProwlarrClient() + self.active_downloads: Dict[str, Dict[str, Any]] = {} + self._lock = threading.Lock() + self.shutdown_check = None + + def set_shutdown_check(self, check_callable): + self.shutdown_check = check_callable + + def reload_settings(self) -> None: + self._prowlarr.reload_settings() + + def is_configured(self) -> bool: + if not self._prowlarr.is_configured(): + return False + adapter = get_active_usenet_adapter() + return bool(adapter and adapter.is_configured()) + + async def check_connection(self) -> bool: + if not self._prowlarr.is_configured(): + return False + adapter = get_active_usenet_adapter() + if not adapter or not adapter.is_configured(): + return False + if not await self._prowlarr.check_connection(): + return False + return await adapter.check_connection() + + # ------------------------------------------------------------------ + # Search + # ------------------------------------------------------------------ + + async def search( + self, + query: str, + timeout: Optional[int] = None, + progress_callback=None, + ) -> Tuple[List[TrackResult], List[AlbumResult]]: + if not self._prowlarr.is_configured(): + return ([], []) + try: + indexer_ids = _parse_indexer_id_filter() + results = await self._prowlarr.search( + query, + categories=DEFAULT_MUSIC_CATEGORIES, + indexer_ids=indexer_ids, + ) + except Exception as e: + logger.error("Usenet plugin search failed: %s", e) + return ([], []) + return self._project_results(results) + + def _project_results( + self, results: List[ProwlarrSearchResult] + ) -> Tuple[List[TrackResult], List[AlbumResult]]: + tracks: List[TrackResult] = [] + albums: List[AlbumResult] = [] + for result in results: + if result.protocol != 'usenet': + continue + if not result.download_url: + continue + filename = f"{result.download_url}{_FILENAME_SEP}{result.title}" + quality = _guess_quality_from_title(result.title) + parsed_artist, parsed_title = _parse_release_title(result.title) + tr = TrackResult( + username='usenet', + filename=filename, + size=result.size, + bitrate=None, + duration=None, + quality=quality, + # Usenet doesn't expose per-uploader concurrency the way + # Soulseek does; fill in neutral non-punishing values. + free_upload_slots=1, + upload_speed=0, + queue_length=0, + # Pre-fill artist + title so TrackResult.__post_init__ + # doesn't auto-parse the filename — same URL-in-filename + # gotcha as the torrent plugin. + artist=parsed_artist or result.indexer_name or 'Usenet', + title=parsed_title or result.title, + album=parsed_title or None, + track_number=None, + _source_metadata={ + 'indexer': result.indexer_name, + 'indexer_id': result.indexer_id, + 'grabs': result.grabs, + 'protocol': 'usenet', + }, + ) + tracks.append(tr) + albums.append(AlbumResult( + username='usenet', + album_path=f"usenet/{result.guid}", + album_title=parsed_title or result.title, + artist=parsed_artist or None, + track_count=1, + total_size=result.size, + tracks=[tr], + dominant_quality=quality, + year=None, + )) + return tracks, albums + + # ------------------------------------------------------------------ + # Download + # ------------------------------------------------------------------ + + async def download( + self, + username: str, + filename: str, + file_size: int = 0, + ) -> Optional[str]: + if not self.is_configured(): + return None + nzb_url, display_name = _decode_filename(filename) + if not nzb_url: + logger.error("Usenet download missing URL in filename: %r", filename) + return None + + download_id = str(uuid.uuid4()) + with self._lock: + self.active_downloads[download_id] = { + 'id': download_id, + 'filename': filename, + 'username': 'usenet', + 'display_name': display_name, + 'state': 'Initializing', + 'progress': 0.0, + 'size': file_size, + 'transferred': 0, + 'speed': 0, + 'file_path': None, + 'audio_files': [], + 'job_id': None, + 'error': None, + } + + thread = threading.Thread( + target=self._download_thread, + args=(download_id, nzb_url), + daemon=True, + name=f'usenet-dl-{download_id[:8]}', + ) + thread.start() + return download_id + + def _download_thread(self, download_id: str, nzb_url: str) -> None: + adapter = get_active_usenet_adapter() + if adapter is None or not adapter.is_configured(): + self._mark_error(download_id, "No usenet client configured") + return + + try: + job_id = run_async(adapter.add_nzb(nzb_url)) + except Exception as e: + self._mark_error(download_id, f"add_nzb failed: {e}") + return + if not job_id: + self._mark_error(download_id, "Usenet client refused the NZB") + return + + with self._lock: + row = self.active_downloads.get(download_id) + if row is not None: + row['job_id'] = job_id + row['state'] = 'InProgress, Downloading' + + deadline = time.monotonic() + _POLL_TIMEOUT_SECONDS + last_save_path: Optional[str] = None + while time.monotonic() < deadline: + if self.shutdown_check and self.shutdown_check(): + return + try: + status = run_async(adapter.get_status(job_id)) + except Exception as e: + logger.warning("Usenet poll error for %s: %s", job_id, e) + status = None + + if status is None: + self._mark_error(download_id, "Usenet job disappeared from client") + return + + with self._lock: + row = self.active_downloads.get(download_id) + if row is not None: + row['progress'] = status.progress * 100.0 + row['transferred'] = status.downloaded + row['speed'] = status.download_speed + row['size'] = status.size or row.get('size', 0) + row['state'] = _adapter_state_to_display(status.state) + row['error'] = status.error + if status.save_path: + last_save_path = status.save_path + + if status.state in _COMPLETE_STATES: + self._finalize_download(download_id, last_save_path) + return + if status.state == 'failed': + self._mark_error(download_id, status.error or "Usenet client reported failure") + return + + time.sleep(_POLL_INTERVAL_SECONDS) + + self._mark_error(download_id, "Usenet download timed out") + + def _finalize_download(self, download_id: str, save_path: Optional[str]) -> None: + if not save_path: + self._mark_error(download_id, "Usenet job completed but no save_path reported") + return + try: + audio_files = collect_audio_after_extraction(Path(save_path)) + except Exception as e: + self._mark_error(download_id, f"Post-extract walk failed: {e}") + return + if not audio_files: + self._mark_error(download_id, f"No audio files found in {save_path}") + return + primary = audio_files[0] + with self._lock: + row = self.active_downloads.get(download_id) + if row is not None: + row['state'] = 'Completed, Succeeded' + row['progress'] = 100.0 + row['file_path'] = str(primary) + row['audio_files'] = [str(path) for path in audio_files] + logger.info("Usenet download complete: %s -> %s (%d audio files)", + download_id[:8], primary.name, len(audio_files)) + + def _mark_error(self, download_id: str, message: str) -> None: + logger.error("Usenet download %s failed: %s", download_id[:8], message) + with self._lock: + row = self.active_downloads.get(download_id) + if row is not None: + row['state'] = 'Completed, Errored' + row['error'] = message + + # ------------------------------------------------------------------ + # Status / lifecycle + # ------------------------------------------------------------------ + + async def get_all_downloads(self) -> List[DownloadStatus]: + with self._lock: + rows = list(self.active_downloads.values()) + return [_row_to_status(r) for r in rows] + + async def get_download_status(self, download_id: str) -> Optional[DownloadStatus]: + with self._lock: + row = self.active_downloads.get(download_id) + if row is None: + return None + return _row_to_status(row) + + async def cancel_download( + self, + download_id: str, + username: Optional[str] = None, + remove: bool = False, + ) -> bool: + adapter = get_active_usenet_adapter() + with self._lock: + row = self.active_downloads.get(download_id) + job_id = row.get('job_id') if row else None + if adapter and job_id: + try: + await adapter.remove(job_id, delete_files=remove) + except Exception as e: + logger.warning("Usenet cancel via adapter failed: %s", e) + with self._lock: + if remove: + self.active_downloads.pop(download_id, None) + else: + row = self.active_downloads.get(download_id) + if row is not None: + row['state'] = 'Cancelled' + return True + + async def clear_all_completed_downloads(self) -> bool: + with self._lock: + for did in list(self.active_downloads.keys()): + state = self.active_downloads[did].get('state', '') + if state.startswith('Completed') or state == 'Cancelled': + self.active_downloads.pop(did, None) + return True + + # ------------------------------------------------------------------ + # Album-bundle flow + # ------------------------------------------------------------------ + + def download_album_to_staging( + self, + album_name: str, + artist_name: str, + staging_dir: str, + progress_callback=None, + ) -> Dict[str, Any]: + """Usenet sibling of ``TorrentDownloadPlugin.download_album_to_staging``. + See that method's docstring for the contract.""" + result: Dict[str, Any] = {'success': False, 'files': [], 'error': None} + if not self.is_configured(): + result['error'] = 'Usenet source not configured' + return result + + adapter = get_active_usenet_adapter() + if adapter is None or not adapter.is_configured(): + result['error'] = 'No active usenet client' + return result + + def _emit(state: str, **extra) -> None: + if progress_callback: + try: + progress_callback({'state': state, **extra}) + except Exception as cb_exc: + logger.debug("[Usenet album] progress callback failed: %s", cb_exc) + + query = f"{artist_name} {album_name}".strip() + _emit('searching', query=query) + try: + search_results = run_async(self._prowlarr.search( + query, categories=DEFAULT_MUSIC_CATEGORIES, + indexer_ids=_parse_indexer_id_filter(), + )) + except Exception as e: + result['error'] = f'Prowlarr search failed: {e}' + return result + + candidates = [r for r in search_results + if r.protocol == 'usenet' and r.download_url] + if not candidates: + result['error'] = f'No usenet results found for "{query}"' + return result + + picked = pick_best_album_release(candidates, _guess_quality_from_title) + if picked is None: + result['error'] = 'No suitable NZB candidate after filtering' + return result + + logger.info("[Usenet album] Picked '%s' (size=%.1fMB grabs=%s indexer=%s)", + picked.title, picked.size / 1_048_576, picked.grabs, picked.indexer_name) + _emit('queued', release=picked.title, size=picked.size, grabs=picked.grabs) + + try: + job_id = run_async(adapter.add_nzb(picked.download_url)) + except Exception as e: + result['error'] = f'Usenet client refused the NZB: {e}' + return result + if not job_id: + result['error'] = 'Usenet client refused the NZB' + return result + + _emit('downloading', release=picked.title) + save_path = self._poll_album_download(adapter, job_id, picked.title, _emit) + if save_path is None: + result['error'] = 'Usenet download failed or timed out' + return result + + _emit('staging', release=picked.title) + try: + audio_files = collect_audio_after_extraction(Path(save_path)) + except Exception as e: + result['error'] = f'Failed to walk audio files: {e}' + return result + if not audio_files: + result['error'] = f'No audio files found in {save_path}' + return result + + copied = copy_audio_files_atomically(audio_files, Path(staging_dir)) + if not copied: + result['error'] = 'No audio files copied to staging' + return result + logger.info("[Usenet album] Staged %d audio files for '%s'", len(copied), album_name) + _emit('staged', count=len(copied)) + result['success'] = True + result['files'] = copied + return result + + def _poll_album_download(self, adapter, job_id, title, emit) -> Optional[str]: + deadline = time.monotonic() + _POLL_TIMEOUT_SECONDS + last_save_path: Optional[str] = None + while time.monotonic() < deadline: + if self.shutdown_check and self.shutdown_check(): + return None + try: + status = run_async(adapter.get_status(job_id)) + except Exception as e: + logger.warning("[Usenet album] Poll error: %s", e) + status = None + if status is None: + logger.error("[Usenet album] '%s' disappeared from client", title) + return None + emit('downloading', progress=status.progress, downloaded=status.downloaded, + speed=status.download_speed) + if status.save_path: + last_save_path = status.save_path + if status.state in _COMPLETE_STATES: + return last_save_path + if status.state == 'failed': + logger.error("[Usenet album] '%s' failed: %s", title, status.error) + return None + time.sleep(_POLL_INTERVAL_SECONDS) + logger.error("[Usenet album] '%s' timed out", title) + return None diff --git a/core/downloads/album_bundle_dispatch.py b/core/downloads/album_bundle_dispatch.py new file mode 100644 index 00000000..6a74914f --- /dev/null +++ b/core/downloads/album_bundle_dispatch.py @@ -0,0 +1,191 @@ +"""Album-bundle dispatch for torrent / usenet single-source downloads. + +Lifted from ``run_full_missing_tracks_process`` so the master +worker doesn't carry a 90-line inline branch and so the gate logic +can be unit-tested in isolation. + +The gate fires only when ALL conditions hold: + +- Batch is an album-context download (``is_album_download`` flag). +- Active download source is ``torrent`` or ``usenet`` (single-source + mode — hybrid stays per-track to preserve fallback). +- Both album-name and artist-name are populated in batch context. +- The resolved plugin exposes ``download_album_to_staging``. + +When the gate engages it runs the plugin synchronously (the master +worker is already on a thread-pool executor) and mirrors the +plugin's lifecycle payloads into the batch state so the Downloads +page can render meaningful progress before per-track tasks exist. + +Return semantics: ``True`` means the gate handled the batch — the +master worker should stop and not run per-track analysis. ``False`` +means the gate didn't engage (or engaged-and-fell-back) — caller +continues the normal per-track flow. + +The ``BatchStateAccess`` Protocol exists so this module doesn't +import ``download_batches`` from runtime_state directly. The +caller (master worker) injects accessors so this module stays +testable without touching live runtime state. +""" + +from __future__ import annotations + +import logging +from pathlib import Path +from typing import Any, Callable, Optional, Protocol + +logger = logging.getLogger(__name__) + + +class BatchStateAccess(Protocol): + """Narrow shim around the batch-state dict ops the dispatch needs. + + Two methods to keep the surface small: + - ``update_fields(batch_id, fields)`` — atomic merge into the + batch dict under tasks_lock. + - ``mark_failed(batch_id, error)`` — convenience for the failure + path (sets phase + error + album_bundle_state in one shot). + """ + + def update_fields(self, batch_id: str, fields: dict) -> None: ... + + def mark_failed(self, batch_id: str, error: str) -> None: ... + + +# Fields the album-bundle progress callback may carry. Anything in +# this set gets mirrored onto the batch row as ``album_bundle_`` +# so the Downloads page can render it without coupling to the +# specific payload shape. +_MIRRORED_KEYS = ('progress', 'release', 'speed', 'downloaded', + 'size', 'seeders', 'grabs', 'count') + + +def is_eligible( + *, + mode: str, + is_album: bool, + album_name: str, + artist_name: str, +) -> bool: + """Pure predicate: does this batch even qualify for the album + flow? Separate from the resolution+run step so tests can pin + the gate logic without standing up a plugin.""" + if not is_album: + return False + if (mode or '').lower() not in ('torrent', 'usenet'): + return False + if not (album_name or '').strip(): + return False + if not (artist_name or '').strip(): + return False + return True + + +def try_dispatch( + *, + batch_id: str, + is_album: bool, + album_context: Optional[dict], + artist_context: Optional[dict], + config_get: Callable[..., Any], + plugin_resolver: Callable[[str], Optional[Any]], + state: BatchStateAccess, +) -> bool: + """Attempt the album-bundle flow. Returns ``True`` iff the + master worker should return early (gate engaged and completed + — success OR failure). ``False`` means fall through to the + normal per-track flow. + + ``config_get`` is a callable shaped like ``config_manager.get``; + ``plugin_resolver`` resolves a source-name string to an + initialised plugin instance (or None); ``state`` is the + BatchStateAccess shim. Injecting these keeps the module + dependency-light + unit-testable. + """ + mode = (config_get('download_source.mode', 'soulseek') or 'soulseek').lower() + album_name = (album_context or {}).get('name') or '' + artist_name = (artist_context or {}).get('name') or '' + + if not is_eligible(mode=mode, is_album=is_album, + album_name=album_name, artist_name=artist_name): + return False + + album_name = album_name.strip() + artist_name = artist_name.strip() + + plugin = None + try: + plugin = plugin_resolver(mode) + except Exception as exc: + logger.warning("[Album Bundle] Could not resolve %s plugin: %s", mode, exc) + + if plugin is None or not hasattr(plugin, 'download_album_to_staging'): + logger.warning( + "[Album Bundle] Gate matched but plugin / context unavailable " + "(mode=%s album=%r artist=%r plugin=%s) — falling back to per-track flow", + mode, album_name, artist_name, + type(plugin).__name__ if plugin else None, + ) + return False + + staging_root = config_get( + 'download_source.album_bundle_staging_path', + 'storage/album_bundle_staging', + ) or 'storage/album_bundle_staging' + staging_dir = str(Path(staging_root) / _safe_batch_dirname(batch_id)) + logger.info( + "[Album Bundle] Engaging %s album flow for '%s' by '%s' -> %s", + mode, album_name, artist_name, staging_dir, + ) + state.update_fields(batch_id, { + 'phase': 'album_downloading', + 'album_bundle_state': 'searching', + 'album_bundle_source': mode, + 'album_bundle_staging_path': staging_dir, + 'album_bundle_private_staging': True, + }) + + def _emit(payload): + """Mirror plugin lifecycle into batch state for UI rendering.""" + try: + fields = {'album_bundle_state': payload.get('state', '')} + for key in _MIRRORED_KEYS: + if key in payload: + fields[f'album_bundle_{key}'] = payload[key] + state.update_fields(batch_id, fields) + except Exception as exc: + logger.debug("[Album Bundle] emit failed: %s", exc) + + try: + outcome = plugin.download_album_to_staging( + album_name, artist_name, staging_dir, _emit, + ) + except Exception as exc: + logger.exception("[Album Bundle] %s plugin raised: %s", mode, exc) + outcome = {'success': False, 'error': f'Plugin error: {exc}'} + + if not outcome.get('success'): + err = outcome.get('error', 'Album bundle download failed') + logger.error("[Album Bundle] %s flow failed for '%s': %s", + mode, album_name, err) + state.mark_failed(batch_id, err) + return True + + logger.info( + "[Album Bundle] %s staged %d files for '%s' — handing off to per-track staging matcher", + mode, len(outcome.get('files', [])), album_name, + ) + state.update_fields(batch_id, { + 'phase': 'analysis', + 'album_bundle_state': 'staged', + }) + # Engaged-and-succeeded: we DON'T early-return because the + # per-track flow needs to run to create + complete the per-track + # task rows. Those tasks will hit try_staging_match and pull the + # files we just staged. + return False + + +def _safe_batch_dirname(batch_id: str) -> str: + safe = ''.join(ch if ch.isalnum() or ch in ('-', '_') else '_' for ch in str(batch_id or 'batch')) + return safe or 'batch' diff --git a/core/downloads/lifecycle.py b/core/downloads/lifecycle.py index 2a29feec..5891e8be 100644 --- a/core/downloads/lifecycle.py +++ b/core/downloads/lifecycle.py @@ -26,9 +26,11 @@ Lifted verbatim from web_server.py. Dependencies injected via from __future__ import annotations import logging +import shutil import time import traceback from dataclasses import dataclass +from pathlib import Path from typing import Any, Callable, Optional from core.downloads.history import record_sync_history_completion @@ -42,6 +44,47 @@ from core.runtime_state import ( logger = logging.getLogger(__name__) +def _safe_batch_dirname(batch_id: str) -> str: + safe = ''.join(ch if ch.isalnum() or ch in ('-', '_') else '_' for ch in str(batch_id or 'batch')) + return safe or 'batch' + + +def _cleanup_private_album_bundle_staging(batch_id: str, batch: dict) -> None: + """Best-effort cleanup for torrent/usenet private staging copies. + + The torrent/usenet clients keep their own completed download folders. + This only removes SoulSync's per-batch copy under album-bundle staging. + """ + if not batch.get('album_bundle_private_staging'): + return + if (batch.get('album_bundle_source') or '').lower() not in ('torrent', 'usenet'): + return + + staging_path = batch.get('album_bundle_staging_path') + if not staging_path: + return + + path = Path(staging_path) + expected_name = _safe_batch_dirname(batch_id) + if path.name != expected_name: + logger.warning( + "[Album Bundle] Refusing to clean private staging path with unexpected name: %s", + staging_path, + ) + return + if not path.exists(): + return + if not path.is_dir(): + logger.warning("[Album Bundle] Refusing to clean non-directory staging path: %s", staging_path) + return + + try: + shutil.rmtree(path) + logger.info("[Album Bundle] Cleaned private staging folder for batch %s: %s", batch_id, staging_path) + except Exception as exc: + logger.warning("[Album Bundle] Could not clean private staging folder %s: %s", staging_path, exc) + + @dataclass class LifecycleDeps: """Bundle of cross-cutting deps the batch lifecycle needs.""" @@ -398,6 +441,7 @@ def on_download_completed(batch_id: str, task_id: str, success: bool, deps: Life logger.info(f"[Batch Manager] Batch {batch_id} complete - stopping monitor") deps.download_monitor.stop_monitoring(batch_id) + _cleanup_private_album_bundle_staging(batch_id, batch) # M3U REGENERATION: Regenerate M3U with real library paths now that # all post-processing (tagging, moving, DB writes) is complete. @@ -606,6 +650,7 @@ def check_batch_completion_v2(batch_id: str, deps: LifecycleDeps) -> Optional[bo logger.info(f"[Completion Check V2] Batch {batch_id} complete - stopping monitor") deps.download_monitor.stop_monitoring(batch_id) + _cleanup_private_album_bundle_staging(batch_id, batch) # REPAIR: Scan all album folders from this batch for track number issues if deps.repair_worker: diff --git a/core/downloads/master.py b/core/downloads/master.py index 118db159..90adbb4e 100644 --- a/core/downloads/master.py +++ b/core/downloads/master.py @@ -35,6 +35,7 @@ from difflib import SequenceMatcher from pathlib import Path from typing import Any, Callable +from core.downloads import album_bundle_dispatch as _album_bundle_dispatch from core.runtime_state import download_batches, download_tasks, tasks_lock logger = logging.getLogger(__name__) @@ -270,6 +271,26 @@ class MasterDeps: reset_wishlist_auto_processing: Callable[[], None] +class _BatchStateAccessImpl: + """Concrete ``BatchStateAccess`` for the runtime ``download_batches`` + dict — wraps the lock + the existing-batch check so the album- + bundle dispatcher stays decoupled from runtime_state.""" + + def update_fields(self, batch_id: str, fields: dict) -> None: + with tasks_lock: + row = download_batches.get(batch_id) + if row is not None: + row.update(fields) + + def mark_failed(self, batch_id: str, error: str) -> None: + with tasks_lock: + row = download_batches.get(batch_id) + if row is not None: + row['phase'] = 'failed' + row['error'] = error + row['album_bundle_state'] = 'failed' + + def run_full_missing_tracks_process(batch_id, playlist_id, tracks_json, deps: MasterDeps): """ A master worker that handles the entire missing tracks process: @@ -311,6 +332,23 @@ def run_full_missing_tracks_process(batch_id, playlist_id, tracks_json, deps: Ma if force_download_all: logger.warning(f"[Force Download] Force download mode enabled for batch {batch_id} - treating all tracks as missing") + # Album-bundle gate for torrent / usenet single-source mode. + # See ``core/downloads/album_bundle_dispatch`` for the full + # narrow-gate rationale. Returns True iff the master worker + # should stop (gate fired and failed); False = engaged-and- + # succeeded OR didn't engage, both fall through to per-track. + _bundle_state = _BatchStateAccessImpl() + if _album_bundle_dispatch.try_dispatch( + batch_id=batch_id, + is_album=batch_is_album, + album_context=batch_album_context, + artist_context=batch_artist_context, + config_get=deps.config_manager.get, + plugin_resolver=deps.download_orchestrator.client, + state=_bundle_state, + ): + return + # Allow duplicate tracks across albums — when enabled, only skip tracks already # owned in THIS album, not tracks owned in other albums allow_duplicates = deps.config_manager.get('wishlist.allow_duplicate_tracks', True) diff --git a/core/downloads/monitor.py b/core/downloads/monitor.py index 62d515e5..c4b645f5 100644 --- a/core/downloads/monitor.py +++ b/core/downloads/monitor.py @@ -34,6 +34,32 @@ _start_next_batch_of_downloads = None _orphaned_download_keys = None missing_download_executor = None download_orchestrator = None +_RELEASE_SOURCE_NAMES = frozenset(('torrent', 'usenet')) + + +def _download_id_key(download_id): + return f"download_id::{download_id}" if download_id else None + + +def _is_release_task(task): + ti = task.get('track_info') if isinstance(task.get('track_info'), dict) else {} + username = task.get('username') or ti.get('username') + return username in _RELEASE_SOURCE_NAMES + + +def _lookup_live_info(task, live_transfers_lookup): + ti = task.get('track_info') if isinstance(task.get('track_info'), dict) else {} + download_id = task.get('download_id') + if _is_release_task(task): + by_id = live_transfers_lookup.get(_download_id_key(download_id)) + if by_id: + return by_id + + task_filename = task.get('filename') or ti.get('filename') + task_username = task.get('username') or ti.get('username') + if not task_filename or not task_username: + return None + return live_transfers_lookup.get(_make_context_key(task_username, task_filename)) def init( @@ -139,55 +165,64 @@ class WebUIDownloadMonitor: for task_id in download_batches[batch_id].get('queue', []): task = download_tasks.get(task_id) - if not task or task['status'] not in ['downloading', 'queued']: + if not task: + continue + release_recoverable = ( + _is_release_task(task) + and task.get('download_id') + and task.get('status') in ['failed', 'not_found'] + ) + if task['status'] not in ['downloading', 'queued'] and not release_recoverable: continue # Check for timeouts and errors - retries handled directly in _should_retry_task # If _should_retry_task returns True, it means retries were exhausted - retry_exhausted = self._should_retry_task(task_id, task, live_transfers_lookup, current_time, deferred_ops) + retry_exhausted = False + if not release_recoverable: + retry_exhausted = self._should_retry_task(task_id, task, live_transfers_lookup, current_time, deferred_ops) # Collect exhausted tasks to handle outside lock (prevents deadlock) if retry_exhausted: exhausted_tasks.append((batch_id, task_id)) - # ENHANCED: Check for successful completions (especially YouTube) - task_filename = task.get('filename') or task.get('track_info', {}).get('filename') - task_username = task.get('username') or task.get('track_info', {}).get('username') + # ENHANCED: Check for successful completions (especially YouTube). + # Release-style sources can report a completed audio file + # name that differs from the original indexer URL/title + # stored on the task, so prefer the stable download_id. + live_info = _lookup_live_info(task, live_transfers_lookup) - if task_filename and task_username: - lookup_key = _make_context_key(task_username, task_filename) - live_info = live_transfers_lookup.get(lookup_key) - - if live_info: - state = live_info.get('state', '') - # Trigger post-processing if download is completed successfully - # slskd uses compound states like 'Completed, Succeeded' - use substring matching - # Must exclude error states first (matching _build_batch_status_data's prioritized checking) - has_error = ('Errored' in state or 'Failed' in state or 'Rejected' in state or 'TimedOut' in state) - has_completion = ('Completed' in state or 'Succeeded' in state) - # Verify bytes actually transferred before trusting state string. - # slskd can report "Completed" before the full file is flushed to disk, - # or on connection drops that leave a partial file. - if has_completion and not has_error: - expected_size = live_info.get('size', 0) - transferred = live_info.get('bytesTransferred', 0) - if expected_size > 0 and transferred < expected_size: - if not task.get('_incomplete_warned'): - logger.debug(f"Monitor: {task_id} state={state} but bytes incomplete ({transferred}/{expected_size}) — waiting") - task['_incomplete_warned'] = True - continue - if has_completion and not has_error and task['status'] == 'downloading': - task.pop('_incomplete_warned', None) - # CRITICAL FIX: Transition to 'post_processing' HERE so downloads - # don't depend on browser polling to trigger post-processing. - # Previously, post-processing was only submitted by _build_batch_status_data - # (called from browser-polled endpoints), meaning closing the browser - # left tasks stuck in 'downloading' forever. - task['status'] = 'post_processing' - task['status_change_time'] = current_time - logger.info(f"Monitor detected completed download for {task_id} ({state}) - submitting post-processing") - # Collect for handling outside the lock to prevent deadlock. - # _on_download_completed acquires tasks_lock which is non-reentrant. - completed_tasks.append((batch_id, task_id)) + if live_info: + state = live_info.get('state', '') + # Trigger post-processing if download is completed successfully + # slskd uses compound states like 'Completed, Succeeded' - use substring matching + # Must exclude error states first (matching _build_batch_status_data's prioritized checking) + has_error = ('Errored' in state or 'Failed' in state or 'Rejected' in state or 'TimedOut' in state) + has_completion = ('Completed' in state or 'Succeeded' in state) + # Verify bytes actually transferred before trusting state string. + # slskd can report "Completed" before the full file is flushed to disk, + # or on connection drops that leave a partial file. + if has_completion and not has_error: + expected_size = live_info.get('size', 0) + transferred = live_info.get('bytesTransferred', 0) + if expected_size > 0 and transferred < expected_size: + if not task.get('_incomplete_warned'): + logger.debug("Monitor: %s state=%s but bytes incomplete (%s/%s) - waiting", task_id, state, transferred, expected_size) + task['_incomplete_warned'] = True + continue + if has_completion and not has_error and ( + task['status'] == 'downloading' or release_recoverable + ): + task.pop('_incomplete_warned', None) + # CRITICAL FIX: Transition to 'post_processing' HERE so downloads + # don't depend on browser polling to trigger post-processing. + # Previously, post-processing was only submitted by _build_batch_status_data + # (called from browser-polled endpoints), meaning closing the browser + # left tasks stuck in 'downloading' forever. + task['status'] = 'post_processing' + task['status_change_time'] = current_time + logger.info(f"Monitor detected completed download for {task_id} ({state}) - submitting post-processing") + # Collect for handling outside the lock to prevent deadlock. + # _on_download_completed acquires tasks_lock which is non-reentrant. + completed_tasks.append((batch_id, task_id)) # ---- All work below runs WITHOUT tasks_lock held ---- if globals().get('IS_SHUTTING_DOWN', False) or not self.monitoring: @@ -308,7 +343,7 @@ class WebUIDownloadMonitor: for download in all_downloads: key = _make_context_key(download.username, download.filename) # Convert DownloadStatus to transfer dict format for monitor compatibility - live_transfers[key] = { + transfer_row = { 'id': download.id, 'filename': download.filename, 'username': download.username, @@ -318,6 +353,10 @@ class WebUIDownloadMonitor: 'bytesTransferred': download.transferred, 'averageSpeed': download.speed, } + live_transfers[key] = transfer_row + id_key = _download_id_key(download.id) + if id_key: + live_transfers[id_key] = transfer_row except Exception as yt_error: logger.error(f"Monitor: Could not fetch streaming source downloads: {yt_error}") @@ -352,7 +391,7 @@ class WebUIDownloadMonitor: return False lookup_key = _make_context_key(task_username, task_filename) - live_info = live_transfers_lookup.get(lookup_key) + live_info = _lookup_live_info(task, live_transfers_lookup) if not live_info: # User-initiated manual pick — skip auto-retry. The status diff --git a/core/downloads/post_processing.py b/core/downloads/post_processing.py index 1b2d04d2..2c17e90f 100644 --- a/core/downloads/post_processing.py +++ b/core/downloads/post_processing.py @@ -19,9 +19,11 @@ from __future__ import annotations import logging import os +import shutil import time import traceback from dataclasses import dataclass +from difflib import SequenceMatcher from pathlib import Path from typing import Any, Callable, Optional @@ -34,7 +36,7 @@ from core.imports.context import ( get_import_original_search, normalize_import_context, ) -from core.imports.filename import extract_track_number_from_filename +from core.imports.filename import extract_track_number_from_filename, parse_filename_metadata from core.metadata import enrichment as metadata_enrichment from core.runtime_state import ( download_tasks, @@ -46,6 +48,85 @@ from core.runtime_state import ( logger = logging.getLogger(__name__) +_AUDIO_EXTENSIONS = { + '.flac', '.ape', '.wav', '.alac', '.dsf', '.dff', '.aiff', '.aif', + '.opus', '.ogg', '.m4a', '.aac', '.mp3', '.wma', +} + + +def _is_audio_file(path: str) -> bool: + return Path(str(path or '')).suffix.lower() in _AUDIO_EXTENSIONS + + +def _reject_non_audio_found_file(found_file: Optional[str], file_location: Optional[str]) -> tuple[Optional[str], Optional[str]]: + if found_file and not _is_audio_file(found_file): + logger.warning( + "[Post-Processing] Ignoring non-audio candidate found during file search: %s", + found_file, + ) + return None, None + return found_file, file_location + + +def _normalize_match_text(value: str) -> str: + return ''.join(ch.lower() for ch in str(value or '') if ch.isalnum()) + + +def _release_audio_match_score(path: str, expected_title: str, expected_artist: str) -> float: + parsed = parse_filename_metadata(path) + parsed_title = parsed.get('title') or Path(path).stem + parsed_artist = parsed.get('artist') or '' + expected_title_norm = _normalize_match_text(expected_title) + parsed_title_norm = _normalize_match_text(parsed_title) + if expected_title_norm and ( + expected_title_norm in parsed_title_norm or parsed_title_norm in expected_title_norm + ): + title_score = 1.0 + else: + title_score = SequenceMatcher( + None, + expected_title_norm, + parsed_title_norm, + ).ratio() + if expected_artist and parsed_artist: + artist_score = SequenceMatcher( + None, + _normalize_match_text(expected_artist), + _normalize_match_text(parsed_artist), + ).ratio() + return (title_score * 0.75) + (artist_score * 0.25) + return title_score + + +def _track_title_from_task(track_info: Any, context: Optional[dict]) -> str: + if isinstance(track_info, dict): + title = track_info.get('name') or track_info.get('title') + if title: + return str(title) + return get_import_clean_title(context or {}, default='') + + +def _copy_release_audio_to_transfer(source_path: str, transfer_dir: str) -> Optional[str]: + try: + src = Path(source_path) + if not src.exists() or not src.is_file(): + return None + dest_dir = Path(transfer_dir) + dest_dir.mkdir(parents=True, exist_ok=True) + dest = dest_dir / src.name + if dest.exists(): + stem, suffix = src.stem, src.suffix + counter = 1 + while dest.exists(): + dest = dest_dir / f"{stem}_release_{counter}{suffix}" + counter += 1 + shutil.copy2(src, dest) + return str(dest) + except Exception as exc: + logger.warning("[Post-Processing] Could not copy release audio to transfer: %s", exc) + return None + + @dataclass class PostProcessDeps: """Bundle of dependencies the post-processing worker needs. @@ -190,14 +271,71 @@ def run_post_processing_worker(task_id: str, batch_id: str, deps: PostProcessDep # CRITICAL FIX: For YouTube downloads, the filename in task is 'id||title' (metadata), # but the actual file on disk is 'Title.mp3'. We must ask the client for the real path. + # Torrent/usenet also use opaque "url||display" filenames. Their completed + # job may contain a full release, so choose the best matching audio file + # and copy it to transfer before importing instead of moving the client's + # original completed download. if (task.get('username') == 'youtube' or '||' in str(task_filename)) and not found_file: - logger.info(f"[Post-Processing] Detected YouTube download task: {task_id}") + logger.info(f"[Post-Processing] Detected engine-backed download task: {task_id}") try: # Query the download orchestrator for the status which contains the real file path # CRITICAL FIX: Use the actual download_id designated by the client, not the internal task_id actual_download_id = task.get('download_id') or task_id status = deps.run_async(deps.download_orchestrator.get_download_status(actual_download_id)) - if status and status.file_path: + if status and task.get('username') in ('torrent', 'usenet'): + audio_files = list(getattr(status, 'audio_files', None) or []) + if not audio_files and getattr(status, 'file_path', None): + audio_files = [status.file_path] + expected_title = _track_title_from_task(track_info, context) + expected_artist = '' + if isinstance(track_info, dict): + artists = track_info.get('artists', []) + if isinstance(artists, list) and artists: + first_artist = artists[0] + expected_artist = first_artist.get('name', '') if isinstance(first_artist, dict) else str(first_artist) + elif isinstance(artists, str): + expected_artist = artists + if not expected_artist and context: + artist_ctx = get_import_context_artist(context) + expected_artist = artist_ctx.get('name', '') if isinstance(artist_ctx, dict) else '' + + scored_files = [ + (_release_audio_match_score(path, expected_title, expected_artist), path) + for path in audio_files + if _is_audio_file(path) + ] + scored_files.sort(reverse=True) + if scored_files: + best_score, best_path = scored_files[0] + logger.info( + "[Post-Processing] Best %s release file for '%s': %s (score %.2f)", + task.get('username'), expected_title, best_path, best_score, + ) + if best_score >= 0.80: + copied_path = _copy_release_audio_to_transfer(best_path, transfer_dir) + if copied_path: + found_file = copied_path + file_location = 'download' + logger.info( + "[Post-Processing] Copied matched %s release file to transfer: %s", + task.get('username'), copied_path, + ) + else: + logger.warning( + "[Post-Processing] No %s release file met match threshold for '%s' (best %.2f)", + task.get('username'), expected_title, best_score, + ) + if not found_file: + with tasks_lock: + if task_id in download_tasks: + download_tasks[task_id]['status'] = 'failed' + download_tasks[task_id]['error_message'] = ( + f"No matching audio file found in {task.get('username')} release" + ) + deps.on_download_completed(batch_id, task_id, False) + return + + if status and status.file_path and not found_file and task.get('username') not in ('torrent', 'usenet'): real_path = status.file_path if os.path.exists(real_path): # Determine if it's in download or transfer directory @@ -227,11 +365,12 @@ def run_post_processing_worker(task_id: str, batch_id: str, deps: PostProcessDep logger.info(f"[Post-Processing] Resolved actual YouTube filename: {found_file} (Location: {file_location})") else: - logger.warning(f"[Post-Processing] YouTube status reported path but file missing: {real_path}") + logger.warning(f"[Post-Processing] Engine status reported path but file missing: {real_path}") else: - logger.warning(f"[Post-Processing] YouTube status returned no file_path for task {task_id}") + if not found_file: + logger.warning(f"[Post-Processing] Engine status returned no file_path for task {task_id}") except Exception as e: - logger.error(f"[Post-Processing] Failed to retrieve YouTube task status: {e}") + logger.error(f"[Post-Processing] Failed to retrieve engine task status: {e}") _file_search_max_retries = 5 for retry_count in range(_file_search_max_retries): @@ -257,6 +396,7 @@ def run_post_processing_worker(task_id: str, batch_id: str, deps: PostProcessDep # Strategy 1: Try with original filename in both downloads and transfer logger.info("[Post-Processing] Strategy 1: Searching with original filename...") found_file, file_location = deps.find_completed_file(download_dir, task_filename, transfer_dir) + found_file, file_location = _reject_non_audio_found_file(found_file, file_location) if found_file: logger.info(f"[Post-Processing] Strategy 1 SUCCESS: Found file with original filename in {file_location}: {found_file}") @@ -269,7 +409,11 @@ def run_post_processing_worker(task_id: str, batch_id: str, deps: PostProcessDep found_result = deps.find_completed_file(transfer_dir, expected_final_filename) if found_result and found_result[0]: found_file, file_location = found_result[0], 'transfer' - logger.info(f"[Post-Processing] Strategy 2 SUCCESS: Found file with expected final filename: {found_file}") + found_file, file_location = _reject_non_audio_found_file(found_file, file_location) + if found_file: + logger.info(f"[Post-Processing] Strategy 2 SUCCESS: Found file with expected final filename: {found_file}") + else: + logger.error("[Post-Processing] Strategy 2 FAILED: Expected final filename resolved to a non-audio file") else: logger.error("[Post-Processing] Strategy 2 FAILED: Expected final filename not found in transfer folder") elif not expected_final_filename: diff --git a/core/downloads/staging.py b/core/downloads/staging.py index 12de4a16..f7ca8906 100644 --- a/core/downloads/staging.py +++ b/core/downloads/staging.py @@ -34,9 +34,12 @@ from __future__ import annotations import logging import os +import re from dataclasses import dataclass from typing import Any, Callable +from core.imports.filename import extract_track_number_from_filename + # `shutil` and `SequenceMatcher` are imported inline inside try_staging_match() # to keep the lift byte-identical with the original web_server.py function body. @@ -50,6 +53,56 @@ from core.runtime_state import ( logger = logging.getLogger(__name__) +def _coerce_positive_int(value: Any, default: int = 0) -> int: + try: + coerced = int(str(value).split('/')[0]) + except (TypeError, ValueError): + return default + return coerced if coerced > 0 else default + + +def _staging_title_variants(title: Any, normalize: Callable[[str], str]) -> list[str]: + """Return conservative title variants for release-file matching. + + Torrent / usenet release files often encode featured artists in the + filename/title while streaming metadata keeps them in the artist credit. + Strip only feature/bonus noise here; keep version words like remix, + extended, live, acoustic, etc. so distinct recordings do not collapse. + """ + raw = str(title or '').strip() + if not raw: + return [] + + compacted_separators = re.sub(r'[_]+', ' ', raw) + compacted_separators = re.sub(r'\s+', ' ', compacted_separators).strip() + + without_feat = re.sub( + r'\s*[\(\[]\s*(?:feat\.?|ft\.?|featuring)\s+[^)\]]*[\)\]]', + '', + compacted_separators, + flags=re.IGNORECASE, + ) + without_feat = re.sub( + r'\s+(?:feat\.?|ft\.?|featuring)\s+.*$', + '', + without_feat, + flags=re.IGNORECASE, + ) + without_bonus = re.sub( + r'\s*[\(\[]\s*bonus\s+track\s*[\)\]]', + '', + without_feat, + flags=re.IGNORECASE, + ) + + variants: list[str] = [] + for candidate in (raw, compacted_separators, without_feat, without_bonus): + normalized = normalize(candidate) + if normalized and normalized not in variants: + variants.append(normalized) + return variants + + @dataclass class StagingDeps: """Bundle of cross-cutting deps the staging-match helper needs.""" @@ -58,6 +111,12 @@ class StagingDeps: get_staging_file_cache: Callable[[str], list] docker_resolve_path: Callable[[str], str] post_process_matched_download_with_verification: Callable + # Optional batch-field accessor. Returns ``download_batches[batch_id].get(field)`` + # when the runtime state is available, ``None`` otherwise. Injected so + # this module doesn't have to import from runtime_state directly — + # keeps the dep surface explicit and the function unit-testable + # without a live batch dict. + get_batch_field: Callable[[str, str], Any] = None # type: ignore[assignment] def try_staging_match(task_id, batch_id, track, deps: StagingDeps): @@ -80,19 +139,24 @@ def try_staging_match(task_id, batch_id, track, deps: StagingDeps): normalize = deps.matching_engine.normalize_string norm_title = normalize(track_title) norm_artist = normalize(track_artist) + title_variants = _staging_title_variants(track_title, normalize) or [norm_title] best_match = None best_score = 0.0 for sf in staging_files: - sf_norm_title = normalize(sf['title']) + sf_title_variants = _staging_title_variants(sf['title'], normalize) sf_norm_artist = normalize(sf['artist']) - if not sf_norm_title: + if not sf_title_variants: continue # Title similarity (primary) - title_sim = SequenceMatcher(None, norm_title, sf_norm_title).ratio() + title_sim = max( + SequenceMatcher(None, expected, candidate).ratio() + for expected in title_variants + for candidate in sf_title_variants + ) if title_sim < 0.80: continue @@ -141,20 +205,51 @@ def try_staging_match(task_id, batch_id, track, deps: StagingDeps): shutil.copy2(best_match['full_path'], dest_path) logger.info(f"[Staging] Copied to transfer: {dest_path}") - # Mark task as completed with staging context + # Mark task as completed with staging context. + # If the batch was populated by the torrent / usenet album-bundle + # flow, prefer that provenance label over generic 'staging' so the + # download history reflects the real source. The accessor is + # injected via StagingDeps so this module doesn't reach into + # runtime_state directly (see deps.get_batch_field docstring). + _provenance_override = None + if batch_id and deps.get_batch_field is not None: + try: + _provenance_override = deps.get_batch_field(batch_id, 'album_bundle_source') + except Exception as _exc: + logger.debug("get_batch_field failed: %s", _exc) + _provenance_username = _provenance_override or 'staging' + _private_album_bundle_staging = False + if batch_id and deps.get_batch_field is not None: + try: + _private_album_bundle_staging = bool( + deps.get_batch_field(batch_id, 'album_bundle_private_staging') + ) + except Exception as _exc: + logger.debug("get_batch_field failed: %s", _exc) with tasks_lock: if task_id in download_tasks: download_tasks[task_id]['status'] = 'post_processing' download_tasks[task_id]['filename'] = dest_path - download_tasks[task_id]['username'] = 'staging' + download_tasks[task_id]['username'] = _provenance_username download_tasks[task_id]['staging_match'] = True + if _private_album_bundle_staging: + try: + os.remove(best_match['full_path']) + logger.debug("[Staging] Removed private album-bundle staging file: %s", best_match['full_path']) + except FileNotFoundError: + pass + except Exception as _exc: + logger.debug("[Staging] Could not remove private album-bundle staging file: %s", _exc) + # Run post-processing (tagging, AcoustID verification, path building) context_key = f"staging_{task_id}" with tasks_lock: track_info = download_tasks.get(task_id, {}).get('track_info', {}) if not isinstance(track_info, dict): track_info = {} + else: + track_info = dict(track_info) # Build spotify_artist / spotify_album context so post-processing can apply # the path template. Without these, _post_process_matched_download returns @@ -223,20 +318,37 @@ def try_staging_match(task_id, batch_id, track, deps: StagingDeps): ) has_clean_data = bool(track_title and track_artist and track_album_name) - track_number = ( - track_info.get('track_number', 0) or - getattr(track, 'track_number', 0) or 0 - ) - disc_number = ( - track_info.get('disc_number', 1) or - getattr(track, 'disc_number', 1) or 1 + file_track_number = ( + _coerce_positive_int(best_match.get('track_number'), 0) or + extract_track_number_from_filename(best_match.get('full_path', '')) ) + file_disc_number = _coerce_positive_int(best_match.get('disc_number'), 1) + if _private_album_bundle_staging: + track_number = file_track_number + disc_number = file_disc_number + else: + track_number = ( + _coerce_positive_int(track_info.get('track_number'), 0) or + _coerce_positive_int(track_info.get('trackNumber'), 0) or + _coerce_positive_int(getattr(track, 'track_number', 0), 0) or + file_track_number + ) + disc_number = ( + _coerce_positive_int(track_info.get('disc_number'), 0) or + _coerce_positive_int(track_info.get('discNumber'), 0) or + _coerce_positive_int(getattr(track, 'disc_number', 0), 0) or + file_disc_number + ) + track_info['track_number'] = track_number + track_info['disc_number'] = disc_number context = { 'track_info': track_info, 'spotify_artist': spotify_artist_ctx, 'spotify_album': spotify_album_ctx, 'original_search_result': { + 'username': _provenance_username, + 'filename': best_match.get('full_path', ''), 'title': track_title, 'artist': track_artist, 'spotify_clean_title': track_title, diff --git a/core/downloads/status.py b/core/downloads/status.py index 7a0ec08c..6e11a763 100644 --- a/core/downloads/status.py +++ b/core/downloads/status.py @@ -88,7 +88,9 @@ class StatusDeps: # slskd's live_transfers path and must NOT hit the engine fallback. _STREAMING_SOURCE_NAMES = frozenset(( 'youtube', 'tidal', 'qobuz', 'hifi', 'deezer_dl', 'lidarr', 'soundcloud', 'amazon', + 'torrent', 'usenet', )) +_RELEASE_SOURCE_NAMES = frozenset(('torrent', 'usenet')) # Keep these in sync with the engine plugins' state strings. _ENGINE_FAILURE_STATES = ('Errored', 'Failed', 'Rejected', 'TimedOut', 'Aborted') @@ -136,21 +138,12 @@ def _apply_engine_state_fallback( Mutates ``task`` in place (status / error_message) the same way the Soulseek branch does, so the next status poll sees the new state. - Submits post-processing on terminal success and fires - ``on_download_completed`` on terminal failure to free the worker - slot. + Submits post-processing on terminal success. Manual-pick failures + are completed here; automatic failures keep the existing retry path + in charge. """ if deps.download_orchestrator is None or deps.run_async is None: return - if task.get('status') in ('completed', 'failed', 'cancelled', 'not_found', 'post_processing'): - return - # Scope this fallback to user-initiated manual picks. Auto attempts - # already flow through the live_transfers_lookup IF branch (the engine - # pre-populates non-Soulseek records via get_all_downloads), and on - # failure the monitor's existing retry path picks the next candidate. - # Marking auto attempts failed here would short-circuit that fallback. - if not task.get('_user_manual_pick'): - return download_id = task.get('download_id') if not download_id: return @@ -158,6 +151,17 @@ def _apply_engine_state_fallback( username = task.get('username') or ti.get('username') if username not in _STREAMING_SOURCE_NAMES: return + manual_pick = bool(task.get('_user_manual_pick')) + if not manual_pick and username not in _RELEASE_SOURCE_NAMES: + return + release_source = username in _RELEASE_SOURCE_NAMES + terminal_status = task.get('status') in ('completed', 'failed', 'cancelled', 'not_found', 'post_processing') + if terminal_status and not ( + release_source + and task.get('status') in ('failed', 'not_found') + and download_id + ): + return try: record = deps.run_async( @@ -189,6 +193,14 @@ def _apply_engine_state_fallback( return if any(s in state_str for s in _ENGINE_FAILURE_STATES): + if not manual_pick: + task_status['status'] = task.get('status', 'downloading') + task_status['progress'] = _engine_progress_pct(record) + logger.info( + "[Engine Fallback] Task %s engine reports '%s' for auto %s attempt; leaving retry handling to monitor", + task_id, state_str, username, + ) + return if task['status'] != 'failed': task['status'] = 'failed' err = getattr(record, 'error_message', None) or getattr(record, 'error', None) or '' @@ -247,6 +259,9 @@ def build_batch_status_data(batch_id: str, batch: dict, live_transfers_lookup: d "playlist_id": batch.get('playlist_id'), # Include playlist_id for rehydration "playlist_name": batch.get('playlist_name'), # Include playlist_name for reference } + album_bundle = _build_album_bundle_status(batch) + if album_bundle: + response_data['album_bundle'] = album_bundle if response_data["phase"] == 'analysis': response_data['analysis_progress'] = { @@ -318,6 +333,17 @@ def build_batch_status_data(batch_id: str, batch: dict, live_transfers_lookup: d _ti = task.get('track_info') if isinstance(task.get('track_info'), dict) else {} task_filename = task.get('filename') or _ti.get('filename') task_username = task.get('username') or _ti.get('username') + if ( + task_username in _RELEASE_SOURCE_NAMES + and task['status'] not in ['completed', 'cancelled', 'post_processing'] + and task.get('download_id') + ): + _apply_engine_state_fallback( + task_id, task, task_status, batch_id, deps, + ) + batch_tasks.append(task_status) + continue + if task_filename and task_username: lookup_key = deps.make_context_key(task_username, task_filename) @@ -610,7 +636,7 @@ def build_unified_downloads_response(limit: int, deps: StatusDeps) -> dict: for bid, batch in download_batches.items(): queue = batch.get('queue', []) statuses = [download_tasks[tid]['status'] for tid in queue if tid in download_tasks] - batch_summaries.append({ + summary = { 'batch_id': bid, 'playlist_id': batch.get('playlist_id', ''), 'batch_name': batch.get('playlist_name') or batch.get('album_name') or '', @@ -621,7 +647,11 @@ def build_unified_downloads_response(limit: int, deps: StatusDeps) -> dict: 'failed': sum(1 for s in statuses if s in ('failed', 'not_found', 'cancelled')), 'active': sum(1 for s in statuses if s in ('downloading', 'searching', 'post_processing')), 'queued': sum(1 for s in statuses if s in ('queued', 'pending')), - }) + } + album_bundle = _build_album_bundle_status(batch) + if album_bundle: + summary['album_bundle'] = album_bundle + batch_summaries.append(summary) return { 'success': True, @@ -630,3 +660,38 @@ def build_unified_downloads_response(limit: int, deps: StatusDeps) -> dict: 'batches': batch_summaries, 'timestamp': time.time(), } + + +def _build_album_bundle_status(batch: dict) -> dict: + """Return public batch-level status for torrent/Usenet album-bundle work.""" + state = batch.get('album_bundle_state') + if not state: + return {} + + status = { + 'state': state, + 'source': batch.get('album_bundle_source'), + 'release': batch.get('album_bundle_release'), + 'progress': batch.get('album_bundle_progress'), + 'progress_percent': _album_bundle_progress_percent( + batch.get('album_bundle_progress') + ), + 'speed': batch.get('album_bundle_speed'), + 'downloaded': batch.get('album_bundle_downloaded'), + 'size': batch.get('album_bundle_size'), + 'seeders': batch.get('album_bundle_seeders'), + 'grabs': batch.get('album_bundle_grabs'), + 'count': batch.get('album_bundle_count'), + } + return {key: value for key, value in status.items() if value is not None} + + +def _album_bundle_progress_percent(value: Any) -> int: + try: + progress = float(value) + except (TypeError, ValueError): + return 0 + + if progress <= 1: + progress *= 100 + return max(0, min(100, int(round(progress)))) diff --git a/core/downloads/task_worker.py b/core/downloads/task_worker.py index c5d79d82..67b2a3fe 100644 --- a/core/downloads/task_worker.py +++ b/core/downloads/task_worker.py @@ -25,12 +25,41 @@ import traceback from dataclasses import dataclass from typing import Any, Callable, Optional -from core.runtime_state import download_tasks, tasks_lock +from core.runtime_state import download_batches, download_tasks, tasks_lock from core.spotify_client import Track as SpotifyTrack logger = logging.getLogger(__name__) +def _private_album_bundle_staging_miss_reason(batch_id: Optional[str], deps: Any) -> Optional[str]: + """Return a user-facing miss reason when per-track search should stop. + + Torrent / usenet album batches first download one private staged release, + then each track claims the matching staged file. If that claim fails after + the release is already staged, falling through to the normal per-track + search only retries release-level sources N times and can keep re-adding + the same torrent. Treat the staged release as authoritative for this pass. + """ + if not batch_id: + return None + + batch = download_batches.get(batch_id) + if not isinstance(batch, dict): + return None + + source = (batch.get('album_bundle_source') or '').lower() + mode = (getattr(deps.download_orchestrator, 'mode', '') or '').lower() + if ( + batch.get('album_bundle_private_staging') + and batch.get('album_bundle_state') == 'staged' + and source in ('torrent', 'usenet') + and mode == source + ): + return f'Track was not found in the staged {source} album release' + + return None + + @dataclass class TaskWorkerDeps: """Bundle of cross-cutting deps the per-task download worker needs.""" @@ -128,6 +157,21 @@ def download_track_worker(task_id: str, batch_id: Optional[str], deps: TaskWorke # === STAGING CHECK: Check staging folder for existing file before searching === if deps.try_staging_match(task_id, batch_id, track): return + staging_miss_reason = _private_album_bundle_staging_miss_reason(batch_id, deps) + if staging_miss_reason: + logger.warning( + "[Modal Worker] %s for '%s'; skipping redundant per-track %s search", + staging_miss_reason, + track.name, + getattr(deps.download_orchestrator, 'mode', 'release-source'), + ) + with tasks_lock: + if task_id in download_tasks: + download_tasks[task_id]['status'] = 'not_found' + download_tasks[task_id]['error_message'] = staging_miss_reason + if batch_id: + deps.on_download_completed(batch_id, task_id, False) + return # Initialize task state tracking (like GUI's parallel_search_tracking) with tasks_lock: @@ -147,6 +191,27 @@ def download_track_worker(task_id: str, batch_id: Optional[str], deps: TaskWorke artist_name = track.artists[0] if track.artists else None track_name = track.name + release_queries = [] + try: + _download_mode = (getattr(deps.download_orchestrator, 'mode', '') or '').lower() + _track_album = (getattr(track, 'album', '') or '').strip() + _track_title = (getattr(track, 'name', '') or '').strip() + _track_artists = list(getattr(track, 'artists', []) or []) + _first_artist = _track_artists[0] if _track_artists else '' + _primary_artist = ( + (_first_artist.get('name', '') if isinstance(_first_artist, dict) else str(_first_artist)) + or '' + ).strip() + if ( + _download_mode in ('torrent', 'usenet') + and _primary_artist + and _track_album + and _track_album.lower() not in ('unknown album', _track_title.lower()) + ): + release_queries.append(f"{_primary_artist} {_track_album}".strip()) + except Exception as _release_query_exc: + logger.debug("[Modal Worker] release query hint failed: %s", _release_query_exc) + # Start with matching engine queries search_queries = deps.matching_engine.generate_download_queries(track) @@ -175,8 +240,14 @@ def download_track_worker(task_id: str, batch_id: Optional[str], deps: TaskWorke if cleaned_name and cleaned_name.lower() != track_name.lower(): legacy_queries.append(cleaned_name.strip()) - # Combine enhanced queries with legacy fallbacks - all_queries = search_queries + legacy_queries + # Combine enhanced queries with legacy fallbacks. + # + # Torrent / usenet can use full album releases as a fallback for + # single-track requests, but trying the album release first makes + # playlist batches download whole albums before checking whether a + # track-shaped release exists. Keep release queries last so singles + # stay light when the indexer has a direct result. + all_queries = search_queries + legacy_queries + release_queries # Remove duplicates while preserving order unique_queries = [] @@ -209,8 +280,30 @@ def download_track_worker(task_id: str, batch_id: Optional[str], deps: TaskWorke logger.debug(f"About to call soulseek search for task {task_id}") try: + # Hybrid + album-context batches must skip torrent / usenet during + # the per-track loop — they're release-level sources, can't match + # individual tracks meaningfully, and album-bundle handling only + # fires in single-source mode (see core/downloads/master.py). The + # exclusion lets the hybrid chain fall through to per-track- + # compatible sources (soulseek / streaming) instead of attempting + # N redundant Prowlarr searches that all download the same album + # torrent and rely on the auto-import sweep to clean up. + _exclude_for_hybrid_album = None + try: + _batch_is_album = False + if batch_id: + from core.runtime_state import download_batches as _db + _b = _db.get(batch_id) + if isinstance(_b, dict): + _batch_is_album = bool(_b.get('is_album_download')) + if _batch_is_album and getattr(deps.download_orchestrator, 'mode', '') == 'hybrid': + _exclude_for_hybrid_album = ['torrent', 'usenet'] + except Exception as _exc_filter_err: + logger.debug("[Modal Worker] album-source-exclusion check failed: %s", _exc_filter_err) # Perform search with timeout - tracks_result, _ = deps.run_async(deps.download_orchestrator.search(query, timeout=30)) + tracks_result, _ = deps.run_async(deps.download_orchestrator.search( + query, timeout=30, exclude_sources=_exclude_for_hybrid_album, + )) logger.debug(f"Search completed for task {task_id}, got {len(tracks_result) if tracks_result else 0} results") # CRITICAL: Check cancellation immediately after search returns diff --git a/core/downloads/validation.py b/core/downloads/validation.py index 3ac38751..339d0e8e 100644 --- a/core/downloads/validation.py +++ b/core/downloads/validation.py @@ -25,6 +25,20 @@ def init(matching_engine_obj, download_orchestrator_obj): download_orchestrator = download_orchestrator_obj +def _torrent_usenet_artist_is_fallback(result): + """True when a release result has no parsed artist, only indexer filler.""" + if getattr(result, 'username', None) not in ('torrent', 'usenet'): + return False + artist = (getattr(result, 'artist', None) or '').strip() + if not artist: + return True + metadata = getattr(result, '_source_metadata', None) or {} + indexer = str(metadata.get('indexer') or '').strip() + if artist.lower() in ('torrent', 'usenet'): + return True + return bool(indexer and artist.lower() == indexer.lower()) + + def filter_soundcloud_previews(results, expected_track): """Drop SoundCloud preview snippets so they never reach the cache, the modal, or the auto-download attempt. @@ -97,8 +111,12 @@ def get_valid_candidates(results, spotify_track, query): return [] # Streaming sources (YouTube, Tidal, Qobuz, HiFi, Deezer, SoundCloud) return structured API results - # with proper artist/title metadata — score using the same matching engine as Soulseek - _streaming_sources = ("youtube", "tidal", "qobuz", "hifi", "deezer_dl", "soundcloud", "amazon") + # with proper artist/title metadata — score using the same matching engine as Soulseek. + # Torrent / usenet results also belong here: their filename field is a download URL, not + # a slskd-style ``Artist/Album/Track.flac`` path, so the Soulseek matcher would extract + # garbage segments from it. Routing them through the streaming path means score_track_match + # reads ``r.title`` and ``r.artist`` directly (which the torrent/usenet projections pre-fill). + _streaming_sources = ("youtube", "tidal", "qobuz", "hifi", "deezer_dl", "soundcloud", "amazon", "torrent", "usenet") if results[0].username in _streaming_sources: source_label = results[0].username.replace('_dl', '').title() expected_artists = spotify_track.artists if spotify_track else [] @@ -132,16 +150,55 @@ def get_valid_candidates(results, spotify_track, query): ) continue - # Score using matching engine's generic scorer (same weights as Soulseek) + # Score using matching engine's generic scorer (same weights as Soulseek). + # Torrent/usenet release projections sometimes only have the indexer name + # in the artist field when a title did not parse as "Artist - Release". + # Treat that as unknown artist, not as a real mismatch. + has_only_fallback_artist = _torrent_usenet_artist_is_fallback(r) + candidate_artists = [] if has_only_fallback_artist else ([r.artist] if r.artist else []) confidence, match_type = matching_engine.score_track_match( source_title=expected_title, source_artists=expected_artists, source_duration_ms=expected_duration, candidate_title=r.title or '', - candidate_artists=[r.artist] if r.artist else [], + candidate_artists=candidate_artists, candidate_duration_ms=r.duration or 0, ) + # Album-name fallback for torrent / usenet per-track results. + # + # When this fallback runs: hybrid mode + non-album batch (single + # track wishlist / playlist of singles). Album-context batches + # never reach here — the album-bundle gate in + # core/downloads/album_bundle_dispatch.py engages the bulk- + # download flow in single-source mode, and the hybrid chain + # filter in core/downloads/task_worker.py strips torrent / + # usenet from album batches in hybrid mode. What's left is the + # single-track-in-hybrid case where a user is searching for one + # track and the only torrent / usenet result is the album that + # contains it. + # + # Without this fallback, "Luther (with SZA)" against a + # candidate titled "GNX (2024) [FLAC]" scores ~0 on track-title + # alone — even though the album torrent does in fact contain + # the wanted track. Scoring the candidate title against the + # wanted track's ALBUM name and taking the max gives album- + # level releases a fair shot. The Auto-Import sweep then picks + # the right file out of the downloaded album folder. + expected_album = getattr(spotify_track, 'album', None) if spotify_track else None + if r.username in ('torrent', 'usenet') and expected_album: + album_conf, _ = matching_engine.score_track_match( + source_title=expected_album, + source_artists=expected_artists, + source_duration_ms=0, # albums don't have one duration + candidate_title=r.title or '', + candidate_artists=candidate_artists, + candidate_duration_ms=0, + ) + if album_conf > confidence: + confidence = album_conf + match_type = 'album_release' + # Version detection penalty — reject live/remix/acoustic when expecting original r_title_lower = (r.title or '').lower() is_wrong_version = False @@ -162,8 +219,10 @@ def get_valid_candidates(results, spotify_track, query): # Artist gate — streaming APIs (Tidal/Qobuz/HiFi/Deezer) have reliable metadata, # so "My Will" by "B. Starr" should never match expected "B小町". - # Skip for YouTube — artist is parsed from video titles and often unreliable. - if r.username != 'youtube': + # YouTube stays excluded because video-title parsing is unreliable. + # Torrent/usenet must also pass this gate so title-only matches + # from the wrong artist do not get downloaded. + if r.username != 'youtube' and not has_only_fallback_artist: from difflib import SequenceMatcher import re as _re _cand_artist_raw = r.artist or '' @@ -195,6 +254,16 @@ def get_valid_candidates(results, spotify_track, query): # so falling to SequenceMatcher means the strings are genuinely # different. 0.5 gives a safer buffer without blocking real # matches that would have scored above 0.85 anyway. + if r.username in ('torrent', 'usenet') and _best_artist < 0.5: + logger.info( + "[%s] Rejecting candidate due to artist mismatch: " + "expected=%s candidate=%r title=%r", + source_label, + list(expected_artists), + _cand_artist_raw, + r.title or '', + ) + continue if _best_artist < 0.5 and confidence < 0.85: continue diff --git a/core/hifi_client.py b/core/hifi_client.py index 8fa51cdd..242983ea 100644 --- a/core/hifi_client.py +++ b/core/hifi_client.py @@ -23,6 +23,8 @@ import re import uuid import time import shutil +import json +import base64 import subprocess import threading from typing import List, Optional, Dict, Any, Tuple @@ -75,6 +77,13 @@ HLS_QUALITY_MAP = { HLS_MAP_TAG_RE = re.compile(r'#EXT-X-MAP:.*URI="([^"]+)"') +TRACK_ENDPOINT_QUALITY_MAP = { + 'hires': 'HI_RES_LOSSLESS', + 'lossless': 'LOSSLESS', + 'high': 'HIGH', + 'low': 'LOW', +} + # Default public hifi-api instances (ordered by preference) DEFAULT_INSTANCES = [ 'https://triton.squid.wtf', @@ -244,6 +253,122 @@ class HiFiClient(DownloadSourcePlugin): return data.get('version') or data.get('data', {}).get('version') return None + @staticmethod + def _extract_manifest_uri(data: Any) -> Optional[str]: + try: + inner = data.get('data', data) if isinstance(data, dict) else data + attrs = inner.get('data', {}).get('attributes', {}) + return attrs.get('uri') + except (AttributeError, KeyError): + return None + + @staticmethod + def _extract_track_manifest_urls(data: Any) -> List[str]: + try: + inner = data.get('data', data) if isinstance(data, dict) else data + manifest_b64 = inner.get('manifest') + if not manifest_b64: + return [] + manifest = json.loads(base64.b64decode(manifest_b64)) + if manifest.get('encryptionType') not in (None, 'NONE'): + return [] + urls = manifest.get('urls') or [] + return [url for url in urls if isinstance(url, str) and url] + except Exception as e: + logger.debug("Failed to extract legacy HiFi track manifest URLs: %s", e) + return [] + + @staticmethod + def _extension_from_track_manifest(data: Any, fallback: str) -> str: + try: + inner = data.get('data', data) if isinstance(data, dict) else data + manifest = json.loads(base64.b64decode(inner.get('manifest') or '')) + mime = (manifest.get('mimeType') or '').lower() + codecs = (manifest.get('codecs') or '').lower() + if 'flac' in mime or 'flac' in codecs: + return 'flac' + if 'mp4' in mime or 'aac' in codecs: + return 'm4a' + except Exception as e: + logger.debug("Failed to infer legacy HiFi track manifest extension: %s", e) + return fallback + + def check_instance_capabilities(self, url: str, timeout: int = 5) -> Dict[str, Any]: + """Probe one public HiFi instance using the endpoints SoulSync needs.""" + entry = { + 'url': url, + 'status': 'unknown', + 'version': None, + 'can_search': False, + 'can_download': False, + } + try: + root = self.session.get( + f'{url}/', + timeout=timeout, + headers={'Accept': 'application/json'}, + ) + if not root.ok: + entry['status'] = f'error (HTTP {root.status_code})' + return entry + + data = root.json() + entry['version'] = data.get('version') or data.get('data', {}).get('version') + entry['status'] = 'online' + + search = self.session.get( + f'{url}/search/', + params={'s': 'test', 'limit': 1}, + timeout=timeout, + ) + entry['can_search'] = search.ok + + manifest = self.session.get( + f'{url}/trackManifests/', + params={ + 'id': '1550546', + 'formats': 'FLAC', + 'usage': 'DOWNLOAD', + 'manifestType': 'HLS', + 'adaptive': 'true', + 'uriScheme': 'HTTPS', + }, + timeout=timeout, + ) + entry['can_download'] = ( + manifest.ok + and bool(self._extract_manifest_uri(manifest.json())) + ) + if not manifest.ok: + entry['download_error'] = f'HTTP {manifest.status_code}' + elif not entry['can_download']: + legacy = self.session.get( + f'{url}/track/', + params={'id': '1550546', 'quality': 'LOSSLESS'}, + timeout=timeout, + ) + entry['can_download'] = ( + legacy.ok + and bool(self._extract_track_manifest_urls(legacy.json())) + ) + if not legacy.ok: + entry['download_error'] = f'HTTP {legacy.status_code}' + elif not entry['can_download']: + entry['download_error'] = 'No playable manifest URL' + else: + entry['download_probe'] = 'track' + else: + entry['download_probe'] = 'trackManifests' + except http_requests.exceptions.SSLError: + entry['status'] = 'ssl_error' + except http_requests.exceptions.ConnectTimeout: + entry['status'] = 'timeout' + except http_requests.exceptions.ConnectionError: + entry['status'] = 'offline' + except Exception as e: + entry['status'] = f'error ({type(e).__name__})' + return entry + def search_tracks(self, title: str = None, artist: str = None, album: str = None, limit: int = 20) -> List[Dict]: params = {'limit': limit} @@ -433,15 +558,12 @@ class HiFiClient(DownloadSourcePlugin): data = self._api_get('/trackManifests/', params=params, timeout=20) if not data: - return None + return self._get_legacy_track_manifest(track_id, quality) - try: - inner = data.get('data', data) if isinstance(data, dict) else data - attrs = inner.get('data', {}).get('attributes', {}) - uri = attrs.get('uri') - except (AttributeError, KeyError) as e: - logger.warning(f"Failed to extract playlist URI from manifest response: {e}") - return None + uri = self._extract_manifest_uri(data) + if uri is None: + logger.warning("Failed to extract playlist URI from manifest response") + return self._get_legacy_track_manifest(track_id, quality) if not uri: logger.warning(f"No playlist URI in manifest for track {track_id}") @@ -488,6 +610,28 @@ class HiFiClient(DownloadSourcePlugin): 'quality': quality, } + def _get_legacy_track_manifest(self, track_id: int, quality: str = 'lossless') -> Optional[Dict]: + q_info = HLS_QUALITY_MAP.get(quality, HLS_QUALITY_MAP['lossless']) + api_quality = TRACK_ENDPOINT_QUALITY_MAP.get(quality, 'LOSSLESS') + data = self._api_get('/track/', params={'id': track_id, 'quality': api_quality}, timeout=20) + if not data: + return None + + direct_urls = self._extract_track_manifest_urls(data) + if not direct_urls: + logger.warning(f"No playable URL in legacy HiFi manifest for track {track_id}") + return None + + extension = self._extension_from_track_manifest(data, q_info['extension']) + logger.info(f"HiFi legacy track manifest for track {track_id}: " + f"{len(direct_urls)} direct URL(s) ({quality})") + return { + 'direct_urls': direct_urls, + 'extension': extension, + 'codec': q_info['codec'], + 'quality': quality, + } + def _demux_flac(self, input_path: Path, output_path: Path) -> None: ffmpeg = shutil.which('ffmpeg') if not ffmpeg: @@ -619,7 +763,13 @@ class HiFiClient(DownloadSourcePlugin): return None manifest_info = self._get_hls_manifest(track_id, quality=q_key) - if not manifest_info or not manifest_info.get('segment_uris'): + if ( + not manifest_info + or ( + not manifest_info.get('segment_uris') + and not manifest_info.get('direct_urls') + ) + ): logger.warning(f"No HLS manifest at quality {q_key}, trying next") continue @@ -628,16 +778,17 @@ class HiFiClient(DownloadSourcePlugin): out_filename = f"{safe_name}.{extension}" out_path = self.download_path / out_filename - is_flac = q_key in ('hires', 'lossless') + is_direct = bool(manifest_info.get('direct_urls')) + is_flac = q_key in ('hires', 'lossless') and not is_direct intermediate_path = out_path.with_suffix('.m4a') if is_flac else out_path try: init_uri = manifest_info.get('init_uri') - segment_uris = manifest_info['segment_uris'] + segment_uris = manifest_info.get('segment_uris') or manifest_info.get('direct_urls') or [] total_segments = len(segment_uris) + (1 if init_uri else 0) logger.info(f"Downloading from HiFi ({q_key}): {out_filename} " - f"({total_segments} segments)") + f"({total_segments} {'URL(s)' if is_direct else 'segments'})") downloaded = 0 speed_start = time.time() diff --git a/core/imports/side_effects.py b/core/imports/side_effects.py index 9b3d9f3a..6513aabf 100644 --- a/core/imports/side_effects.py +++ b/core/imports/side_effects.py @@ -198,6 +198,10 @@ def record_library_history_download(context: Dict[str, Any]) -> None: "deezer_dl": "Deezer", "lidarr": "Lidarr", "soundcloud": "SoundCloud", + "amazon": "Amazon", + "staging": "Staging", + "torrent": "Torrent", + "usenet": "Usenet", # Auto-import isn't a download source, but flows through the # same post-process pipeline (file lands → record provenance # + history → write to library DB). Tagging it as "Auto-Import" @@ -276,6 +280,7 @@ def record_download_provenance(context: Dict[str, Any]) -> None: "deezer_dl": "deezer", "lidarr": "lidarr", "soundcloud": "soundcloud", + "amazon": "amazon", # Auto-import: surfaced in provenance so the redownload modal # can tell the user "this came from staging on " instead # of falsely listing soulseek as the source. The underlying @@ -283,6 +288,16 @@ def record_download_provenance(context: Dict[str, Any]) -> None: # separately via the source-aware ID columns on the tracks # row itself. "auto_import": "auto_import", + # Generic staging-match (user dropped files manually OR a + # source we don't have a more specific label for). Better + # than defaulting to 'soulseek' which would falsely tag the + # provenance. + "staging": "staging", + # Torrent / usenet album-bundle flow — the staging matcher + # overrides 'staging' with the bundle source so the history + # shows where the files actually came from. + "torrent": "torrent", + "usenet": "usenet", }.get(username, "soulseek") ti = context.get("track_info") or context.get("search_result") or {} diff --git a/core/metadata/cache.py b/core/metadata/cache.py index 2974c5eb..b49daae5 100644 --- a/core/metadata/cache.py +++ b/core/metadata/cache.py @@ -8,6 +8,7 @@ Transparent to callers: check cache before API call, store after success. import json import logging import threading +import time from datetime import datetime from typing import Optional, Dict, List, Tuple @@ -49,6 +50,33 @@ class MetadataCache: from database.music_database import get_database return get_database() + @staticmethod + def _is_transient_sqlite_io_error(exc: Exception) -> bool: + return 'disk i/o error' in str(exc).lower() + + def _run_maintenance_write(self, label: str, operation, default: int = 0) -> int: + """Run a maintenance write with one retry for transient SQLite I/O.""" + for attempt in range(2): + try: + db = self._get_db() + conn = db._get_connection() + try: + return operation(conn) + finally: + conn.close() + except Exception as e: + if self._is_transient_sqlite_io_error(e) and attempt == 0: + logger.warning( + "%s hit transient SQLite disk I/O error; retrying once: %s", + label, + e, + ) + time.sleep(0.25) + continue + logger.error("%s error: %s", label, e) + return default + return default + # ─── Entity Methods ─────────────────────────────────────────────── def get_entity(self, source: str, entity_type: str, entity_id: str) -> Optional[dict]: @@ -566,137 +594,113 @@ class MetadataCache: def evict_expired(self) -> int: """Delete entries that have exceeded their TTL. Returns count of evicted entries.""" - try: - db = self._get_db() - conn = db._get_connection() - try: - cursor = conn.cursor() + def _operation(conn): + cursor = conn.cursor() - # Entities - cursor.execute(""" - DELETE FROM metadata_cache_entities - WHERE julianday('now') - julianday(updated_at) > ttl_days - """) - entity_count = cursor.rowcount + # Entities + cursor.execute(""" + DELETE FROM metadata_cache_entities + WHERE julianday('now') - julianday(updated_at) > ttl_days + """) + entity_count = cursor.rowcount - # Searches - cursor.execute(""" - DELETE FROM metadata_cache_searches - WHERE julianday('now') - julianday(created_at) > ttl_days - """) - search_count = cursor.rowcount + # Searches + cursor.execute(""" + DELETE FROM metadata_cache_searches + WHERE julianday('now') - julianday(created_at) > ttl_days + """) + search_count = cursor.rowcount - conn.commit() - total = entity_count + search_count - if total > 0: - logger.info(f"Evicted {total} expired cache entries ({entity_count} entities, {search_count} searches)") - return total - finally: - conn.close() - except Exception as e: - logger.error(f"Cache eviction error: {e}") - return 0 + conn.commit() + total = entity_count + search_count + if total > 0: + logger.info(f"Evicted {total} expired cache entries ({entity_count} entities, {search_count} searches)") + return total + + return self._run_maintenance_write("Cache eviction", _operation) def clean_junk_entities(self) -> int: """Delete cached entities with empty/placeholder names.""" - try: - db = self._get_db() - conn = db._get_connection() - try: - cursor = conn.cursor() - junk_names = "', '".join(self._JUNK_NAMES - {''}) # exclude empty, handled separately - cursor.execute(f""" - DELETE FROM metadata_cache_entities - WHERE (name IS NULL - OR TRIM(name) = '' - OR LOWER(TRIM(name)) IN ('{junk_names}')) - AND entity_id NOT LIKE '%\\_features' ESCAPE '\\' - AND entity_id NOT LIKE '%\\_tracks' ESCAPE '\\' - """) - count = cursor.rowcount - conn.commit() - if count > 0: - logger.info(f"Cleaned {count} junk entities from cache") - return count - finally: - conn.close() - except Exception as e: - logger.error(f"Junk cleanup error: {e}") - return 0 + def _operation(conn): + cursor = conn.cursor() + junk_names = "', '".join(self._JUNK_NAMES - {''}) # exclude empty, handled separately + cursor.execute(f""" + DELETE FROM metadata_cache_entities + WHERE (name IS NULL + OR TRIM(name) = '' + OR LOWER(TRIM(name)) IN ('{junk_names}')) + AND entity_id NOT LIKE '%\\_features' ESCAPE '\\' + AND entity_id NOT LIKE '%\\_tracks' ESCAPE '\\' + """) + count = cursor.rowcount + conn.commit() + if count > 0: + logger.info(f"Cleaned {count} junk entities from cache") + return count + + return self._run_maintenance_write("Junk cleanup", _operation) def clean_orphaned_searches(self) -> int: """Delete search results where <50% of referenced entities still exist.""" - try: - db = self._get_db() - conn = db._get_connection() - try: - cursor = conn.cursor() - cursor.execute("SELECT id, source, search_type, result_ids FROM metadata_cache_searches") - rows = cursor.fetchall() + def _operation(conn): + cursor = conn.cursor() + cursor.execute("SELECT id, source, search_type, result_ids FROM metadata_cache_searches") + rows = cursor.fetchall() - dead_ids = [] - for row in rows: - try: - result_ids = json.loads(row['result_ids'] or '[]') - except (json.JSONDecodeError, TypeError): - dead_ids.append(row['id']) - continue + dead_ids = [] + for row in rows: + try: + result_ids = json.loads(row['result_ids'] or '[]') + except (json.JSONDecodeError, TypeError): + dead_ids.append(row['id']) + continue - if not result_ids: - dead_ids.append(row['id']) - continue + if not result_ids: + dead_ids.append(row['id']) + continue - # Check how many referenced entities still exist - placeholders = ','.join('?' * len(result_ids)) - cursor.execute(f""" - SELECT COUNT(*) FROM metadata_cache_entities - WHERE source = ? AND entity_type = ? AND entity_id IN ({placeholders}) - """, [row['source'], row['search_type']] + list(result_ids)) - found = cursor.fetchone()[0] + # Check how many referenced entities still exist + placeholders = ','.join('?' * len(result_ids)) + cursor.execute(f""" + SELECT COUNT(*) FROM metadata_cache_entities + WHERE source = ? AND entity_type = ? AND entity_id IN ({placeholders}) + """, [row['source'], row['search_type']] + list(result_ids)) + found = cursor.fetchone()[0] - if found < len(result_ids) * 0.5: - dead_ids.append(row['id']) + if found < len(result_ids) * 0.5: + dead_ids.append(row['id']) - if dead_ids: - # Delete in chunks to stay under SQLite variable limit - for i in range(0, len(dead_ids), 400): - chunk = dead_ids[i:i + 400] - placeholders = ','.join('?' * len(chunk)) - cursor.execute(f"DELETE FROM metadata_cache_searches WHERE id IN ({placeholders})", chunk) - conn.commit() + if dead_ids: + # Delete in chunks to stay under SQLite variable limit + for i in range(0, len(dead_ids), 400): + chunk = dead_ids[i:i + 400] + placeholders = ','.join('?' * len(chunk)) + cursor.execute(f"DELETE FROM metadata_cache_searches WHERE id IN ({placeholders})", chunk) + conn.commit() - count = len(dead_ids) - if count > 0: - logger.info(f"Cleaned {count} orphaned search results from cache") - return count - finally: - conn.close() - except Exception as e: - logger.error(f"Orphan search cleanup error: {e}") - return 0 + count = len(dead_ids) + if count > 0: + logger.info(f"Cleaned {count} orphaned search results from cache") + return count + + return self._run_maintenance_write("Orphan search cleanup", _operation) def clean_stale_musicbrainz_nulls(self, max_age_days: int = 30) -> int: """Delete MusicBrainz cache entries where lookup found nothing (null MBID) and age > max_age_days.""" - try: - db = self._get_db() - conn = db._get_connection() - try: - cursor = conn.cursor() - cursor.execute(""" - DELETE FROM musicbrainz_cache - WHERE musicbrainz_id IS NULL - AND julianday('now') - julianday(last_updated) > ? - """, (max_age_days,)) - count = cursor.rowcount - conn.commit() - if count > 0: - logger.info(f"Cleaned {count} stale MusicBrainz null entries (>{max_age_days} days)") - return count - finally: - conn.close() - except Exception as e: - logger.error(f"MusicBrainz null cleanup error: {e}") - return 0 + def _operation(conn): + cursor = conn.cursor() + cursor.execute(""" + DELETE FROM musicbrainz_cache + WHERE musicbrainz_id IS NULL + AND julianday('now') - julianday(last_updated) > ? + """, (max_age_days,)) + count = cursor.rowcount + conn.commit() + if count > 0: + logger.info(f"Cleaned {count} stale MusicBrainz null entries (>{max_age_days} days)") + return count + + return self._run_maintenance_write("MusicBrainz null cleanup", _operation) def get_health_stats(self) -> dict: """Return cache health statistics for the repair dashboard. diff --git a/core/repair_worker.py b/core/repair_worker.py index 227d144a..9af2ff30 100644 --- a/core/repair_worker.py +++ b/core/repair_worker.py @@ -35,6 +35,64 @@ logger = get_logger("repair_worker") AUDIO_EXTENSIONS = {'.mp3', '.flac', '.ogg', '.opus', '.m4a', '.aac', '.wav', '.wma', '.aiff', '.aif'} +def _album_fill_artist_names_match(expected_artist: str, candidate_artist: str) -> bool: + """Strict artist gate for Album Completeness auto-fill. + + Auto-fill moves/copies files into an existing album, so artist identity + must outrank album/title similarity. Use the alias-aware matcher when it + is available, then fall back to conservative normalized similarity. + """ + expected = (expected_artist or '').strip() + candidate = (candidate_artist or '').strip() + if not expected or not candidate: + return False + + try: + from core.matching.artist_aliases import artist_names_match + matched, score = artist_names_match(expected, candidate, threshold=0.82) + if matched: + return True + if score < 0.82: + return False + except Exception as alias_err: + logger.debug("artist_names_match unavailable, using fallback: %s", alias_err) + + try: + from core.matching_engine import MusicMatchingEngine + engine = MusicMatchingEngine() + expected_norm = engine.clean_artist(expected) + candidate_norm = engine.clean_artist(candidate) + except Exception: + expected_norm = expected.lower() + candidate_norm = candidate.lower() + + if not expected_norm or not candidate_norm: + return False + return expected_norm == candidate_norm or SequenceMatcher(None, expected_norm, candidate_norm).ratio() >= 0.82 + + +def _album_fill_target_artist_allows_track(album_artist: str, track_artists: List[str]) -> bool: + """Return whether a source track can be auto-filled into an album artist. + + Compilation-style album artists are allowed to contain varied track + artists. Normal albums require at least one source track artist to match + the target album artist before any local candidate is considered. + """ + album_artist = (album_artist or '').strip() + if not album_artist: + return False + + normalized_album_artist = album_artist.lower().strip() + if normalized_album_artist in {'various artists', 'various', 'soundtrack'}: + return True + + source_artists = [str(a).strip() for a in (track_artists or []) if str(a or '').strip()] + if not source_artists: + return True + + return any(_album_fill_artist_names_match(album_artist, artist) for artist in source_artists) + + def _split_acoustid_credit(credit: str) -> List[str]: """Split an AcoustID artist credit into individual contributor names. @@ -2097,6 +2155,21 @@ class RepairWorker: track_details.append({'track': track_name, 'status': 'skipped', 'reason': 'no track name'}) continue + if not _album_fill_target_artist_allows_track(artist_name, track_artists): + skipped_count += 1 + logger.warning( + "Album auto-fill skipped '%s': source artist(s) %s do not match target album artist '%s'", + track_name, track_artists, artist_name, + ) + track_details.append({ + 'track': track_name, + 'status': 'skipped', + 'reason': 'source artist does not match target album artist', + 'source_artists': track_artists, + 'target_artist': artist_name, + }) + continue + # Search library for this track candidates = self.db.search_tracks(title=track_name, artist=artist_search, limit=20) @@ -2117,8 +2190,22 @@ class RepairWorker: # Artist match (more lenient) cand_artist = getattr(cand, 'artist_name', '') or '' - artist_sim = SequenceMatcher(None, artist_search.lower(), cand_artist.lower()).ratio() - if artist_sim < 0.50: + candidate_artist_fields = [ + cand_artist, + getattr(cand, 'track_artist', '') or '', + ] + expected_artist_names = track_artists or [artist_name] + if not any( + _album_fill_artist_names_match(expected, candidate) + for expected in expected_artist_names + for candidate in candidate_artist_fields + ): + logger.debug( + "Album auto-fill rejected candidate '%s' by '%s' for expected artist(s) %s", + getattr(cand, 'title', ''), + cand_artist, + expected_artist_names, + ) continue # Quality gate diff --git a/core/torrent_clients/qbittorrent.py b/core/torrent_clients/qbittorrent.py index 3373086e..579c8ec8 100644 --- a/core/torrent_clients/qbittorrent.py +++ b/core/torrent_clients/qbittorrent.py @@ -164,28 +164,60 @@ class QBittorrentAdapter: category: str, save_path: Optional[str], ) -> Optional[str]: - data = {'urls': url_or_magnet, 'category': category or self._category} + cat = category or self._category + # Snapshot the current set of torrent hashes BEFORE adding — + # qBittorrent's /add endpoint returns 200 "Ok." regardless of + # whether the URL was actually accepted or registered, and + # category-filtered lookups race the add (qBit hasn't + # categorised the new torrent yet on the first poll). Diffing + # before / after is the only reliable way to recover the hash. + before = self._all_hashes() + if before is None: + return None + data = {'urls': url_or_magnet, 'category': cat} if save_path or self._save_path: data['savepath'] = save_path or self._save_path resp = self._call('POST', '/api/v2/torrents/add', data=data) if not resp or not resp.ok: + logger.warning("qBittorrent /torrents/add returned HTTP %s body=%r", + resp.status_code if resp else 'no-response', + (resp.text[:200] if resp else '')) return None - # qBittorrent's /add endpoint returns 200 "Ok." on success but - # NOT the resulting info-hash. We have to look it up via - # /torrents/info filtered by category — the just-added torrent - # will be the newest entry in that category. - return self._lookup_latest_hash(category or self._category) + if resp.text and resp.text.strip() and resp.text.strip() != 'Ok.': + logger.warning("qBittorrent /torrents/add unexpected body: %r", resp.text[:200]) + return None + new_hash = self._poll_for_new_hash(before) + if not new_hash: + logger.error("qBittorrent accepted the request but no new torrent appeared — " + "URL may have been rejected (bad magnet, unreachable HTTPS, " + "duplicate hash, etc.)") + return new_hash - def _lookup_latest_hash(self, category: str) -> Optional[str]: - resp = self._call('GET', '/api/v2/torrents/info', params={'category': category, 'sort': 'added_on', 'reverse': 'true', 'limit': 1}) + def _all_hashes(self) -> Optional[set]: + """Return the set of every torrent hash qBit currently tracks, + or None on lookup failure.""" + resp = self._call('GET', '/api/v2/torrents/info') if not resp or not resp.ok: return None try: - items = resp.json() - if items: - return items[0].get('hash') + return {item.get('hash') for item in resp.json() if item.get('hash')} except Exception as e: logger.error("qBittorrent /torrents/info parse failed: %s", e) + return None + + def _poll_for_new_hash(self, before: set) -> Optional[str]: + """Poll up to ~5s for a new torrent to appear (qBit takes a + moment to fetch the .torrent file from the URL and register + it). Returns the new hash, or None if nothing showed up.""" + import time as _time + for _ in range(10): + _time.sleep(0.5) + current = self._all_hashes() + if current is None: + continue + new = current - before + if new: + return next(iter(new)) return None async def add_torrent_file( @@ -205,14 +237,18 @@ class QBittorrentAdapter: category: str, save_path: Optional[str], ) -> Optional[str]: - data = {'category': category or self._category} + cat = category or self._category + before = self._all_hashes() + if before is None: + return None + data = {'category': cat} if save_path or self._save_path: data['savepath'] = save_path or self._save_path files = {'torrents': ('soulsync.torrent', file_bytes, 'application/x-bittorrent')} resp = self._call('POST', '/api/v2/torrents/add', data=data, files=files) if not resp or not resp.ok: return None - return self._lookup_latest_hash(category or self._category) + return self._poll_for_new_hash(before) async def get_status(self, torrent_id: str) -> Optional[TorrentStatus]: loop = asyncio.get_event_loop() diff --git a/core/worker_utils.py b/core/worker_utils.py index d6f42523..e572c702 100644 --- a/core/worker_utils.py +++ b/core/worker_utils.py @@ -64,6 +64,17 @@ def set_album_api_track_count(cursor, album_id, count): (count, album_id), ) except Exception as e: + if "api_track_count" in str(e) and "no such column" in str(e).lower(): + try: + cursor.execute("ALTER TABLE albums ADD COLUMN api_track_count INTEGER DEFAULT NULL") + cursor.execute( + "UPDATE albums SET api_track_count = ? WHERE id = ?", + (count, album_id), + ) + logger.info("Repaired missing api_track_count column while caching album track count") + return + except Exception as repair_error: + e = repair_error logger.warning( "Failed to cache api_track_count for album %s: %s", album_id, e ) diff --git a/database/music_database.py b/database/music_database.py index 02cc5246..0171d88e 100644 --- a/database/music_database.py +++ b/database/music_database.py @@ -766,6 +766,8 @@ class MusicDatabase: except Exception as ps_err: logger.error(f"Personalized-playlist schema init failed: {ps_err}") + self._ensure_core_media_schema_columns(cursor) + conn.commit() logger.info("Database initialized successfully") @@ -775,6 +777,29 @@ class MusicDatabase: self._init_manual_library_match_table() + def _ensure_core_media_schema_columns(self, cursor): + """Repair required media-library columns that older migrations may miss. + + A few legacy migrations rebuild artists/albums/tracks in place. Newer + installs get these columns from CREATE TABLE, but upgraded databases can + occasionally miss one if a previous migration path failed or was marked + complete before the column existed. + """ + try: + cursor.execute("PRAGMA table_info(tracks)") + track_cols = {c[1] for c in cursor.fetchall()} + if track_cols and 'file_size' not in track_cols: + cursor.execute("ALTER TABLE tracks ADD COLUMN file_size INTEGER") + logger.info("Repaired missing file_size column on tracks table") + + cursor.execute("PRAGMA table_info(albums)") + album_cols = {c[1] for c in cursor.fetchall()} + if album_cols and 'api_track_count' not in album_cols: + cursor.execute("ALTER TABLE albums ADD COLUMN api_track_count INTEGER DEFAULT NULL") + logger.info("Repaired missing api_track_count column on albums table") + except Exception as e: + logger.error("Error repairing core media schema columns: %s", e) + def _add_mirrored_playlist_explored_column(self, cursor): """Add explored_at column to mirrored_playlists to persist explore badge.""" try: @@ -4566,45 +4591,72 @@ class MusicDatabase: # VACUUM to actually shrink the database file and reclaim disk space logger.info("Vacuuming database to reclaim disk space...") - cursor.execute("VACUUM") + self._vacuum_best_effort(cursor) logger.info("All database data cleared and file compacted") except Exception as e: logger.error(f"Error clearing database: {e}") raise + + def _vacuum_best_effort(self, cursor): + """Run VACUUM without making the caller fail if compaction hiccups.""" + try: + cursor.execute("VACUUM") + except Exception as e: + logger.warning( + "Database VACUUM failed after data was already cleared; continuing without compaction: %s", + e, + ) + + @staticmethod + def _is_transient_sqlite_io_error(exc: Exception) -> bool: + return "disk i/o error" in str(exc).lower() def clear_server_data(self, server_source: str): """Clear data for specific server only (server-aware full refresh)""" - try: - with self._get_connection() as conn: - cursor = conn.cursor() - - # Delete only data from the specified server - # Order matters: tracks -> albums -> artists (foreign key constraints) - cursor.execute("DELETE FROM tracks WHERE server_source = ?", (server_source,)) - tracks_deleted = cursor.rowcount - - cursor.execute("DELETE FROM albums WHERE server_source = ?", (server_source,)) - albums_deleted = cursor.rowcount - - cursor.execute("DELETE FROM artists WHERE server_source = ?", (server_source,)) - artists_deleted = cursor.rowcount - - conn.commit() - - # Only VACUUM if we deleted a significant amount of data - if tracks_deleted > 1000 or albums_deleted > 100: - logger.info("Vacuuming database to reclaim disk space...") - cursor.execute("VACUUM") - - logger.info(f"Cleared {server_source} data: {artists_deleted} artists, {albums_deleted} albums, {tracks_deleted} tracks") - - # Note: Watchlist and wishlist are preserved as they are server-agnostic - - except Exception as e: - logger.error(f"Error clearing {server_source} database data: {e}") - raise + for attempt in range(2): + try: + with self._get_connection() as conn: + cursor = conn.cursor() + + # Delete only data from the specified server + # Order matters: tracks -> albums -> artists (foreign key constraints) + cursor.execute("DELETE FROM tracks WHERE server_source = ?", (server_source,)) + tracks_deleted = cursor.rowcount + + cursor.execute("DELETE FROM albums WHERE server_source = ?", (server_source,)) + albums_deleted = cursor.rowcount + + cursor.execute("DELETE FROM artists WHERE server_source = ?", (server_source,)) + artists_deleted = cursor.rowcount + + conn.commit() + + # Only VACUUM if we deleted a significant amount of data + if tracks_deleted > 1000 or albums_deleted > 100: + logger.info("Vacuuming database to reclaim disk space...") + self._vacuum_best_effort(cursor) + + logger.info( + f"Cleared {server_source} data: {artists_deleted} artists, " + f"{albums_deleted} albums, {tracks_deleted} tracks" + ) + + # Note: Watchlist and wishlist are preserved as they are server-agnostic + return + + except Exception as e: + if self._is_transient_sqlite_io_error(e) and attempt == 0: + logger.warning( + "Transient disk I/O error clearing %s database data; retrying once: %s", + server_source, + e, + ) + time.sleep(0.25) + continue + logger.error(f"Error clearing {server_source} database data: {e}") + raise def cleanup_orphaned_records(self) -> Dict[str, int]: """Remove artists and albums that have no associated tracks""" @@ -5499,6 +5551,25 @@ class MusicDatabase: return True except Exception as e: + error_text = str(e).lower() + if ( + 'file_size' in error_text + and ('no such column' in error_text or 'no column named' in error_text) + and retry_count < max_retries - 1 + ): + try: + repair_conn = conn if 'conn' in locals() else self._get_connection() + repair_cursor = repair_conn.cursor() + self._ensure_core_media_schema_columns(repair_cursor) + repair_conn.commit() + if repair_conn is not conn: + repair_conn.close() + retry_count += 1 + logger.info("Repaired missing file_size column while importing media track; retrying") + continue + except Exception as schema_error: + logger.error("Failed to repair tracks.file_size during track import: %s", schema_error) + retry_count += 1 if "database is locked" in str(e).lower() and retry_count < max_retries: logger.warning(f"Database locked on track '{getattr(track_obj, 'title', 'Unknown')}', retrying {retry_count}/{max_retries}...") diff --git a/docker-compose.yml b/docker-compose.yml index a2d8a0d4..fe5ed945 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -48,6 +48,26 @@ services: # - /path/to/music-library:/host/music:rw # Optional: Mount your music library for Plex/Jellyfin/Navidrome access # - /path/to/your/music:/music:ro + # + # ─── Torrent / Usenet download sources (Prowlarr integration) ──────── + # If you use the Torrent Only or Usenet Only download sources, SoulSync + # needs to read the files your torrent / usenet client downloads. + # + # EASIEST SETUP: point qBittorrent, SABnzbd / NZBGet, AND slskd at the + # SAME download folder. The ./downloads mount above is reused; nothing + # new to add here. Just configure each client's download path to + # /downloads (inside its own container) — files land in the same place + # SoulSync already reads from. + # + # SEPARATE FOLDERS (if you prefer): uncomment + edit these to point at + # the client's download folder. The in-container path must MATCH the + # save_path the client reports. + # - /path/to/qbittorrent/downloads:/downloads/torrents:rw + # - /path/to/sabnzbd/complete:/downloads/usenet:rw + # Then set the torrent client's save path to /downloads/torrents and + # SAB / NZBGet's complete folder to /downloads/usenet (or whatever + # in-container path you chose). Skip these if your downloader runs + # outside Docker and writes to a path already mounted above. extra_hosts: # Allow container to reach host services - "host.docker.internal:host-gateway" diff --git a/pr_description.md b/pr_description.md new file mode 100644 index 00000000..83fda10a --- /dev/null +++ b/pr_description.md @@ -0,0 +1,76 @@ +## Summary + +Adds torrent and usenet as release-oriented download sources backed by Prowlarr and the configured downloader clients. These sources can download full releases, stage the resulting audio files, and let SoulSync match/import the requested tracks through the existing post-processing pipeline. + +This PR also improves the torrent/usenet user experience with clearer release-download progress, source-aware service labels, library history provenance, and safer staged-release matching for album files that include featured-artist or bonus-track filename noise. + +## Scope + +- Adds torrent and usenet plugin support for release downloads. +- Adds album-bundle staging for single-source torrent/usenet album downloads. +- Keeps hybrid album downloads on per-track-capable sources by excluding torrent/usenet from hybrid album per-track searches. +- Allows torrent/usenet in hybrid for non-album track downloads, such as playlist singles and wishlist tracks. +- Selects the requested audio file from completed release folders instead of importing the first audio file blindly. +- Reports live album-bundle progress to the Downloads page and download modal. +- Records torrent/usenet provenance in library history. +- Adds UI polish for release-first download states. + +## Behavior Gates + +- Users who do not select torrent or usenet as a download source should not hit the new download paths. +- Single-source `torrent` / `usenet` album downloads use the release staging flow. +- Hybrid album downloads skip torrent/usenet and continue through the existing per-track source chain. +- Hybrid non-album downloads may use torrent/usenet when they are included in the hybrid order. + +## Notable Implementation Details + +- Torrent/usenet release downloads use private per-batch staging folders under the configured album-bundle staging root. +- Post-processing receives the real source label (`torrent` / `usenet`) so library history no longer falls back to `Soulseek`. +- Staging matching strips only conservative noise like `(feat. Artist)` and `(Bonus Track)` while preserving meaningful version text like `remix`, `extended`, `live`, and `acoustic`. +- When a staged release does not contain a requested track, the task is marked not found instead of repeatedly searching/downloading the same release. +- Completed release downloads expose all discovered audio files so post-processing can choose the best matching track. + +## Testing + +Manually tested: + +- Torrent-only album download for `good kid, m.A.A.d city (Deluxe)`. +- Torrent playlist/single-track flow. +- Album-bundle progress in the download modal and Downloads page. +- Torrent source labeling in library history for new imports. +- Staging match behavior for featured-artist filenames and bonus-track labels. +- Quarantine behavior for wrong-version matches. + +Automated tests run during development: + +```bash +.venv/bin/python -m pytest \ + tests/downloads/test_downloads_status.py \ + tests/test_album_bundle_dispatch.py \ + tests/downloads/test_downloads_staging.py \ + tests/test_torrent_usenet_plugins.py +``` + +```bash +.venv/bin/python -m pytest \ + tests/downloads/test_downloads_validation.py \ + tests/test_manual_pick_no_auto_retry.py \ + tests/downloads/test_downloads_post_processing.py \ + tests/downloads/test_downloads_task_worker.py \ + tests/imports/test_import_side_effects.py +``` + +Focused checks also passed for: + +- staged release feature suffix matching +- bonus-track title matching +- wrong-version separation +- private torrent album staging miss handling +- torrent/usenet history source labels + +## Reviewer Notes + +- This is intentionally gated behind torrent/usenet source selection, but it is still a new release-oriented download path and should be considered beta/experimental for first release. +- Remote downloader setups need SoulSync to be able to read the downloader save path. Local/all-in-one setups should be the easiest path. +- Existing library history rows that were previously recorded as `Soulseek` are not backfilled by this PR. +- Release matching is naturally fuzzier than track-native sources, so reviewer focus should stay on false positives, version handling, and staged-file selection. diff --git a/tests/downloads/test_downloads_lifecycle.py b/tests/downloads/test_downloads_lifecycle.py index fb67d157..8be74876 100644 --- a/tests/downloads/test_downloads_lifecycle.py +++ b/tests/downloads/test_downloads_lifecycle.py @@ -329,6 +329,48 @@ def test_batch_completion_emits_batch_complete_when_all_done(): assert 'b1' in monitor.stopped +def test_batch_completion_cleans_private_album_bundle_staging(tmp_path): + staging_dir = tmp_path / 'b1' + staging_dir.mkdir() + (staging_dir / 'leftover.flac').write_bytes(b'audio') + + download_tasks['t1'] = {'status': 'completed', 'track_info': {'name': 'X'}} + download_batches['b1'] = { + 'queue': ['t1'], 'queue_index': 1, 'active_count': 1, + 'max_concurrent': 1, 'permanently_failed_tracks': [], + 'cancelled_tracks': set(), + 'album_bundle_private_staging': True, + 'album_bundle_source': 'torrent', + 'album_bundle_staging_path': str(staging_dir), + } + deps, _ = _build_deps() + + lc.on_download_completed('b1', 't1', True, deps) + + assert not staging_dir.exists() + + +def test_batch_completion_keeps_unexpected_staging_path(tmp_path): + staging_dir = tmp_path / 'shared-staging' + staging_dir.mkdir() + (staging_dir / 'leftover.flac').write_bytes(b'audio') + + download_tasks['t1'] = {'status': 'completed', 'track_info': {'name': 'X'}} + download_batches['b1'] = { + 'queue': ['t1'], 'queue_index': 1, 'active_count': 1, + 'max_concurrent': 1, 'permanently_failed_tracks': [], + 'cancelled_tracks': set(), + 'album_bundle_private_staging': True, + 'album_bundle_source': 'torrent', + 'album_bundle_staging_path': str(staging_dir), + } + deps, _ = _build_deps() + + lc.on_download_completed('b1', 't1', True, deps) + + assert staging_dir.exists() + + def test_batch_completion_skips_emit_when_zero_successful(): """Don't emit batch_complete if nothing actually downloaded.""" download_tasks['t1'] = {'status': 'failed', 'track_info': {'name': 'X'}, 'track_index': 0} diff --git a/tests/downloads/test_downloads_post_processing.py b/tests/downloads/test_downloads_post_processing.py index a5f02d8c..455b2237 100644 --- a/tests/downloads/test_downloads_post_processing.py +++ b/tests/downloads/test_downloads_post_processing.py @@ -232,6 +232,28 @@ def test_file_found_in_downloads_with_context_runs_post_process_with_verificatio assert any(c[0] == 'post_process' for c in rec.calls) +def test_file_search_ignores_non_audio_candidates(monkeypatch): + download_tasks['t1'] = { + 'status': 'post_processing', + 'filename': 'Artist - Album.cue', + 'username': 'torrent', + 'track_info': {'name': 'Money'}, + } + matched_downloads_context['torrent::Artist - Album.cue'] = { + 'original_search_result': {'title': 'Money', 'track_number': 1}, + } + monkeypatch.setattr(pp.time, 'sleep', lambda s: None) + deps, rec = _build_deps( + find_completed_file=lambda *a, **kw: ('/downloads/Artist - Album.cue', 'download'), + ) + + pp.run_post_processing_worker('t1', 'b1', deps) + + assert download_tasks['t1']['status'] == 'failed' + assert not any(c[0] == 'post_process' for c in rec.calls) + assert ('on_complete', ('b1', 't1', False), {}) in rec.calls + + def test_file_found_in_downloads_no_context_marks_completed_directly(): """No matched context for the file → just mark completed since file exists.""" download_tasks['t1'] = { @@ -323,6 +345,139 @@ def test_youtube_task_uses_get_download_status_to_resolve_path(monkeypatch): assert any(c[0] == 'mark_completed' for c in rec.calls) +def test_torrent_release_copies_best_matching_audio_to_transfer(tmp_path): + release_dir = tmp_path / 'release' + release_dir.mkdir() + wrong = release_dir / '01 - Intro.flac' + right = release_dir / '02 - Money.flac' + wrong.write_bytes(b'wrong') + right.write_bytes(b'right') + transfer_dir = tmp_path / 'transfer' + + filename = 'magnet:?xt=abc||Artist - Album' + download_tasks['t1'] = { + 'status': 'post_processing', + 'filename': filename, + 'username': 'torrent', + 'download_id': 'dl-torrent-1', + 'track_info': {'name': 'Money', 'artists': [{'name': 'Artist'}]}, + } + matched_downloads_context[f'torrent::{filename}'] = { + 'original_search_result': {'title': 'Money', 'track_number': 2}, + } + + class _FakeStatus: + file_path = str(wrong) + audio_files = [str(wrong), str(right)] + + class _FakeTorrentClient: + def get_download_status(self, dl_id): + assert dl_id == 'dl-torrent-1' + return _FakeStatus() + + deps, rec = _build_deps( + config=_FakeConfig({'soulseek.transfer_path': str(transfer_dir)}), + download_orchestrator=_FakeTorrentClient(), + run_async=lambda coro: coro, + ) + + pp.run_post_processing_worker('t1', 'b1', deps) + + copied = transfer_dir / '02 - Money.flac' + assert copied.exists() + assert right.exists() + assert any(c[0] == 'post_process' and c[1][2] == str(copied) for c in rec.calls) + + +def test_torrent_release_prefers_task_title_over_release_context(tmp_path): + release_dir = tmp_path / 'release' + release_dir.mkdir() + wrong = release_dir / '09.Harry Styles - Pop.flac' + right = release_dir / '10.Harry Styles - American Girls.flac' + wrong.write_bytes(b'wrong') + right.write_bytes(b'right') + transfer_dir = tmp_path / 'transfer' + + filename = 'http://prowlarr/download?id=1||Harry Styles - Kiss All The Time' + download_tasks['t1'] = { + 'status': 'post_processing', + 'filename': filename, + 'username': 'torrent', + 'download_id': 'dl-torrent-1', + 'track_info': {'name': 'American Girls', 'artists': [{'name': 'Harry Styles'}]}, + } + matched_downloads_context[f'torrent::{filename}'] = { + 'original_search_result': {'title': 'Pop', 'clean_title': 'Pop', 'track_number': 9}, + } + + class _FakeStatus: + file_path = str(wrong) + audio_files = [str(wrong), str(right)] + + class _FakeTorrentClient: + def get_download_status(self, dl_id): + assert dl_id == 'dl-torrent-1' + return _FakeStatus() + + deps, rec = _build_deps( + config=_FakeConfig({'soulseek.transfer_path': str(transfer_dir)}), + download_orchestrator=_FakeTorrentClient(), + run_async=lambda coro: coro, + ) + + pp.run_post_processing_worker('t1', 'b1', deps) + + copied = transfer_dir / '10.Harry Styles - American Girls.flac' + assert copied.exists() + assert any(c[0] == 'post_process' and c[1][2] == str(copied) for c in rec.calls) + + +def test_torrent_release_without_matching_file_does_not_fallback_to_generic_search(tmp_path): + release_dir = tmp_path / 'release' + release_dir.mkdir() + wrong = release_dir / '09.Harry Styles - Pop.flac' + wrong.write_bytes(b'wrong') + transfer_dir = tmp_path / 'transfer' + + filename = 'http://prowlarr/download?id=1||Harry Styles - Kiss All The Time' + download_tasks['t1'] = { + 'status': 'post_processing', + 'filename': filename, + 'username': 'torrent', + 'download_id': 'dl-torrent-1', + 'track_info': {'name': 'American Girls', 'artists': [{'name': 'Harry Styles'}]}, + } + matched_downloads_context[f'torrent::{filename}'] = { + 'original_search_result': {'title': 'Pop', 'clean_title': 'Pop', 'track_number': 9}, + } + + class _FakeStatus: + file_path = str(wrong) + audio_files = [str(wrong)] + + class _FakeTorrentClient: + def get_download_status(self, dl_id): + assert dl_id == 'dl-torrent-1' + return _FakeStatus() + + def _unexpected_search(*args, **kwargs): + raise AssertionError("torrent releases should not fall back to generic file search") + + deps, rec = _build_deps( + config=_FakeConfig({'soulseek.transfer_path': str(transfer_dir)}), + download_orchestrator=_FakeTorrentClient(), + run_async=lambda coro: coro, + find_completed_file=_unexpected_search, + ) + + pp.run_post_processing_worker('t1', 'b1', deps) + + assert download_tasks['t1']['status'] == 'failed' + assert 'No matching audio file' in download_tasks['t1']['error_message'] + assert any(c[0] == 'on_complete' and c[1] == ('b1', 't1', False) for c in rec.calls) + assert not list(transfer_dir.glob('*')) + + def test_fuzzy_context_matching_when_exact_key_missing(monkeypatch): """When exact key isn't in matched_downloads_context, worker tries fuzzy match constrained to same Soulseek username.""" diff --git a/tests/downloads/test_downloads_staging.py b/tests/downloads/test_downloads_staging.py index d11153c4..70e86b6b 100644 --- a/tests/downloads/test_downloads_staging.py +++ b/tests/downloads/test_downloads_staging.py @@ -60,6 +60,7 @@ def _build_deps( transfer_path, staging_files=None, post_process_calls=None, + get_batch_field=None, ): post_process_calls = post_process_calls if post_process_calls is not None else [] deps = ds.StagingDeps( @@ -68,6 +69,7 @@ def _build_deps( get_staging_file_cache=lambda batch_id: staging_files or [], docker_resolve_path=lambda p: p, # passthrough post_process_matched_download_with_verification=lambda *a, **kw: post_process_calls.append((a, kw)), + get_batch_field=get_batch_field, ) deps._post_process_calls = post_process_calls return deps @@ -157,6 +159,56 @@ def test_exact_match_copies_to_transfer_and_marks_post_processing(tmp_path): assert context_key == 'staging_t4' +def test_private_album_bundle_staging_source_is_removed_after_claim(tmp_path): + src_file = tmp_path / 'private' / 'Hello.flac' + src_file.parent.mkdir() + src_file.write_bytes(b'fake audio') + + transfer_dir = tmp_path / 'transfer' + + def get_batch_field(_batch_id, field): + if field == 'album_bundle_source': + return 'torrent' + if field == 'album_bundle_private_staging': + return True + return None + + deps = _build_deps( + transfer_path=str(transfer_dir), + staging_files=[ + {'full_path': str(src_file), 'title': 'Hello', 'artist': 'Artist One'}, + ], + get_batch_field=get_batch_field, + ) + _seed_task('t_private') + + result = ds.try_staging_match('t_private', 'b_private', _Track(), deps) + + assert result is True + assert (transfer_dir / 'Hello.flac').exists() + assert not src_file.exists() + assert download_tasks['t_private']['username'] == 'torrent' + + +def test_public_staging_source_is_kept_after_match(tmp_path): + src_file = tmp_path / 'staging' / 'Hello.flac' + src_file.parent.mkdir() + src_file.write_bytes(b'fake audio') + + deps = _build_deps( + transfer_path=str(tmp_path / 'transfer'), + staging_files=[ + {'full_path': str(src_file), 'title': 'Hello', 'artist': 'Artist One'}, + ], + ) + _seed_task('t_public') + + result = ds.try_staging_match('t_public', 'b_public', _Track(), deps) + + assert result is True + assert src_file.exists() + + def test_existing_file_in_transfer_gets_staging_suffix(tmp_path): """If destination already exists, suffix '_staging' added to avoid overwrite.""" src_file = tmp_path / 'staging' / 'Hello.flac' @@ -226,6 +278,180 @@ def test_explicit_album_context_uses_real_data(tmp_path): assert ctx['staging_source'] is True +def test_staging_context_falls_back_to_matched_file_track_number(tmp_path): + """Album-bundle staging can recover numbering from the selected audio file.""" + src_file = tmp_path / 'staging' / '03 - Backseat Freestyle.flac' + src_file.parent.mkdir() + src_file.touch() + + deps = _build_deps( + transfer_path=str(tmp_path / 'transfer'), + staging_files=[ + { + 'full_path': str(src_file), + 'title': 'Backseat Freestyle', + 'artist': 'Kendrick Lamar', + 'track_number': 3, + 'disc_number': 1, + }, + ], + ) + _seed_task('t6b', track_info={ + '_is_explicit_album_download': True, + '_explicit_album_context': {'id': 'alb', 'name': 'good kid, m.A.A.d city (Deluxe)'}, + '_explicit_artist_context': {'id': 'art', 'name': 'Kendrick Lamar'}, + }) + + ds.try_staging_match( + 't6b', 'b1', + _Track(name='Backseat Freestyle', artists=['Kendrick Lamar']), + deps, + ) + + ctx = matched_downloads_context['staging_t6b'] + assert ctx['original_search_result']['track_number'] == 3 + assert ctx['original_search_result']['disc_number'] == 1 + + +def test_private_album_bundle_staging_overrides_default_track_info_number(tmp_path): + """Private release staging trusts the selected file number over weak task defaults.""" + src_file = tmp_path / 'staging' / '04-kendrick_lamar-the_art_of_peer_pressure.flac' + src_file.parent.mkdir() + src_file.touch() + + def get_batch_field(_batch_id, field): + if field == 'album_bundle_source': + return 'torrent' + if field == 'album_bundle_private_staging': + return True + return None + + deps = _build_deps( + transfer_path=str(tmp_path / 'transfer'), + staging_files=[ + { + 'full_path': str(src_file), + 'title': 'The Art of Peer Pressure', + 'artist': 'Kendrick Lamar', + }, + ], + get_batch_field=get_batch_field, + ) + _seed_task('t6c', track_info={ + '_is_explicit_album_download': True, + '_explicit_album_context': {'id': 'alb', 'name': 'good kid, m.A.A.d city (Deluxe)'}, + '_explicit_artist_context': {'id': 'art', 'name': 'Kendrick Lamar'}, + 'track_number': 1, + }) + + ds.try_staging_match( + 't6c', 'b1', + _Track(name='The Art of Peer Pressure', artists=['Kendrick Lamar']), + deps, + ) + + ctx = matched_downloads_context['staging_t6c'] + assert ctx['track_info']['track_number'] == 4 + assert ctx['original_search_result']['track_number'] == 4 + assert ctx['original_search_result']['username'] == 'torrent' + assert ctx['original_search_result']['filename'] == str(src_file) + + +def test_staging_title_match_accepts_feature_suffix_from_release_file(tmp_path): + """Album releases can include featured artists in filenames.""" + src_file = tmp_path / 'staging' / '05-kendrick_lamar-money_trees_(feat._jay_rock).flac' + src_file.parent.mkdir() + src_file.touch() + + deps = _build_deps( + transfer_path=str(tmp_path / 'transfer'), + staging_files=[ + { + 'full_path': str(src_file), + 'title': 'money_trees_(feat._jay_rock)', + 'artist': 'Kendrick Lamar', + 'track_number': 5, + }, + ], + ) + _seed_task('t_feature', track_info={ + '_is_explicit_album_download': True, + '_explicit_album_context': {'id': 'alb', 'name': 'good kid, m.A.A.d city (Deluxe)'}, + '_explicit_artist_context': {'id': 'art', 'name': 'Kendrick Lamar'}, + }) + + result = ds.try_staging_match( + 't_feature', 'b1', + _Track(name='Money Trees', artists=['Kendrick Lamar']), + deps, + ) + + assert result is True + assert matched_downloads_context['staging_t_feature']['track_info']['track_number'] == 5 + + +def test_staging_title_match_accepts_bonus_track_against_release_file(tmp_path): + """Expected bonus labels should not block matching the actual release file.""" + src_file = tmp_path / 'staging' / '13-kendrick_lamar-the_recipe_(feat._dr._dre).flac' + src_file.parent.mkdir() + src_file.touch() + + deps = _build_deps( + transfer_path=str(tmp_path / 'transfer'), + staging_files=[ + { + 'full_path': str(src_file), + 'title': 'the_recipe_(feat._dr._dre)', + 'artist': 'Kendrick Lamar', + 'track_number': 13, + }, + ], + ) + _seed_task('t_bonus', track_info={ + '_is_explicit_album_download': True, + '_explicit_album_context': {'id': 'alb', 'name': 'good kid, m.A.A.d city (Deluxe)'}, + '_explicit_artist_context': {'id': 'art', 'name': 'Kendrick Lamar'}, + }) + + result = ds.try_staging_match( + 't_bonus', 'b1', + _Track(name='The Recipe (Bonus Track)', artists=['Kendrick Lamar']), + deps, + ) + + assert result is True + assert matched_downloads_context['staging_t_bonus']['track_info']['track_number'] == 13 + + +def test_staging_title_match_keeps_wrong_versions_separate(tmp_path): + """Do not strip remix/extended wording when matching staged release files.""" + src_file = tmp_path / 'staging' / '17-kendrick_lamar-swimming_pools_(drank)_(black_hippy_remix).flac' + src_file.parent.mkdir() + src_file.touch() + + deps = _build_deps( + transfer_path=str(tmp_path / 'transfer'), + staging_files=[ + { + 'full_path': str(src_file), + 'title': 'swimming_pools_(drank)_(black_hippy_remix)', + 'artist': 'Kendrick Lamar', + 'track_number': 17, + }, + ], + ) + _seed_task('t_wrong_version') + + result = ds.try_staging_match( + 't_wrong_version', 'b1', + _Track(name='Swimming Pools (Drank) (Extended Version)', artists=['Kendrick Lamar']), + deps, + ) + + assert result is False + assert 'staging_t_wrong_version' not in matched_downloads_context + + def test_fallback_context_synthesizes_from_track(tmp_path): """Without explicit context, synthesizes spotify_artist/album from the track.""" src_file = tmp_path / 'staging' / 'Hello.flac' diff --git a/tests/downloads/test_downloads_status.py b/tests/downloads/test_downloads_status.py index aa690866..706f9131 100644 --- a/tests/downloads/test_downloads_status.py +++ b/tests/downloads/test_downloads_status.py @@ -40,6 +40,8 @@ def _build_deps( make_key=None, submit_pp=None, cached_transfers=None, + download_orchestrator=None, + run_async=None, ): submitted = [] @@ -53,6 +55,8 @@ def _build_deps( make_context_key=make_key or (lambda u, f: f"{u}::{f}"), submit_post_processing=submit_pp or _default_submit, get_cached_transfer_data=cached_transfers or (lambda: {}), + download_orchestrator=download_orchestrator, + run_async=run_async, ) return deps, submitted @@ -85,6 +89,38 @@ def test_analysis_phase_includes_analysis_progress_and_results(): assert out['analysis_results'] == [{'track_index': 0, 'found': True}] +def test_album_downloading_phase_exposes_bundle_progress_without_task_safety_valve(): + deps, _ = _build_deps(config=_FakeConfig({'soulseek.download_timeout': 1})) + download_tasks['t1'] = { + 'track_index': 0, + 'status': 'queued', + 'track_info': {}, + 'status_change_time': 0, + } + batch = { + 'phase': 'album_downloading', + 'queue': ['t1'], + 'album_bundle_state': 'downloading', + 'album_bundle_source': 'torrent', + 'album_bundle_release': 'GNX [FLAC]', + 'album_bundle_progress': 0.42, + 'album_bundle_speed': 2048, + 'album_bundle_size': 4096, + } + out = st.build_batch_status_data('b1', batch, {}, deps) + assert out['album_bundle'] == { + 'state': 'downloading', + 'source': 'torrent', + 'release': 'GNX [FLAC]', + 'progress': 0.42, + 'progress_percent': 42, + 'speed': 2048, + 'size': 4096, + } + assert 'tasks' not in out + assert download_tasks['t1']['status'] == 'queued' + + def test_complete_phase_includes_wishlist_summary_when_present(): deps, _ = _build_deps() batch = { @@ -254,6 +290,128 @@ def test_post_processing_status_progress_is_95(): assert out['tasks'][0]['progress'] == 95 +def test_auto_torrent_without_live_entry_uses_engine_success_fallback(): + class _Record: + state = 'Completed, Succeeded' + progress = 100 + + class _Orchestrator: + def get_download_status(self, download_id): + assert download_id == 'dl1' + return _Record() + + deps, submitted = _build_deps( + download_orchestrator=_Orchestrator(), + run_async=lambda value: value, + ) + download_tasks['t1'] = { + 'track_index': 0, + 'status': 'downloading', + 'track_info': {}, + 'filename': 'song.flac', + 'username': 'torrent', + 'download_id': 'dl1', + } + batch = {'phase': 'downloading', 'queue': ['t1']} + out = st.build_batch_status_data('b1', batch, {}, deps) + assert out['tasks'][0]['status'] == 'post_processing' + assert download_tasks['t1']['status'] == 'post_processing' + assert submitted == [('t1', 'b1')] + + +def test_auto_torrent_prefers_engine_success_over_live_entry(): + class _Record: + state = 'Completed, Succeeded' + progress = 100 + + class _Orchestrator: + def get_download_status(self, download_id): + assert download_id == 'dl1' + return _Record() + + deps, submitted = _build_deps( + download_orchestrator=_Orchestrator(), + run_async=lambda value: value, + ) + download_tasks['t1'] = { + 'track_index': 0, + 'status': 'downloading', + 'track_info': {}, + 'filename': 'song.flac', + 'username': 'torrent', + 'download_id': 'dl1', + } + live = {'torrent::song.flac': { + 'state': 'InProgress, Downloading', + 'percentComplete': 100, + }} + batch = {'phase': 'downloading', 'queue': ['t1']} + out = st.build_batch_status_data('b1', batch, live, deps) + assert out['tasks'][0]['status'] == 'post_processing' + assert download_tasks['t1']['status'] == 'post_processing' + assert submitted == [('t1', 'b1')] + + +def test_auto_torrent_engine_failure_does_not_bypass_monitor_retry(): + class _Record: + state = 'Completed, Errored' + progress = 100 + error = 'client failed' + + class _Orchestrator: + def get_download_status(self, download_id): + assert download_id == 'dl1' + return _Record() + + deps, submitted = _build_deps( + download_orchestrator=_Orchestrator(), + run_async=lambda value: value, + ) + download_tasks['t1'] = { + 'track_index': 0, + 'status': 'downloading', + 'track_info': {}, + 'filename': 'song.flac', + 'username': 'torrent', + 'download_id': 'dl1', + } + batch = {'phase': 'downloading', 'queue': ['t1']} + out = st.build_batch_status_data('b1', batch, {}, deps) + assert out['tasks'][0]['status'] == 'downloading' + assert download_tasks['t1']['status'] == 'downloading' + assert submitted == [] + + +def test_auto_torrent_engine_success_recovers_premature_failed_task(): + class _Record: + state = 'Completed, Succeeded' + progress = 100 + + class _Orchestrator: + def get_download_status(self, download_id): + assert download_id == 'dl1' + return _Record() + + deps, submitted = _build_deps( + download_orchestrator=_Orchestrator(), + run_async=lambda value: value, + ) + download_tasks['t1'] = { + 'track_index': 0, + 'status': 'failed', + 'track_info': {}, + 'filename': 'release-url||Release', + 'username': 'torrent', + 'download_id': 'dl1', + 'error_message': 'premature failure', + } + batch = {'phase': 'downloading', 'queue': ['t1']} + out = st.build_batch_status_data('b1', batch, {}, deps) + assert out['tasks'][0]['status'] == 'post_processing' + assert download_tasks['t1']['status'] == 'post_processing' + assert submitted == [('t1', 'b1')] + + # --------------------------------------------------------------------------- # Safety valve (stuck task handling) # --------------------------------------------------------------------------- @@ -340,6 +498,26 @@ def test_batched_status_no_filter_returns_all_batches(): assert set(out['batches'].keys()) == {'b1', 'b2'} +def test_unified_downloads_response_includes_album_bundle_summary(): + deps, _ = _build_deps() + download_batches['b1'] = { + 'phase': 'album_downloading', + 'playlist_id': 'pl1', + 'playlist_name': 'GNX', + 'queue': [], + 'album_bundle_state': 'downloading', + 'album_bundle_source': 'torrent', + 'album_bundle_progress': 75, + } + out = st.build_unified_downloads_response(300, deps) + assert out['batches'][0]['album_bundle'] == { + 'state': 'downloading', + 'source': 'torrent', + 'progress': 75, + 'progress_percent': 75, + } + + def test_batched_status_metadata_present(): deps, _ = _build_deps() download_batches['b1'] = {'phase': 'unknown'} diff --git a/tests/downloads/test_downloads_task_worker.py b/tests/downloads/test_downloads_task_worker.py index 11ccdfca..e5156b4a 100644 --- a/tests/downloads/test_downloads_task_worker.py +++ b/tests/downloads/test_downloads_task_worker.py @@ -5,14 +5,16 @@ from __future__ import annotations import pytest from core.downloads import task_worker as tw -from core.runtime_state import download_tasks +from core.runtime_state import download_batches, download_tasks @pytest.fixture(autouse=True) def reset_state(): download_tasks.clear() + download_batches.clear() yield download_tasks.clear() + download_batches.clear() # --------------------------------------------------------------------------- @@ -37,7 +39,7 @@ class _FakeClient: orchestrator); plain attrs (hybrid_order, hybrid_primary, etc.) are set as attributes so getattr() lookups still resolve them.""" _CLIENT_NAMES = {'soulseek', 'youtube', 'tidal', 'qobuz', 'hifi', - 'deezer_dl', 'lidarr', 'soundcloud'} + 'deezer_dl', 'lidarr', 'soundcloud', 'torrent', 'usenet'} def __init__(self, results=None, mode='soulseek', subclients=None): self._results = results if results is not None else [] @@ -54,7 +56,7 @@ class _FakeClient: def client(self, name): return self._client_map.get(name) - async def search(self, query, timeout=30): + async def search(self, query, timeout=30, exclude_sources=None): self.search_calls.append((query, timeout)) return (self._results, None) @@ -194,6 +196,33 @@ def test_staging_match_hit_returns_immediately(): assert rec.calls == [] +def test_private_torrent_album_staging_miss_skips_per_track_search(): + _seed_task(track_info={ + 'id': 'sp-1', 'name': 'Money Trees', 'artists': ['Kendrick Lamar'], + 'album': 'good kid, m.A.A.d city (Deluxe)', 'duration_ms': 387000, + }) + download_batches['b1'] = { + 'album_bundle_private_staging': True, + 'album_bundle_state': 'staged', + 'album_bundle_source': 'torrent', + } + client = _FakeClient(results=['should-not-search'], mode='torrent') + rec = _Recorder() + deps, _ = _build_deps( + soulseek=client, + matching=_FakeMatchEngine(queries=['Kendrick Lamar Money Trees']), + try_staging_match=lambda *a, **kw: False, + on_download_completed=rec('done'), + ) + + tw.download_track_worker('t1', 'b1', deps) + + assert client.search_calls == [] + assert download_tasks['t1']['status'] == 'not_found' + assert 'staged torrent album release' in download_tasks['t1']['error_message'] + assert ('done', ('b1', 't1', False), {}) in rec.calls + + # --------------------------------------------------------------------------- # Search loop happy path # --------------------------------------------------------------------------- @@ -219,6 +248,25 @@ def test_first_query_success_returns_after_storing_source(): assert download_tasks['t1']['status'] == 'searching' +def test_torrent_mode_uses_album_release_after_track_queries(): + _seed_task(track_info={ + 'id': 'sp-1', 'name': 'Money', 'artists': ['Pink Floyd'], + 'album': 'The Dark Side of the Moon', 'duration_ms': 383000, + }) + client = _FakeClient(results=[], mode='torrent') + rec = _Recorder() + deps, _ = _build_deps( + soulseek=client, + matching=_FakeMatchEngine(queries=['Pink Floyd Money']), + on_download_completed=rec('done'), + ) + + tw.download_track_worker('t1', 'b1', deps) + + assert client.search_calls[0][0] == 'Pink Floyd Money' + assert client.search_calls[-1][0] == 'Pink Floyd The Dark Side of the Moon' + + def test_no_results_marks_not_found_and_calls_completion(): _seed_task() rec = _Recorder() @@ -271,7 +319,7 @@ def test_cancellation_mid_query_returns_without_completion(): _seed_task() rec = _Recorder() - def _cancel_during_search(query, timeout=30): + def _cancel_during_search(query, timeout=30, exclude_sources=None): download_tasks['t1']['status'] = 'cancelled' async def _empty(): diff --git a/tests/downloads/test_downloads_validation.py b/tests/downloads/test_downloads_validation.py index ddb5346a..1acdf91c 100644 --- a/tests/downloads/test_downloads_validation.py +++ b/tests/downloads/test_downloads_validation.py @@ -123,3 +123,47 @@ def test_keeps_tidal_candidate_inside_integrity_duration_tolerance(monkeypatch): result = get_valid_candidates([tidal], expected, 'Artist Song') assert result == [tidal] + + +def test_rejects_torrent_title_match_from_wrong_artist(monkeypatch): + monkeypatch.setattr(validation, 'matching_engine', _MatchingEngine()) + expected = _Track(duration_ms=180_000, name='The Man I Need', artists=('Olivia Dean',)) + wrong_artist = _Candidate( + username='torrent', + duration=None, + title='The Man I Need', + artist='Tinkabelle', + ) + + assert get_valid_candidates([wrong_artist], expected, 'Olivia Dean The Man I Need') == [] + + +def test_keeps_torrent_title_match_from_expected_artist(monkeypatch): + monkeypatch.setattr(validation, 'matching_engine', _MatchingEngine()) + expected = _Track(duration_ms=180_000, name='The Man I Need', artists=('Olivia Dean',)) + correct_artist = _Candidate( + username='torrent', + duration=None, + title='The Man I Need', + artist='Olivia Dean', + ) + + result = get_valid_candidates([correct_artist], expected, 'Olivia Dean The Man I Need') + + assert result == [correct_artist] + + +def test_keeps_torrent_title_match_when_artist_is_indexer_fallback(monkeypatch): + monkeypatch.setattr(validation, 'matching_engine', _MatchingEngine()) + expected = _Track(duration_ms=180_000, name='The Man I Need', artists=('Olivia Dean',)) + candidate = _Candidate( + username='torrent', + duration=None, + title='The Man I Need', + artist='Indexer', + ) + candidate._source_metadata = {'indexer': 'Indexer'} + + result = get_valid_candidates([candidate], expected, 'Olivia Dean The Man I Need') + + assert result == [candidate] diff --git a/tests/downloads/test_hifi_pinning.py b/tests/downloads/test_hifi_pinning.py index 24ca15f1..d5fc9a8b 100644 --- a/tests/downloads/test_hifi_pinning.py +++ b/tests/downloads/test_hifi_pinning.py @@ -102,3 +102,153 @@ def test_cancel_download_marks_cancelled(hifi_client_with_engine): ok = _run_async(client.cancel_download('dl-1', None, remove=False)) assert ok is True assert engine.get_record('hifi', 'dl-1')['state'] == 'Cancelled' + + +def test_instance_capability_probe_uses_track_manifests_not_legacy_track(): + class _Response: + def __init__(self, *, ok=True, status_code=200, payload=None): + self.ok = ok + self.status_code = status_code + self._payload = payload or {} + + def json(self): + return self._payload + + class _Session: + def __init__(self): + self.calls = [] + + def get(self, url, **kwargs): + self.calls.append((url, kwargs)) + if url.endswith('/search/'): + return _Response(payload={'items': []}) + if url.endswith('/trackManifests/'): + return _Response(payload={ + 'data': { + 'data': { + 'attributes': { + 'uri': 'https://cdn.example/playlist.m3u8', + }, + }, + }, + }) + if url.endswith('/'): + return _Response(payload={'version': 'test'}) + return _Response(ok=False, status_code=404) + + client = HiFiClient.__new__(HiFiClient) + client.session = _Session() + + result = client.check_instance_capabilities('https://hifi.example') + + called_urls = [url for url, _ in client.session.calls] + assert result['can_search'] is True + assert result['can_download'] is True + assert any(url.endswith('/trackManifests/') for url in called_urls) + assert not any(url.endswith('/track') for url in called_urls) + + +def test_instance_capability_probe_reports_manifest_without_uri_as_limited(): + class _Response: + ok = True + status_code = 200 + + def __init__(self, payload): + self._payload = payload + + def json(self): + return self._payload + + class _Session: + def get(self, url, **kwargs): + if url.endswith('/search/'): + return _Response({'items': []}) + if url.endswith('/trackManifests/'): + return _Response({'data': {'data': {'attributes': {}}}}) + if url.endswith('/'): + return _Response({'version': 'test'}) + return _Response({'data': {'data': {'attributes': {}}}}) + + client = HiFiClient.__new__(HiFiClient) + client.session = _Session() + + result = client.check_instance_capabilities('https://hifi.example') + + assert result['can_search'] is True + assert result['can_download'] is False + assert result['download_error'] == 'No playable manifest URL' + + +def test_instance_capability_probe_accepts_legacy_track_manifest(): + import base64 + import json + + class _Response: + def __init__(self, payload, *, ok=True, status_code=200): + self._payload = payload + self.ok = ok + self.status_code = status_code + + def json(self): + return self._payload + + class _Session: + def __init__(self): + self.calls = [] + + def get(self, url, **kwargs): + self.calls.append((url, kwargs)) + if url.endswith('/search/'): + return _Response({'items': []}) + if url.endswith('/trackManifests/'): + return _Response({'data': {'data': {'attributes': {}}}}) + if url.endswith('/track/'): + manifest = base64.b64encode(json.dumps({ + 'mimeType': 'audio/flac', + 'codecs': 'flac', + 'encryptionType': 'NONE', + 'urls': ['https://cdn.example/track.flac'], + }).encode()).decode() + return _Response({'data': {'manifest': manifest}}) + if url.endswith('/'): + return _Response({'version': 'test'}) + return _Response({}, ok=False, status_code=404) + + client = HiFiClient.__new__(HiFiClient) + client.session = _Session() + + result = client.check_instance_capabilities('https://hifi.example') + + assert result['can_search'] is True + assert result['can_download'] is True + assert result['download_probe'] == 'track' + assert any(url.endswith('/track/') for url, _ in client.session.calls) + + +def test_get_hls_manifest_falls_back_to_legacy_track_endpoint(): + import base64 + import json + + client = HiFiClient.__new__(HiFiClient) + calls = [] + + def _fake_api_get(path, params=None, timeout=15): + calls.append((path, params)) + if path == '/trackManifests/': + return None + manifest = base64.b64encode(json.dumps({ + 'mimeType': 'audio/flac', + 'codecs': 'flac', + 'encryptionType': 'NONE', + 'urls': ['https://cdn.example/track.flac'], + }).encode()).decode() + return {'data': {'manifest': manifest}} + + client._api_get = _fake_api_get + + result = client._get_hls_manifest(123, quality='lossless') + + assert result['direct_urls'] == ['https://cdn.example/track.flac'] + assert result['extension'] == 'flac' + assert calls[0][0] == '/trackManifests/' + assert calls[1][0] == '/track/' diff --git a/tests/imports/test_import_side_effects.py b/tests/imports/test_import_side_effects.py index 455d2d4f..136ee765 100644 --- a/tests/imports/test_import_side_effects.py +++ b/tests/imports/test_import_side_effects.py @@ -2,6 +2,8 @@ import os import sqlite3 from types import SimpleNamespace +import pytest + from core.imports import side_effects @@ -364,6 +366,42 @@ def test_library_history_labels_auto_import(monkeypatch): assert captured["title"] == "Auto-Imported Track" +@pytest.mark.parametrize( + ("username", "expected"), + [ + ("torrent", "Torrent"), + ("usenet", "Usenet"), + ("staging", "Staging"), + ], +) +def test_library_history_labels_release_and_staging_sources(monkeypatch, username, expected): + """Release/staging imports should not fall through to the Soulseek label.""" + captured = {} + + class _DBStub: + def add_library_history_entry(self, **kwargs): + captured.update(kwargs) + + monkeypatch.setattr(side_effects, "get_database", lambda: _DBStub()) + + context = { + "track_info": { + "name": "Imported Track", + "artists": [{"name": "Some Artist"}], + "album": "Some Album", + }, + "original_search_result": { + "username": username, + "filename": "source-file.flac", + }, + "_final_processed_path": "/library/some-album/01.flac", + } + + side_effects.record_library_history_download(context) + + assert captured["download_source"] == expected + + # --------------------------------------------------------------------------- # Album duration parity — must equal sum of all track durations, not whatever # the first imported track happened to be. diff --git a/tests/metadata/test_cache_maintenance_retry.py b/tests/metadata/test_cache_maintenance_retry.py new file mode 100644 index 00000000..dc5ee85c --- /dev/null +++ b/tests/metadata/test_cache_maintenance_retry.py @@ -0,0 +1,44 @@ +import sqlite3 +from types import SimpleNamespace + +import core.metadata.cache as cache_module +from core.metadata.cache import MetadataCache + + +def test_maintenance_write_retries_once_after_disk_io(monkeypatch): + cache = MetadataCache() + attempts = [] + + class _Conn: + def close(self): + pass + + monkeypatch.setattr(cache, "_get_db", lambda: SimpleNamespace(_get_connection=lambda: _Conn())) + monkeypatch.setattr(cache_module.time, "sleep", lambda _seconds: None) + + def _operation(_conn): + attempts.append(1) + if len(attempts) == 1: + raise sqlite3.OperationalError("disk I/O error") + return 9 + + assert cache._run_maintenance_write("Cache eviction", _operation) == 9 + assert len(attempts) == 2 + + +def test_maintenance_write_does_not_retry_non_io_errors(monkeypatch): + cache = MetadataCache() + attempts = [] + + class _Conn: + def close(self): + pass + + monkeypatch.setattr(cache, "_get_db", lambda: SimpleNamespace(_get_connection=lambda: _Conn())) + + def _operation(_conn): + attempts.append(1) + raise sqlite3.OperationalError("syntax error") + + assert cache._run_maintenance_write("Cache eviction", _operation) == 0 + assert len(attempts) == 1 diff --git a/tests/test_album_bundle.py b/tests/test_album_bundle.py new file mode 100644 index 00000000..48733c31 --- /dev/null +++ b/tests/test_album_bundle.py @@ -0,0 +1,301 @@ +"""Tests for ``core/download_plugins/album_bundle.py``. + +The shared helpers used by both the torrent and usenet album-bundle +flows. Pins the pick heuristic, the atomic-copy invariant +(no partial files ever visible at the audio extension), the +collision-suffix logic, and the config-driven poll cadence so a +future tweak in either plugin can't break the contract. +""" + +from __future__ import annotations + +import os +import threading +import time +from dataclasses import dataclass +from pathlib import Path +from typing import Optional +from unittest.mock import patch + +import pytest + +from core.download_plugins.album_bundle import ( + ALBUM_PICK_MAX_BYTES, + ALBUM_PICK_MIN_BYTES, + DEFAULT_POLL_INTERVAL_SECONDS, + DEFAULT_POLL_TIMEOUT_SECONDS, + atomic_copy_to_staging, + copy_audio_files_atomically, + get_poll_interval, + get_poll_timeout, + pick_best_album_release, + quality_score, + unique_staging_path, +) + + +# Minimal release-result shim — duck-types the fields the picker reads. +@dataclass +class _Release: + title: str + size: int + seeders: Optional[int] = None + grabs: Optional[int] = None + + +def _flac_quality_guess(title: str) -> str: + """Stand-in for the plugin's title→quality function.""" + t = (title or '').lower() + if 'flac' in t: + return 'flac' + if 'aac' in t: + return 'aac' + if 'ogg' in t: + return 'ogg' + return 'mp3' + + +# --------------------------------------------------------------------------- +# pick_best_album_release +# --------------------------------------------------------------------------- + + +def test_picker_returns_none_for_empty_input() -> None: + assert pick_best_album_release([], _flac_quality_guess) is None + + +def test_picker_drops_singletons_when_albums_present() -> None: + """Single-track torrents under 40 MB shouldn't beat an album-sized + candidate even if the single has thousands of seeders.""" + single = _Release(title='Track [MP3]', size=10_000_000, seeders=10_000) + album = _Release(title='Album [MP3]', size=120_000_000, seeders=5) + assert pick_best_album_release([single, album], _flac_quality_guess) is album + + +def test_picker_prefers_flac_when_tied_on_seeders() -> None: + flac = _Release(title='Album [FLAC]', size=400_000_000, seeders=50) + mp3 = _Release(title='Album [MP3]', size=130_000_000, seeders=50) + assert pick_best_album_release([flac, mp3], _flac_quality_guess) is flac + + +def test_picker_uses_grabs_when_seeders_is_none() -> None: + """Usenet results have ``seeders=None`` — the picker should fall + back to ``grabs`` so popularity still drives the ranking.""" + cold = _Release(title='Album A [MP3]', size=200_000_000, seeders=None, grabs=1) + popular = _Release(title='Album B [MP3]', size=200_000_000, seeders=None, grabs=999) + assert pick_best_album_release([cold, popular], _flac_quality_guess) is popular + + +def test_picker_falls_back_when_all_below_floor() -> None: + """When every candidate is below the 40 MB album-size floor, + return the most-seeded one rather than None — the user still + wants a download attempt.""" + small_low = _Release(title='X', size=5_000_000, seeders=10) + small_high = _Release(title='Y', size=8_000_000, seeders=200) + assert pick_best_album_release([small_low, small_high], _flac_quality_guess) is small_high + + +def test_picker_size_floor_matches_constant() -> None: + """If someone moves the constant the floor moves with it — pin + the relationship to catch accidental literals creeping back in.""" + just_below = _Release(title='Below', size=ALBUM_PICK_MIN_BYTES - 1, seeders=999) + just_above = _Release(title='Above', size=ALBUM_PICK_MIN_BYTES + 1, seeders=1) + assert pick_best_album_release([just_below, just_above], _flac_quality_guess) is just_above + + +def test_picker_rejects_oversized_box_sets() -> None: + """Anything past 3 GB drops out of the preferred pool — most likely + a multi-disc box set with scans + bonus material, not what the + user asked for.""" + sane = _Release(title='Album [FLAC]', size=400_000_000, seeders=10) + box = _Release(title='Album Box [FLAC]', size=ALBUM_PICK_MAX_BYTES + 1_000_000, seeders=999) + # Sane wins even with 100x fewer seeders, because box is outside + # the preferred range. + assert pick_best_album_release([sane, box], _flac_quality_guess) is sane + + +# --------------------------------------------------------------------------- +# quality_score +# --------------------------------------------------------------------------- + + +def test_quality_score_orders_formats() -> None: + assert quality_score('Album [FLAC]', _flac_quality_guess) > quality_score('Album [MP3]', _flac_quality_guess) + assert quality_score('Album [AAC]', _flac_quality_guess) > quality_score('Album [MP3]', _flac_quality_guess) + assert quality_score('Bare title', _flac_quality_guess) == quality_score('Album [MP3]', _flac_quality_guess) + + +# --------------------------------------------------------------------------- +# unique_staging_path +# --------------------------------------------------------------------------- + + +def test_unique_staging_path_returns_natural_when_clear(tmp_path: Path) -> None: + src = tmp_path / 'src.flac' + src.write_bytes(b'fLaC') + staging = tmp_path / 'staging' + staging.mkdir() + assert unique_staging_path(staging, src) == staging / 'src.flac' + + +def test_unique_staging_path_suffixes_on_collision(tmp_path: Path) -> None: + src = tmp_path / 'src.flac' + src.write_bytes(b'fLaC') + staging = tmp_path / 'staging' + staging.mkdir() + (staging / 'src.flac').write_bytes(b'existing') + assert unique_staging_path(staging, src) == staging / 'src_1.flac' + + +def test_unique_staging_path_increments_suffix(tmp_path: Path) -> None: + src = tmp_path / 'src.flac' + src.write_bytes(b'fLaC') + staging = tmp_path / 'staging' + staging.mkdir() + (staging / 'src.flac').write_bytes(b'1') + (staging / 'src_1.flac').write_bytes(b'2') + (staging / 'src_2.flac').write_bytes(b'3') + assert unique_staging_path(staging, src) == staging / 'src_3.flac' + + +# --------------------------------------------------------------------------- +# atomic_copy_to_staging +# --------------------------------------------------------------------------- + + +def test_atomic_copy_lands_at_final_path(tmp_path: Path) -> None: + src = tmp_path / 'src.flac' + src.write_bytes(b'fLaC payload') + dest = tmp_path / 'staging' / 'track.flac' + dest.parent.mkdir() + assert atomic_copy_to_staging(src, dest) is True + assert dest.read_bytes() == b'fLaC payload' + + +def test_atomic_copy_leaves_no_tmp_files_after_success(tmp_path: Path) -> None: + """The .tmp. sidecar must be cleaned up by the rename — + no orphan files left behind on a successful copy.""" + src = tmp_path / 'src.flac' + src.write_bytes(b'data') + dest = tmp_path / 'staging' / 'track.flac' + dest.parent.mkdir() + atomic_copy_to_staging(src, dest) + tmp_files = list(dest.parent.glob('*.tmp.*')) + assert tmp_files == [] + + +def test_atomic_copy_never_exposes_partial_to_extension_scanner(tmp_path: Path) -> None: + """Auto-Import filters by audio extension — the in-flight file + must NEVER be visible at its final extension until the copy is + complete. We probe this by scanning the staging dir in parallel + with the copy and assert the audio file is either absent OR + fully written. + """ + src = tmp_path / 'src.flac' + src.write_bytes(b'x' * (2 * 1024 * 1024)) + dest = tmp_path / 'staging' / 'track.flac' + dest.parent.mkdir() + + stop = threading.Event() + saw_partial = threading.Event() + expected_size = src.stat().st_size + + def _scan_loop(): + while not stop.is_set(): + try: + files = [p for p in dest.parent.iterdir() if p.suffix == '.flac'] + except FileNotFoundError: + continue + for fp in files: + size = fp.stat().st_size + if 0 < size < expected_size: + saw_partial.set() + return + + scanner = threading.Thread(target=_scan_loop, daemon=True) + scanner.start() + try: + for i in range(5): + target = dest.with_name(f'track_{i}.flac') + atomic_copy_to_staging(src, target) + # Give the scanner a moment to drain any final scan iteration. + time.sleep(0.05) + finally: + stop.set() + scanner.join(timeout=1.0) + + assert not saw_partial.is_set(), \ + "Scanner observed a partial audio file — atomic copy contract broken" + + +def test_copy_audio_files_atomically_skips_failures(tmp_path: Path) -> None: + """One file failing to copy shouldn't stop the rest from being + staged — partial results are better than a complete bailout.""" + src_a = tmp_path / 'a.flac' + src_a.write_bytes(b'a') + src_missing = tmp_path / 'does-not-exist.flac' # never created + src_c = tmp_path / 'c.flac' + src_c.write_bytes(b'c') + staging = tmp_path / 'staging' + out = copy_audio_files_atomically([src_a, src_missing, src_c], staging) + assert len(out) == 2 + landed = sorted(Path(p).name for p in out) + assert landed == ['a.flac', 'c.flac'] + + +def test_copy_audio_files_atomically_creates_staging_dir(tmp_path: Path) -> None: + src = tmp_path / 'a.flac' + src.write_bytes(b'a') + staging = tmp_path / 'nested' / 'staging' / 'dir' + out = copy_audio_files_atomically([src], staging) + assert len(out) == 1 + assert staging.exists() + + +# --------------------------------------------------------------------------- +# Config-driven poll cadence +# --------------------------------------------------------------------------- + + +def test_get_poll_interval_uses_default_when_unset() -> None: + with patch('core.download_plugins.album_bundle.config_manager') as cm: + cm.get.return_value = DEFAULT_POLL_INTERVAL_SECONDS + assert get_poll_interval() == DEFAULT_POLL_INTERVAL_SECONDS + + +def test_get_poll_interval_honours_override() -> None: + with patch('core.download_plugins.album_bundle.config_manager') as cm: + cm.get.return_value = 5 + assert get_poll_interval() == 5.0 + + +def test_get_poll_interval_falls_back_on_garbage() -> None: + """Non-numeric / non-positive values fall back to the default + rather than crashing the poll loop.""" + with patch('core.download_plugins.album_bundle.config_manager') as cm: + cm.get.return_value = 'not-a-number' + assert get_poll_interval() == DEFAULT_POLL_INTERVAL_SECONDS + cm.get.return_value = -1 + assert get_poll_interval() == DEFAULT_POLL_INTERVAL_SECONDS + + +def test_get_poll_timeout_uses_default_when_unset() -> None: + with patch('core.download_plugins.album_bundle.config_manager') as cm: + cm.get.return_value = DEFAULT_POLL_TIMEOUT_SECONDS + assert get_poll_timeout() == DEFAULT_POLL_TIMEOUT_SECONDS + + +def test_get_poll_timeout_honours_override() -> None: + """Users with slow trackers / large box sets can extend the + deadline without touching code.""" + with patch('core.download_plugins.album_bundle.config_manager') as cm: + cm.get.return_value = 86_400 # 24h + assert get_poll_timeout() == 86_400.0 + + +def test_get_poll_timeout_falls_back_on_garbage() -> None: + with patch('core.download_plugins.album_bundle.config_manager') as cm: + cm.get.return_value = '' + assert get_poll_timeout() == DEFAULT_POLL_TIMEOUT_SECONDS + cm.get.return_value = 0 + assert get_poll_timeout() == DEFAULT_POLL_TIMEOUT_SECONDS diff --git a/tests/test_album_bundle_dispatch.py b/tests/test_album_bundle_dispatch.py new file mode 100644 index 00000000..68579f6d --- /dev/null +++ b/tests/test_album_bundle_dispatch.py @@ -0,0 +1,322 @@ +"""Tests for ``core/downloads/album_bundle_dispatch.py``. + +Pins the gate predicate, the resolution + run flow, and the +fail / fall-through return contract. Mocks the config, plugin +resolver, and state access so the dispatcher is testable without +standing up runtime_state or a real plugin. +""" + +from __future__ import annotations + +from unittest.mock import MagicMock + +import pytest + +from core.downloads.album_bundle_dispatch import ( + BatchStateAccess, + is_eligible, + try_dispatch, +) + + +class _FakeState: + """In-memory ``BatchStateAccess`` for tests — records every + update so assertions can check the sequence of fields set.""" + + def __init__(self) -> None: + self.fields: dict = {} + self.update_calls: list = [] + self.failed_with: str = '' + + def update_fields(self, batch_id: str, fields: dict) -> None: + self.update_calls.append((batch_id, dict(fields))) + self.fields.update(fields) + + def mark_failed(self, batch_id: str, error: str) -> None: + self.failed_with = error + self.fields['phase'] = 'failed' + self.fields['error'] = error + self.fields['album_bundle_state'] = 'failed' + + +def _config(values: dict): + """Build a config_get callable from a flat dict.""" + def _get(key, default=None): + return values.get(key, default) + return _get + + +# --------------------------------------------------------------------------- +# is_eligible pure predicate +# --------------------------------------------------------------------------- + + +def test_is_eligible_requires_album_flag() -> None: + assert is_eligible(mode='torrent', is_album=False, + album_name='X', artist_name='Y') is False + + +def test_is_eligible_requires_torrent_or_usenet_mode() -> None: + for mode in ('soulseek', 'youtube', 'tidal', 'qobuz', 'hifi', + 'deezer_dl', 'amazon', 'lidarr', 'soundcloud', 'hybrid'): + assert is_eligible(mode=mode, is_album=True, + album_name='X', artist_name='Y') is False + + +def test_is_eligible_accepts_torrent_and_usenet() -> None: + assert is_eligible(mode='torrent', is_album=True, + album_name='X', artist_name='Y') is True + assert is_eligible(mode='usenet', is_album=True, + album_name='X', artist_name='Y') is True + + +def test_is_eligible_requires_non_empty_names() -> None: + assert is_eligible(mode='torrent', is_album=True, + album_name='', artist_name='Y') is False + assert is_eligible(mode='torrent', is_album=True, + album_name='X', artist_name='') is False + assert is_eligible(mode='torrent', is_album=True, + album_name=' ', artist_name='Y') is False + + +def test_is_eligible_case_insensitive_mode() -> None: + assert is_eligible(mode='TORRENT', is_album=True, + album_name='X', artist_name='Y') is True + + +# --------------------------------------------------------------------------- +# try_dispatch — gate evaluation +# --------------------------------------------------------------------------- + + +def test_dispatch_returns_false_when_not_album() -> None: + state = _FakeState() + plugin = MagicMock() + result = try_dispatch( + batch_id='b1', is_album=False, + album_context={'name': 'X'}, artist_context={'name': 'Y'}, + config_get=_config({'download_source.mode': 'torrent'}), + plugin_resolver=lambda _name: plugin, state=state, + ) + assert result is False + assert state.update_calls == [] + plugin.download_album_to_staging.assert_not_called() + + +def test_dispatch_returns_false_for_non_torrent_modes() -> None: + state = _FakeState() + plugin = MagicMock() + result = try_dispatch( + batch_id='b1', is_album=True, + album_context={'name': 'X'}, artist_context={'name': 'Y'}, + config_get=_config({'download_source.mode': 'soulseek'}), + plugin_resolver=lambda _name: plugin, state=state, + ) + assert result is False + assert state.update_calls == [] + + +def test_dispatch_returns_false_when_plugin_missing() -> None: + """No plugin available → fall through to per-track flow with a + warning. The state SHOULD NOT have been touched.""" + state = _FakeState() + result = try_dispatch( + batch_id='b1', is_album=True, + album_context={'name': 'X'}, artist_context={'name': 'Y'}, + config_get=_config({'download_source.mode': 'torrent'}), + plugin_resolver=lambda _name: None, state=state, + ) + assert result is False + assert state.update_calls == [] + + +def test_dispatch_returns_false_when_plugin_lacks_method() -> None: + state = _FakeState() + # Plugin that doesn't implement download_album_to_staging. + class _LegacyPlugin: + pass + result = try_dispatch( + batch_id='b1', is_album=True, + album_context={'name': 'X'}, artist_context={'name': 'Y'}, + config_get=_config({'download_source.mode': 'torrent'}), + plugin_resolver=lambda _name: _LegacyPlugin(), state=state, + ) + assert result is False + assert state.update_calls == [] + + +def test_dispatch_returns_false_when_resolver_raises() -> None: + """Plugin resolution can fail (registry not initialised); we log + and fall through rather than crashing the master worker.""" + state = _FakeState() + def _boom(_name): + raise RuntimeError("registry not initialised") + result = try_dispatch( + batch_id='b1', is_album=True, + album_context={'name': 'X'}, artist_context={'name': 'Y'}, + config_get=_config({'download_source.mode': 'torrent'}), + plugin_resolver=_boom, state=state, + ) + assert result is False + + +# --------------------------------------------------------------------------- +# try_dispatch — success / failure paths +# --------------------------------------------------------------------------- + + +def test_dispatch_success_returns_false_so_per_track_can_run() -> None: + """Success → master worker should CONTINUE to per-track flow so + each task can hit try_staging_match and find its file.""" + state = _FakeState() + plugin = MagicMock() + plugin.download_album_to_staging.return_value = { + 'success': True, 'files': ['/tmp/a.flac', '/tmp/b.flac'], 'error': None, + } + result = try_dispatch( + batch_id='b1', is_album=True, + album_context={'name': 'GNX'}, artist_context={'name': 'Kendrick Lamar'}, + config_get=_config({ + 'download_source.mode': 'torrent', + 'import.staging_path': '/staging/path', + }), + plugin_resolver=lambda _name: plugin, state=state, + ) + assert result is False + # Plugin was called with the right args. + args = plugin.download_album_to_staging.call_args + assert args.args[0] == 'GNX' + assert args.args[1] == 'Kendrick Lamar' + assert args.args[2].replace('\\', '/').endswith('storage/album_bundle_staging/b1') + # Phase transitioned through searching → analysis. + assert state.fields['phase'] == 'analysis' + assert state.fields['album_bundle_state'] == 'staged' + assert state.fields['album_bundle_source'] == 'torrent' + assert state.fields['album_bundle_private_staging'] is True + assert state.fields['album_bundle_staging_path'].replace('\\', '/').endswith('storage/album_bundle_staging/b1') + assert state.failed_with == '' + + +def test_dispatch_uses_configured_private_album_bundle_staging_root() -> None: + state = _FakeState() + plugin = MagicMock() + plugin.download_album_to_staging.return_value = {'success': True, 'files': ['/tmp/a.flac']} + + try_dispatch( + batch_id='batch:with/slash', is_album=True, + album_context={'name': 'GNX'}, artist_context={'name': 'Kendrick Lamar'}, + config_get=_config({ + 'download_source.mode': 'torrent', + 'download_source.album_bundle_staging_path': '/private/staging', + }), + plugin_resolver=lambda _name: plugin, state=state, + ) + + staging_arg = plugin.download_album_to_staging.call_args.args[2].replace('\\', '/') + assert staging_arg == '/private/staging/batch_with_slash' + assert state.fields['album_bundle_staging_path'].replace('\\', '/') == staging_arg + + +def test_dispatch_failure_returns_true_so_master_stops() -> None: + state = _FakeState() + plugin = MagicMock() + plugin.download_album_to_staging.return_value = { + 'success': False, 'files': [], 'error': 'No torrent results found', + } + result = try_dispatch( + batch_id='b1', is_album=True, + album_context={'name': 'GNX'}, artist_context={'name': 'Kendrick Lamar'}, + config_get=_config({'download_source.mode': 'torrent'}), + plugin_resolver=lambda _name: plugin, state=state, + ) + assert result is True + assert state.failed_with == 'No torrent results found' + assert state.fields['phase'] == 'failed' + + +def test_dispatch_plugin_exception_treated_as_failure() -> None: + """A bug / network error in the plugin must not propagate into + the master worker — caught + treated as a normal failure so + the batch reports the error cleanly.""" + state = _FakeState() + plugin = MagicMock() + plugin.download_album_to_staging.side_effect = RuntimeError("network down") + result = try_dispatch( + batch_id='b1', is_album=True, + album_context={'name': 'GNX'}, artist_context={'name': 'Kendrick Lamar'}, + config_get=_config({'download_source.mode': 'torrent'}), + plugin_resolver=lambda _name: plugin, state=state, + ) + assert result is True + assert 'network down' in state.failed_with + + +def test_dispatch_strips_whitespace_from_names() -> None: + """Trailing whitespace in batch context shouldn't fail the + eligibility predicate AND should be cleaned before passing to + the plugin.""" + state = _FakeState() + plugin = MagicMock() + plugin.download_album_to_staging.return_value = {'success': True, 'files': ['/x']} + try_dispatch( + batch_id='b1', is_album=True, + album_context={'name': ' GNX '}, artist_context={'name': ' Kendrick '}, + config_get=_config({'download_source.mode': 'torrent'}), + plugin_resolver=lambda _name: plugin, state=state, + ) + args = plugin.download_album_to_staging.call_args + assert args.args[0] == 'GNX' + assert args.args[1] == 'Kendrick' + + +def test_dispatch_progress_callback_mirrors_payload_to_state() -> None: + """The progress callback the plugin gets must mirror its + payload onto the batch state under ``album_bundle_*`` keys so + the Downloads page can render progress while the torrent + download runs.""" + state = _FakeState() + captured_emit = {} + + def _capture(album, artist, staging, emit): + captured_emit['fn'] = emit + emit({'state': 'searching', 'release': 'GNX [FLAC]'}) + emit({'state': 'downloading', 'progress': 0.42, 'speed': 1024 * 1024}) + emit({'state': 'staged', 'count': 12}) + return {'success': True, 'files': []} + + plugin = MagicMock() + plugin.download_album_to_staging.side_effect = _capture + try_dispatch( + batch_id='b1', is_album=True, + album_context={'name': 'GNX'}, artist_context={'name': 'Kendrick Lamar'}, + config_get=_config({'download_source.mode': 'torrent'}), + plugin_resolver=lambda _name: plugin, state=state, + ) + # State should have seen each of the three lifecycle emissions. + states_seen = [fields.get('album_bundle_state') + for _, fields in state.update_calls + if 'album_bundle_state' in fields] + assert 'searching' in states_seen + assert 'downloading' in states_seen + assert 'staged' in states_seen + # Numeric progress + release name made it through. + assert state.fields['album_bundle_release'] == 'GNX [FLAC]' + assert state.fields['album_bundle_progress'] == 0.42 + assert state.fields['album_bundle_count'] == 12 + + +# --------------------------------------------------------------------------- +# Protocol conformance — runtime impl must satisfy the contract +# --------------------------------------------------------------------------- + + +def test_runtime_state_impl_matches_protocol() -> None: + """Sanity check that the concrete BatchStateAccess impl in + master.py implements both methods. We don't import master.py + here (would pull in heavy deps); duck-check on the _FakeState + instead since it's a sibling impl of the same Protocol.""" + state: BatchStateAccess = _FakeState() + state.update_fields('b1', {'x': 1}) + state.mark_failed('b1', 'oops') + assert state.fields['x'] == 1 + assert state.fields['error'] == 'oops' diff --git a/tests/test_archive_pipeline.py b/tests/test_archive_pipeline.py new file mode 100644 index 00000000..053c200a --- /dev/null +++ b/tests/test_archive_pipeline.py @@ -0,0 +1,238 @@ +"""Tests for ``core/archive_pipeline.py``. + +Covers the audio-file walker, archive detector, and zip / tar +extraction (rar / 7z paths use optional deps so they're only +exercised when the libs are present in the test environment). +Path-traversal protection gets explicit coverage — a malicious +archive must not escape the extraction directory. +""" + +from __future__ import annotations + +import os +import tarfile +import zipfile +from pathlib import Path + +import pytest + +from core.archive_pipeline import ( + AUDIO_EXTENSIONS, + ARCHIVE_EXTENSIONS, + collect_audio_after_extraction, + extract_archive, + extract_all_in_dir, + find_archives_in_dir, + is_archive, + walk_audio_files, +) + + +# --------------------------------------------------------------------------- +# Constants +# --------------------------------------------------------------------------- + + +def test_audio_extensions_cover_common_formats() -> None: + for ext in ('.flac', '.mp3', '.m4a', '.ogg', '.opus', '.wav', '.aac', '.wma'): + assert ext in AUDIO_EXTENSIONS + + +def test_archive_extensions_cover_common_formats() -> None: + for ext in ('.zip', '.rar', '.tar', '.7z'): + assert ext in ARCHIVE_EXTENSIONS + + +# --------------------------------------------------------------------------- +# is_archive +# --------------------------------------------------------------------------- + + +def test_is_archive_detects_simple_extensions(tmp_path: Path) -> None: + zip_path = tmp_path / 'x.zip' + zip_path.write_bytes(b'PK\x03\x04') # minimal — is_archive doesn't validate content + assert is_archive(zip_path) is True + + +def test_is_archive_detects_compound_tar_extensions(tmp_path: Path) -> None: + """``.tar.gz`` etc. — Path.suffix only catches the last suffix, + so the detector has to special-case compound extensions.""" + targz = tmp_path / 'x.tar.gz' + targz.write_bytes(b'\x1f\x8b') + assert is_archive(targz) is True + + +def test_is_archive_returns_false_for_audio(tmp_path: Path) -> None: + flac = tmp_path / 'song.flac' + flac.write_bytes(b'fLaC') + assert is_archive(flac) is False + + +def test_is_archive_returns_false_for_missing_file(tmp_path: Path) -> None: + assert is_archive(tmp_path / 'does-not-exist.zip') is False + + +# --------------------------------------------------------------------------- +# walk_audio_files +# --------------------------------------------------------------------------- + + +def test_walk_audio_files_finds_nested(tmp_path: Path) -> None: + # Layout: root/album/disc1/track.flac + root/album/disc2/track.mp3 + (tmp_path / 'album' / 'disc1').mkdir(parents=True) + (tmp_path / 'album' / 'disc2').mkdir(parents=True) + (tmp_path / 'album' / 'disc1' / 'track1.flac').write_bytes(b'fLaC') + (tmp_path / 'album' / 'disc2' / 'track1.mp3').write_bytes(b'ID3') + (tmp_path / 'album' / 'cover.jpg').write_bytes(b'\xff\xd8') + found = walk_audio_files(tmp_path) + names = sorted(p.name for p in found) + assert names == ['track1.flac', 'track1.mp3'] + + +def test_walk_audio_files_returns_empty_for_missing(tmp_path: Path) -> None: + assert walk_audio_files(tmp_path / 'does-not-exist') == [] + + +def test_walk_audio_files_ignores_non_audio(tmp_path: Path) -> None: + (tmp_path / 'readme.txt').write_text('hi') + (tmp_path / 'cover.png').write_bytes(b'\x89PNG') + assert walk_audio_files(tmp_path) == [] + + +def test_walk_audio_files_case_insensitive_extension(tmp_path: Path) -> None: + """Lots of torrents have uppercase extensions (.MP3, .FLAC) — + the walker must catch those too.""" + (tmp_path / 'TRACK.MP3').write_bytes(b'ID3') + (tmp_path / 'TRACK.FLAC').write_bytes(b'fLaC') + found = walk_audio_files(tmp_path) + assert len(found) == 2 + + +# --------------------------------------------------------------------------- +# find_archives_in_dir +# --------------------------------------------------------------------------- + + +def test_find_archives_in_dir_only_top_level(tmp_path: Path) -> None: + """find_archives doesn't recurse — torrents put the archive at + the top of the dir; deeper search risks extracting unrelated + archives that ship inside a sample folder, etc.""" + (tmp_path / 'album.zip').write_bytes(b'PK\x03\x04') + nested = tmp_path / 'subdir' + nested.mkdir() + (nested / 'nested.zip').write_bytes(b'PK\x03\x04') + found = find_archives_in_dir(tmp_path) + assert [p.name for p in found] == ['album.zip'] + + +def test_find_archives_in_dir_empty(tmp_path: Path) -> None: + assert find_archives_in_dir(tmp_path) == [] + assert find_archives_in_dir(tmp_path / 'missing') == [] + + +# --------------------------------------------------------------------------- +# extract_archive — zip +# --------------------------------------------------------------------------- + + +def test_extract_zip_writes_files(tmp_path: Path) -> None: + zip_path = tmp_path / 'album.zip' + with zipfile.ZipFile(zip_path, 'w') as zf: + zf.writestr('track1.mp3', b'ID3 track1 data') + zf.writestr('track2.flac', b'fLaC data') + result = extract_archive(zip_path) + assert result == zip_path.parent + assert (tmp_path / 'track1.mp3').exists() + assert (tmp_path / 'track2.flac').exists() + + +def test_extract_zip_rejects_path_traversal(tmp_path: Path) -> None: + """A malicious archive trying to write ``../../etc/passwd`` must + be refused without extracting anything.""" + zip_path = tmp_path / 'evil.zip' + extract_dest = tmp_path / 'staging' + extract_dest.mkdir() + with zipfile.ZipFile(zip_path, 'w') as zf: + zf.writestr('../escaped.txt', b'evil') + zf.writestr('safe.mp3', b'ID3') + extract_archive(zip_path, extract_to=extract_dest) + # Neither file should have landed — extraction aborts on the first + # traversal attempt. + assert not (extract_dest / 'safe.mp3').exists() + assert not (tmp_path / 'escaped.txt').exists() + + +def test_extract_zip_returns_none_for_bad_zip(tmp_path: Path) -> None: + bad = tmp_path / 'not-a-zip.zip' + bad.write_bytes(b'this is not a zip') + assert extract_archive(bad) is None + + +def test_extract_archive_missing_file_returns_none(tmp_path: Path) -> None: + assert extract_archive(tmp_path / 'does-not-exist.zip') is None + + +# --------------------------------------------------------------------------- +# extract_archive — tar +# --------------------------------------------------------------------------- + + +def test_extract_tar_gz_writes_files(tmp_path: Path) -> None: + payload = tmp_path / 'track.mp3' + payload.write_bytes(b'ID3 track') + tar_path = tmp_path / 'album.tar.gz' + with tarfile.open(tar_path, 'w:gz') as tf: + tf.add(payload, arcname='track.mp3') + payload.unlink() # remove the source so we can verify the extract recreated it + extract_archive(tar_path) + assert (tmp_path / 'track.mp3').exists() + + +def test_extract_tar_rejects_path_traversal(tmp_path: Path) -> None: + extract_dest = tmp_path / 'staging' + extract_dest.mkdir() + tar_path = tmp_path / 'evil.tar' + payload = tmp_path / 'src.txt' + payload.write_bytes(b'evil') + with tarfile.open(tar_path, 'w') as tf: + info = tf.gettarinfo(str(payload), arcname='../escaped.txt') + with payload.open('rb') as fh: + tf.addfile(info, fh) + extract_archive(tar_path, extract_to=extract_dest) + assert not (tmp_path / 'escaped.txt').exists() + + +# --------------------------------------------------------------------------- +# extract_all_in_dir + collect_audio_after_extraction +# --------------------------------------------------------------------------- + + +def test_extract_all_in_dir_handles_multiple_archives(tmp_path: Path) -> None: + (tmp_path / 'one.zip') # placeholder + z1 = tmp_path / 'one.zip' + z2 = tmp_path / 'two.zip' + with zipfile.ZipFile(z1, 'w') as zf: + zf.writestr('a.mp3', b'a') + with zipfile.ZipFile(z2, 'w') as zf: + zf.writestr('b.mp3', b'b') + out = extract_all_in_dir(tmp_path) + assert len(out) == 2 + assert (tmp_path / 'a.mp3').exists() + assert (tmp_path / 'b.mp3').exists() + + +def test_collect_audio_after_extraction_combines_loose_and_extracted(tmp_path: Path) -> None: + """The typical mixed case: torrent dropped a .zip and also some + loose .mp3 files alongside it. The collector returns BOTH.""" + (tmp_path / 'bonus.mp3').write_bytes(b'ID3') + zip_path = tmp_path / 'main.zip' + with zipfile.ZipFile(zip_path, 'w') as zf: + zf.writestr('track1.flac', b'fLaC') + zf.writestr('track2.flac', b'fLaC') + found = collect_audio_after_extraction(tmp_path) + names = sorted(p.name for p in found) + assert names == ['bonus.mp3', 'track1.flac', 'track2.flac'] + + +def test_collect_audio_after_extraction_no_archives_no_audio(tmp_path: Path) -> None: + assert collect_audio_after_extraction(tmp_path) == [] diff --git a/tests/test_database_io_resilience.py b/tests/test_database_io_resilience.py new file mode 100644 index 00000000..37da1cc7 --- /dev/null +++ b/tests/test_database_io_resilience.py @@ -0,0 +1,98 @@ +import sqlite3 + +from database.music_database import MusicDatabase + + +def test_clear_server_data_does_not_fail_when_vacuum_hits_disk_io(): + db = object.__new__(MusicDatabase) + + class _Cursor: + rowcount = 0 + + def __init__(self): + self.calls = [] + + def execute(self, query, params=None): + self.calls.append((query, params)) + if query == "VACUUM": + raise sqlite3.OperationalError("disk I/O error") + if "tracks" in query: + self.rowcount = 1500 + elif "albums" in query: + self.rowcount = 200 + elif "artists" in query: + self.rowcount = 20 + + class _Conn: + def __init__(self): + self.cursor_obj = _Cursor() + self.commits = 0 + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc, tb): + return False + + def cursor(self): + return self.cursor_obj + + def commit(self): + self.commits += 1 + + conn = _Conn() + db._get_connection = lambda: conn + + db.clear_server_data("jellyfin") + + assert conn.commits == 1 + assert any(call[0] == "VACUUM" for call in conn.cursor_obj.calls) + + +def test_clear_server_data_retries_transient_disk_io_before_commit(monkeypatch): + db = object.__new__(MusicDatabase) + connections = [] + + class _Cursor: + rowcount = 0 + + def __init__(self, fail_first_delete=False): + self.fail_first_delete = fail_first_delete + self.calls = [] + + def execute(self, query, params=None): + self.calls.append((query, params)) + if self.fail_first_delete and "DELETE FROM tracks" in query: + self.fail_first_delete = False + raise sqlite3.OperationalError("disk I/O error") + self.rowcount = 1 + + class _Conn: + def __init__(self, fail_first_delete=False): + self.cursor_obj = _Cursor(fail_first_delete=fail_first_delete) + self.commits = 0 + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc, tb): + return False + + def cursor(self): + return self.cursor_obj + + def commit(self): + self.commits += 1 + + def _connect(): + conn = _Conn(fail_first_delete=not connections) + connections.append(conn) + return conn + + db._get_connection = _connect + monkeypatch.setattr("database.music_database.time.sleep", lambda _seconds: None) + + db.clear_server_data("jellyfin") + + assert len(connections) == 2 + assert connections[1].commits == 1 diff --git a/tests/test_download_plugin_conformance.py b/tests/test_download_plugin_conformance.py index 4f380076..68483adc 100644 --- a/tests/test_download_plugin_conformance.py +++ b/tests/test_download_plugin_conformance.py @@ -85,6 +85,7 @@ def test_default_registry_registers_all_sources(): expected = { 'soulseek', 'youtube', 'tidal', 'qobuz', 'hifi', 'deezer', 'lidarr', 'soundcloud', 'amazon', + 'torrent', 'usenet', } assert set(registry.names()) == expected diff --git a/tests/test_library_disk_usage.py b/tests/test_library_disk_usage.py index 00707aa7..1214da8c 100644 --- a/tests/test_library_disk_usage.py +++ b/tests/test_library_disk_usage.py @@ -69,6 +69,53 @@ def test_file_size_column_exists_after_init(db: MusicDatabase) -> None: assert 'file_size' in cols +def test_legacy_media_schema_repairs_required_refresh_columns(tmp_path: Path) -> None: + """Upgraded installs can have old library tables plus migration markers. + Startup must repair the columns full refresh writes later.""" + db_path = tmp_path / 'legacy_missing_media_columns.db' + conn = sqlite3.connect(str(db_path)) + cur = conn.cursor() + cur.execute("CREATE TABLE metadata (key TEXT PRIMARY KEY, value TEXT)") + cur.execute("INSERT INTO metadata (key, value) VALUES ('id_columns_migrated', 'true')") + cur.execute(""" + CREATE TABLE artists ( + id TEXT PRIMARY KEY, + name TEXT NOT NULL + ) + """) + cur.execute(""" + CREATE TABLE albums ( + id TEXT PRIMARY KEY, + artist_id TEXT NOT NULL, + title TEXT NOT NULL + ) + """) + cur.execute(""" + CREATE TABLE tracks ( + id TEXT PRIMARY KEY, + album_id TEXT NOT NULL, + artist_id TEXT NOT NULL, + title TEXT NOT NULL, + file_path TEXT, + bitrate INTEGER + ) + """) + conn.commit() + conn.close() + + repaired = MusicDatabase(database_path=str(db_path)) + conn = repaired._get_connection() + cur = conn.cursor() + cur.execute("PRAGMA table_info(tracks)") + track_cols = {row[1] for row in cur.fetchall()} + cur.execute("PRAGMA table_info(albums)") + album_cols = {row[1] for row in cur.fetchall()} + conn.close() + + assert 'file_size' in track_cols + assert 'api_track_count' in album_cols + + def test_existing_tracks_have_null_file_size_after_migration(db: MusicDatabase) -> None: """Backward-compat: rows inserted via the OLD schema (no file_size) must still be readable, and querying file_size returns NULL — not diff --git a/tests/test_manual_pick_no_auto_retry.py b/tests/test_manual_pick_no_auto_retry.py index efa88359..9c70f0f8 100644 --- a/tests/test_manual_pick_no_auto_retry.py +++ b/tests/test_manual_pick_no_auto_retry.py @@ -165,3 +165,132 @@ def test_monitor_waits_for_post_processing_before_batch_success(monkeypatch): dm.download_batches.update(previous_batches) +def test_monitor_matches_release_download_by_id_when_filename_changes(monkeypatch): + """Torrent/usenet rows can expose the completed audio filename, not + the original indexer URL/title stored on the task. The monitor must + still claim the completed release by stable download_id. + """ + monkeypatch.setattr(dm, '_make_context_key', lambda u, f: f"{u}::{f}") + monkeypatch.setattr(dm.WebUIDownloadMonitor, '_validate_worker_counts', lambda self: None) + + submitted = [] + + class FakeExecutor: + def submit(self, func, task_id, batch_id): + submitted.append((func, task_id, batch_id)) + + def fake_post_processing_worker(task_id, batch_id): + return None + + monkeypatch.setattr(dm, 'missing_download_executor', FakeExecutor()) + monkeypatch.setattr(dm, '_run_post_processing_worker', fake_post_processing_worker) + monkeypatch.setattr(dm, '_on_download_completed', lambda *args: None) + + with dm.tasks_lock: + previous_tasks = dict(dm.download_tasks) + previous_batches = dict(dm.download_batches) + dm.download_tasks.clear() + dm.download_batches.clear() + dm.download_tasks['task-1'] = { + 'track_info': {'name': 'Ran To Atlanta'}, + 'username': 'torrent', + 'filename': 'http://prowlarr/download?id=123||Drake - ICEMAN', + 'status': 'downloading', + 'download_id': 'torrent-1', + 'status_change_time': time.time(), + } + dm.download_batches['batch-1'] = {'queue': ['task-1']} + + try: + monitor = dm.WebUIDownloadMonitor() + monitor.monitoring = True + monitor.monitored_batches.add('batch-1') + monkeypatch.setattr( + monitor, + '_get_live_transfers', + lambda: { + 'download_id::torrent-1': { + 'id': 'torrent-1', + 'username': 'torrent', + 'filename': '01. Drake - Make Them Cry.flac', + 'state': 'Completed, Succeeded', + 'size': 100, + 'bytesTransferred': 100, + } + }, + ) + + monitor._check_all_downloads() + + assert submitted == [(fake_post_processing_worker, 'task-1', 'batch-1')] + assert dm.download_tasks['task-1']['status'] == 'post_processing' + finally: + with dm.tasks_lock: + dm.download_tasks.clear() + dm.download_tasks.update(previous_tasks) + dm.download_batches.clear() + dm.download_batches.update(previous_batches) + + +def test_monitor_recovers_premature_failed_release_download(monkeypatch): + monkeypatch.setattr(dm, '_make_context_key', lambda u, f: f"{u}::{f}") + monkeypatch.setattr(dm.WebUIDownloadMonitor, '_validate_worker_counts', lambda self: None) + + submitted = [] + + class FakeExecutor: + def submit(self, func, task_id, batch_id): + submitted.append((func, task_id, batch_id)) + + def fake_post_processing_worker(task_id, batch_id): + return None + + monkeypatch.setattr(dm, 'missing_download_executor', FakeExecutor()) + monkeypatch.setattr(dm, '_run_post_processing_worker', fake_post_processing_worker) + monkeypatch.setattr(dm, '_on_download_completed', lambda *args: None) + + with dm.tasks_lock: + previous_tasks = dict(dm.download_tasks) + previous_batches = dict(dm.download_batches) + dm.download_tasks.clear() + dm.download_batches.clear() + dm.download_tasks['task-1'] = { + 'track_info': {'name': 'DAISIES'}, + 'username': 'torrent', + 'filename': 'http://prowlarr/download?id=123||Justin Bieber - Swag', + 'status': 'failed', + 'download_id': 'torrent-1', + 'status_change_time': time.time(), + } + dm.download_batches['batch-1'] = {'queue': ['task-1']} + + try: + monitor = dm.WebUIDownloadMonitor() + monitor.monitoring = True + monitor.monitored_batches.add('batch-1') + monkeypatch.setattr( + monitor, + '_get_live_transfers', + lambda: { + 'download_id::torrent-1': { + 'id': 'torrent-1', + 'username': 'torrent', + 'filename': '02. Justin Bieber - DAISIES.flac', + 'state': 'Completed, Succeeded', + 'size': 100, + 'bytesTransferred': 100, + } + }, + ) + + monitor._check_all_downloads() + + assert submitted == [(fake_post_processing_worker, 'task-1', 'batch-1')] + assert dm.download_tasks['task-1']['status'] == 'post_processing' + finally: + with dm.tasks_lock: + dm.download_tasks.clear() + dm.download_tasks.update(previous_tasks) + dm.download_batches.clear() + dm.download_batches.update(previous_batches) + diff --git a/tests/test_repair_worker_album_fill.py b/tests/test_repair_worker_album_fill.py index 57a44bcd..ee1bf28e 100644 --- a/tests/test_repair_worker_album_fill.py +++ b/tests/test_repair_worker_album_fill.py @@ -44,6 +44,167 @@ if "config.settings" not in sys.modules: from core.repair_worker import RepairWorker +def test_incomplete_album_auto_fill_skips_source_artist_mismatch(tmp_path): + """Album Completeness must never fill a target album with another artist.""" + existing_path = tmp_path / "Gut" / "Light Years" / "01 - Wound Fuck.flac" + existing_path.parent.mkdir(parents=True) + existing_path.write_bytes(b"existing") + + candidate_path = tmp_path / "Jamiroquai" / "Light Years.flac" + candidate_path.parent.mkdir(parents=True) + candidate_path.write_bytes(b"candidate") + + class _FakeDB: + def __init__(self): + self.search_calls = [] + self.wishlist = [] + + def get_tracks_by_album(self, album_id): + if album_id == "target-album": + return [ + SimpleNamespace( + id="target-existing-1", + album_id="target-album", + artist_id="gut", + title="Wound Fuck", + track_number=1, + file_path=str(existing_path), + bitrate=9999, + ), + ] + return [] + + def search_tracks(self, title="", artist="", limit=50, server_source=None): + self.search_calls.append((title, artist, limit, server_source)) + return [ + SimpleNamespace( + id="candidate-1", + album_id="single-album", + artist_id="jamiroquai", + artist_name="Jamiroquai", + title="Light Years", + track_number=1, + file_path=str(candidate_path), + bitrate=9999, + ), + ] + + def add_to_wishlist(self, *args, **kwargs): + self.wishlist.append((args, kwargs)) + + worker = RepairWorker.__new__(RepairWorker) + worker.db = _FakeDB() + worker.transfer_folder = str(tmp_path) + worker._config_manager = None + + result = worker._fix_incomplete_album( + "album", + "target-album", + None, + { + "album_id": "target-album", + "album_title": "Light Years", + "artist": "Gut", + "missing_tracks": [ + { + "name": "Light Years", + "track_number": 2, + "disc_number": 1, + "source": "spotify", + "source_track_id": "sp-light-years", + "artists": ["Jamiroquai"], + }, + ], + }, + ) + + assert result["success"] is False + assert result["fixed"] == 0 + assert result["skipped"] == 1 + assert result["details"][0]["reason"] == "source artist does not match target album artist" + assert worker.db.search_calls == [] + assert worker.db.wishlist == [] + + +def test_incomplete_album_auto_fill_rejects_wrong_artist_candidate(tmp_path): + """Exact title is not enough when the candidate artist differs.""" + existing_path = tmp_path / "album" / "01 - Existing.flac" + existing_path.parent.mkdir(parents=True) + existing_path.write_bytes(b"existing") + candidate_path = tmp_path / "wrong" / "02 - Light Years.flac" + candidate_path.parent.mkdir(parents=True) + candidate_path.write_bytes(b"wrong") + + class _FakeDB: + def __init__(self): + self.wishlist = [] + + def get_tracks_by_album(self, album_id): + if album_id == "target-album": + return [ + SimpleNamespace( + id="target-existing-1", + album_id="target-album", + artist_id="jamiroquai", + title="Existing", + track_number=1, + file_path=str(existing_path), + bitrate=9999, + ), + ] + return [] + + def search_tracks(self, title="", artist="", limit=50, server_source=None): + return [ + SimpleNamespace( + id="candidate-1", + album_id="wrong-album", + artist_id="gut", + artist_name="Gut", + title="Light Years", + track_number=1, + file_path=str(candidate_path), + bitrate=9999, + ), + ] + + def add_to_wishlist(self, *args, **kwargs): + self.wishlist.append((args, kwargs)) + + worker = RepairWorker.__new__(RepairWorker) + worker.db = _FakeDB() + worker.transfer_folder = str(tmp_path) + worker._config_manager = None + + result = worker._fix_incomplete_album( + "album", + "target-album", + None, + { + "album_id": "target-album", + "album_title": "Light Years", + "artist": "Jamiroquai", + "spotify_album_id": "sp-album", + "expected_tracks": 2, + "missing_tracks": [ + { + "name": "Light Years", + "track_number": 2, + "disc_number": 1, + "source": "spotify", + "source_track_id": "sp-light-years", + "artists": ["Jamiroquai"], + }, + ], + }, + ) + + assert result["success"] is True + assert result["fixed"] == 0 + assert result["wishlisted"] == 1 + assert worker.db.wishlist + + def test_perform_album_fill_copy_branch_generates_track_id(tmp_path, monkeypatch): src_path = tmp_path / "source.flac" src_path.write_bytes(b"fake-audio") diff --git a/tests/test_staging_album_provenance.py b/tests/test_staging_album_provenance.py new file mode 100644 index 00000000..fc62a411 --- /dev/null +++ b/tests/test_staging_album_provenance.py @@ -0,0 +1,198 @@ +"""Tests for the album-bundle provenance override in +``core/downloads/staging.py``. + +Verifies that when ``StagingDeps.get_batch_field`` returns a source +override (i.e. the batch was populated by the torrent / usenet +album-bundle flow), the staging matcher records that source on the +task instead of the generic 'staging' username. Provenance recording +downstream uses ``task['username']`` to set ``source_service`` on +the persisted download row — so this is the single point that +controls whether the history modal shows 'Torrent' / 'Usenet' vs +'Staging' / 'Soulseek'. + +Mocks the rest of StagingDeps so the test doesn't touch the +filesystem, AcoustID, or post-processing. +""" + +from __future__ import annotations + +from types import SimpleNamespace +from unittest.mock import MagicMock + +from core.downloads.staging import StagingDeps, try_staging_match +from core.runtime_state import download_tasks, tasks_lock + + +def _make_deps(staging_file, transfer_dir, batch_field_value=None): + """Build a StagingDeps with mocked collaborators. ``batch_field_value`` + is what get_batch_field returns for ``album_bundle_source`` — None + means no override (generic staging match), 'torrent' / 'usenet' + means the album-bundle flow seeded the staging folder.""" + me = MagicMock() + me.normalize_string.side_effect = lambda s: (s or '').lower().strip() + config = MagicMock() + config.get.return_value = transfer_dir + return StagingDeps( + config_manager=config, + matching_engine=me, + get_staging_file_cache=lambda _b: [staging_file], + docker_resolve_path=lambda p: p, + post_process_matched_download_with_verification=lambda *a, **kw: None, + get_batch_field=(lambda _b, _f: batch_field_value) if batch_field_value is not None else (lambda _b, _f: None), + ) + + +def _seed_task(task_id: str, track_name: str, track_artist: str) -> None: + """Register a task in the runtime_state dict so try_staging_match + has something to mark complete.""" + with tasks_lock: + download_tasks[task_id] = { + 'status': 'searching', + 'track_info': { + 'name': track_name, + 'artists': [{'name': track_artist}], + '_is_explicit_album_download': False, + }, + } + + +def _cleanup_task(task_id: str) -> None: + with tasks_lock: + download_tasks.pop(task_id, None) + + +def test_staging_match_uses_torrent_override_when_present(tmp_path) -> None: + src = tmp_path / 'staging' / 'gnx_track_01.flac' + src.parent.mkdir() + src.write_bytes(b'fLaC') + transfer = tmp_path / 'transfer' + deps = _make_deps( + staging_file={'full_path': str(src), 'title': 'Luther', 'artist': 'Kendrick Lamar'}, + transfer_dir=str(transfer), + batch_field_value='torrent', + ) + track = SimpleNamespace(name='Luther', artists=['Kendrick Lamar']) + task_id = 'test_task_torrent_override' + _seed_task(task_id, 'Luther', 'Kendrick Lamar') + try: + ok = try_staging_match(task_id, 'batch_x', track, deps) + assert ok is True + with tasks_lock: + row = download_tasks[task_id] + assert row['username'] == 'torrent', \ + f"Expected provenance override 'torrent', got {row['username']!r}" + assert row['staging_match'] is True + finally: + _cleanup_task(task_id) + + +def test_staging_match_uses_usenet_override_when_present(tmp_path) -> None: + src = tmp_path / 'staging' / 'gnx_track_01.flac' + src.parent.mkdir() + src.write_bytes(b'fLaC') + transfer = tmp_path / 'transfer' + deps = _make_deps( + staging_file={'full_path': str(src), 'title': 'Luther', 'artist': 'Kendrick Lamar'}, + transfer_dir=str(transfer), + batch_field_value='usenet', + ) + track = SimpleNamespace(name='Luther', artists=['Kendrick Lamar']) + task_id = 'test_task_usenet_override' + _seed_task(task_id, 'Luther', 'Kendrick Lamar') + try: + try_staging_match(task_id, 'batch_x', track, deps) + with tasks_lock: + assert download_tasks[task_id]['username'] == 'usenet' + finally: + _cleanup_task(task_id) + + +def test_staging_match_falls_back_to_staging_without_override(tmp_path) -> None: + """When no batch override is present (manual file drop, or + batch has no album_bundle_source field), the staging matcher + uses the historical 'staging' username.""" + src = tmp_path / 'staging' / 'gnx_track_01.flac' + src.parent.mkdir() + src.write_bytes(b'fLaC') + transfer = tmp_path / 'transfer' + deps = _make_deps( + staging_file={'full_path': str(src), 'title': 'Luther', 'artist': 'Kendrick Lamar'}, + transfer_dir=str(transfer), + batch_field_value=None, + ) + track = SimpleNamespace(name='Luther', artists=['Kendrick Lamar']) + task_id = 'test_task_no_override' + _seed_task(task_id, 'Luther', 'Kendrick Lamar') + try: + try_staging_match(task_id, 'batch_x', track, deps) + with tasks_lock: + assert download_tasks[task_id]['username'] == 'staging' + finally: + _cleanup_task(task_id) + + +def test_staging_match_handles_missing_batch_field_callable(tmp_path) -> None: + """Backward compat: callers that build StagingDeps without + supplying ``get_batch_field`` (it defaults to None) still + work — staging matcher falls back to the 'staging' username.""" + src = tmp_path / 'staging' / 'gnx_track_01.flac' + src.parent.mkdir() + src.write_bytes(b'fLaC') + transfer = tmp_path / 'transfer' + + me = MagicMock() + me.normalize_string.side_effect = lambda s: (s or '').lower().strip() + config = MagicMock() + config.get.return_value = str(transfer) + deps = StagingDeps( + config_manager=config, + matching_engine=me, + get_staging_file_cache=lambda _b: [{'full_path': str(src), 'title': 'Luther', 'artist': 'Kendrick Lamar'}], + docker_resolve_path=lambda p: p, + post_process_matched_download_with_verification=lambda *a, **kw: None, + # get_batch_field omitted — defaults to None + ) + track = SimpleNamespace(name='Luther', artists=['Kendrick Lamar']) + task_id = 'test_task_no_accessor' + _seed_task(task_id, 'Luther', 'Kendrick Lamar') + try: + try_staging_match(task_id, 'batch_x', track, deps) + with tasks_lock: + assert download_tasks[task_id]['username'] == 'staging' + finally: + _cleanup_task(task_id) + + +def test_staging_match_swallows_accessor_exception(tmp_path) -> None: + """If the injected accessor raises (e.g. the batch was deleted + mid-process), the staging matcher should fall back to 'staging' + rather than failing the whole match.""" + src = tmp_path / 'staging' / 'gnx_track_01.flac' + src.parent.mkdir() + src.write_bytes(b'fLaC') + transfer = tmp_path / 'transfer' + + def _boom(_b, _f): + raise RuntimeError("batch went away") + + me = MagicMock() + me.normalize_string.side_effect = lambda s: (s or '').lower().strip() + config = MagicMock() + config.get.return_value = str(transfer) + deps = StagingDeps( + config_manager=config, + matching_engine=me, + get_staging_file_cache=lambda _b: [{'full_path': str(src), 'title': 'Luther', 'artist': 'Kendrick Lamar'}], + docker_resolve_path=lambda p: p, + post_process_matched_download_with_verification=lambda *a, **kw: None, + get_batch_field=_boom, + ) + track = SimpleNamespace(name='Luther', artists=['Kendrick Lamar']) + task_id = 'test_task_accessor_raises' + _seed_task(task_id, 'Luther', 'Kendrick Lamar') + try: + try_staging_match(task_id, 'batch_x', track, deps) + with tasks_lock: + assert download_tasks[task_id]['username'] == 'staging' + finally: + _cleanup_task(task_id) diff --git a/tests/test_torrent_usenet_plugins.py b/tests/test_torrent_usenet_plugins.py new file mode 100644 index 00000000..57f11079 --- /dev/null +++ b/tests/test_torrent_usenet_plugins.py @@ -0,0 +1,479 @@ +"""Tests for ``core/download_plugins/torrent.py`` and ``usenet.py``. + +Both plugins compose a Prowlarr client + an adapter + the archive +pipeline. The tests mock the Prowlarr client and the active adapter +factory so we can pin the projection logic, filename encoding / +decoding, finalize path, and the cancel / clear lifecycle without +touching the network or filesystem (beyond ``tmp_path``). +""" + +from __future__ import annotations + +import asyncio +from pathlib import Path +from typing import Optional +from unittest.mock import AsyncMock, MagicMock, patch + +import pytest + +from core.download_plugins.torrent import ( + TorrentDownloadPlugin, + _adapter_state_to_display, + _decode_filename, + _FILENAME_SEP, + _guess_quality_from_title, + _parse_release_title, +) +from core.download_plugins.usenet import UsenetDownloadPlugin +from core.prowlarr_client import ProwlarrSearchResult +from core.torrent_clients.base import TorrentStatus +from core.usenet_clients.base import UsenetStatus + + +def _run(coro): + return asyncio.new_event_loop().run_until_complete(coro) + + +# --------------------------------------------------------------------------- +# Pure helpers +# --------------------------------------------------------------------------- + + +def test_decode_filename_splits_on_separator() -> None: + url, display = _decode_filename(f"https://x/y.torrent{_FILENAME_SEP}Album Name") + assert url == 'https://x/y.torrent' + assert display == 'Album Name' + + +def test_decode_filename_without_separator_returns_none_url() -> None: + url, display = _decode_filename('just a name') + assert url is None + assert display == 'just a name' + + +def test_decode_filename_handles_magnet_with_embedded_separators() -> None: + """Magnet URIs contain ``=`` and ``&`` but no ``||`` — so a + magnet must round-trip cleanly through the encoder.""" + magnet = 'magnet:?xt=urn:btih:abc123&dn=Album+Name' + encoded = f"{magnet}{_FILENAME_SEP}Display" + url, display = _decode_filename(encoded) + assert url == magnet + assert display == 'Display' + + +def test_guess_quality_from_title() -> None: + assert _guess_quality_from_title('Album [FLAC]') == 'flac' + assert _guess_quality_from_title('Album 24-bit Hi-Res') == 'flac' + assert _guess_quality_from_title('Album [MP3 320]') == 'mp3' + assert _guess_quality_from_title('Album [AAC 256]') == 'aac' + assert _guess_quality_from_title('Album [OGG]') == 'ogg' + # Default fallback so quality_score doesn't crash on bare titles. + assert _guess_quality_from_title('Just A Title') == 'mp3' + assert _guess_quality_from_title('') == 'mp3' + + +def test_parse_release_title_splits_artist_dash_title() -> None: + """Most release titles follow 'Artist - Title' / 'Artist - Album'.""" + assert _parse_release_title('Danny Brown - Atrocity Exhibition') == ('Danny Brown', 'Atrocity Exhibition') + assert _parse_release_title('Kendrick Lamar - DAMN.') == ('Kendrick Lamar', 'DAMN.') + + +def test_parse_release_title_strips_trailing_tags() -> None: + """Quality / year tags at the end shouldn't pollute the title.""" + artist, title = _parse_release_title('Danny Brown - Atrocity Exhibition [FLAC]') + assert artist == 'Danny Brown' + assert title == 'Atrocity Exhibition' + artist, title = _parse_release_title('Danny Brown - Atrocity Exhibition (2016)') + assert artist == 'Danny Brown' + assert title == 'Atrocity Exhibition' + + +def test_parse_release_title_handles_no_dash() -> None: + """Some indexers post bare titles. Caller should fall back to + the indexer name as the 'artist' field.""" + artist, title = _parse_release_title('JustATitle') + assert artist == '' + assert title == 'JustATitle' + + +def test_parse_release_title_handles_dashes_in_title() -> None: + """Track titles can themselves contain dashes — only split on + the FIRST one so subtitles survive.""" + artist, title = _parse_release_title('Artist - Title - Live Version') + assert artist == 'Artist' + assert title == 'Title - Live Version' + + +def test_parse_release_title_rejects_url_prefix() -> None: + """Defensive: if a URL somehow lands in the title field, refuse + to call it an artist.""" + artist, title = _parse_release_title('https://example.com/x - Album') + assert artist == '' + + +def test_adapter_state_mapping_covers_complete_states() -> None: + assert _adapter_state_to_display('downloading') == 'InProgress, Downloading' + assert _adapter_state_to_display('seeding') == 'Completed, Succeeded' + assert _adapter_state_to_display('completed') == 'Completed, Succeeded' + assert _adapter_state_to_display('error') == 'Completed, Errored' + assert _adapter_state_to_display('stalled') == 'InProgress, Stalled' + # Unknown state falls through with title-casing rather than crashing. + assert _adapter_state_to_display('weird') == 'Weird' + + +# --------------------------------------------------------------------------- +# Torrent plugin — search projection +# --------------------------------------------------------------------------- + + +def _make_torrent_result(**overrides) -> ProwlarrSearchResult: + base = dict( + guid='guid-1', title='Danny Brown - Atrocity Exhibition [FLAC]', indexer_id=3, + indexer_name='Indexer', protocol='torrent', + download_url='https://x/y.torrent', magnet_uri=None, + info_url=None, size=500_000_000, seeders=12, leechers=3, + grabs=100, publish_date='2026-01-01', categories=[3040], + raw={}, + ) + base.update(overrides) + return ProwlarrSearchResult(**base) + + +def test_torrent_project_results_drops_non_torrent_protocol() -> None: + plugin = TorrentDownloadPlugin() + results = [ + _make_torrent_result(), + _make_torrent_result(protocol='usenet', title='Usenet Album'), + ] + tracks, albums = plugin._project_results(results) + assert len(tracks) == 1 + assert tracks[0].title == 'Atrocity Exhibition' + assert tracks[0].artist == 'Danny Brown' + assert len(albums) == 1 + + +def test_torrent_project_results_drops_releases_without_download_url() -> None: + plugin = TorrentDownloadPlugin() + results = [_make_torrent_result(download_url=None, magnet_uri=None)] + tracks, albums = plugin._project_results(results) + assert tracks == [] + assert albums == [] + + +def test_torrent_project_results_prefers_magnet_when_available() -> None: + plugin = TorrentDownloadPlugin() + magnet = 'magnet:?xt=urn:btih:abc' + results = [_make_torrent_result(magnet_uri=magnet, download_url='https://x/y.torrent')] + tracks, _ = plugin._project_results(results) + url, _ = _decode_filename(tracks[0].filename) + assert url == magnet + + +def test_torrent_project_results_encodes_url_and_title_in_filename() -> None: + plugin = TorrentDownloadPlugin() + tracks, _ = plugin._project_results([_make_torrent_result()]) + url, display = _decode_filename(tracks[0].filename) + assert url == 'https://x/y.torrent' + assert display == 'Danny Brown - Atrocity Exhibition [FLAC]' + + +def test_torrent_project_falls_back_to_indexer_name_when_title_lacks_dash() -> None: + """When the title has no 'Artist -' prefix we'd auto-parse the + filename (which starts with the indexer download URL) and end + up showing the URL in the UI's 'by' field. Pre-filling artist + with the indexer name avoids that.""" + plugin = TorrentDownloadPlugin() + tracks, _ = plugin._project_results([_make_torrent_result(title='JustATitle')]) + assert tracks[0].artist == 'Indexer' + # And the URL is definitely not the artist. + assert 'http' not in tracks[0].artist + assert '||' not in tracks[0].artist + + +def test_torrent_project_results_neutralizes_soulseek_specific_fields() -> None: + """TrackResult.quality_score punishes results with no upload + slots; torrent results don't have that concept so the + projection has to fill in non-punishing neutral values.""" + plugin = TorrentDownloadPlugin() + tracks, _ = plugin._project_results([_make_torrent_result(seeders=0)]) + # seeders=0 means we should still hand the picker something + # usable. free_upload_slots floors at 1 to avoid the 0-slot + # penalty applied to dead Soulseek peers. + assert tracks[0].free_upload_slots >= 1 + + +# --------------------------------------------------------------------------- +# Torrent plugin — is_configured / check_connection +# --------------------------------------------------------------------------- + + +def test_torrent_is_configured_requires_both_sides() -> None: + plugin = TorrentDownloadPlugin() + with patch.object(plugin._prowlarr, 'is_configured', return_value=False), \ + patch('core.download_plugins.torrent.get_active_torrent_adapter', return_value=None): + assert plugin.is_configured() is False + fake_adapter = MagicMock() + fake_adapter.is_configured.return_value = False + with patch.object(plugin._prowlarr, 'is_configured', return_value=True), \ + patch('core.download_plugins.torrent.get_active_torrent_adapter', return_value=fake_adapter): + assert plugin.is_configured() is False + fake_adapter.is_configured.return_value = True + with patch.object(plugin._prowlarr, 'is_configured', return_value=True), \ + patch('core.download_plugins.torrent.get_active_torrent_adapter', return_value=fake_adapter): + assert plugin.is_configured() is True + + +# --------------------------------------------------------------------------- +# Torrent plugin — finalize / cancel / clear +# --------------------------------------------------------------------------- + + +def test_torrent_finalize_picks_first_audio_file(tmp_path: Path) -> None: + plugin = TorrentDownloadPlugin() + # Seed an in-flight download row + plugin.active_downloads['dl-1'] = { + 'id': 'dl-1', 'filename': 'x', 'username': 'torrent', + 'display_name': 'X', 'state': 'InProgress, Downloading', + 'progress': 50.0, 'size': 0, 'transferred': 0, 'speed': 0, + 'file_path': None, 'torrent_hash': 'h1', 'error': None, + } + # Drop two audio files in the save dir + (tmp_path / 'b.flac').write_bytes(b'fLaC') + (tmp_path / 'a.mp3').write_bytes(b'ID3') + plugin._finalize_download('dl-1', str(tmp_path)) + row = plugin.active_downloads['dl-1'] + assert row['state'] == 'Completed, Succeeded' + assert row['progress'] == 100.0 + # Walker sorts → 'a.mp3' wins as first. + assert row['file_path'].endswith('a.mp3') + + +def test_torrent_finalize_marks_error_when_no_audio(tmp_path: Path) -> None: + plugin = TorrentDownloadPlugin() + plugin.active_downloads['dl-1'] = { + 'id': 'dl-1', 'filename': 'x', 'username': 'torrent', + 'display_name': 'X', 'state': 'InProgress, Downloading', + 'progress': 50.0, 'size': 0, 'transferred': 0, 'speed': 0, + 'file_path': None, 'torrent_hash': 'h1', 'error': None, + } + # tmp_path has no audio files + plugin._finalize_download('dl-1', str(tmp_path)) + assert plugin.active_downloads['dl-1']['state'] == 'Completed, Errored' + assert 'No audio files' in plugin.active_downloads['dl-1']['error'] + + +def test_torrent_finalize_marks_error_when_save_path_missing() -> None: + plugin = TorrentDownloadPlugin() + plugin.active_downloads['dl-1'] = { + 'id': 'dl-1', 'filename': 'x', 'username': 'torrent', + 'display_name': 'X', 'state': 'InProgress, Downloading', + 'progress': 50.0, 'size': 0, 'transferred': 0, 'speed': 0, + 'file_path': None, 'torrent_hash': 'h1', 'error': None, + } + plugin._finalize_download('dl-1', None) + assert plugin.active_downloads['dl-1']['state'] == 'Completed, Errored' + assert 'no save_path' in plugin.active_downloads['dl-1']['error'].lower() + + +def test_torrent_clear_completed_drops_only_done_rows() -> None: + plugin = TorrentDownloadPlugin() + plugin.active_downloads['a'] = {'id': 'a', 'state': 'InProgress, Downloading'} + plugin.active_downloads['b'] = {'id': 'b', 'state': 'Completed, Succeeded'} + plugin.active_downloads['c'] = {'id': 'c', 'state': 'Completed, Errored'} + plugin.active_downloads['d'] = {'id': 'd', 'state': 'Cancelled'} + _run(plugin.clear_all_completed_downloads()) + assert list(plugin.active_downloads.keys()) == ['a'] + + +def test_torrent_get_all_returns_status_objects() -> None: + plugin = TorrentDownloadPlugin() + plugin.active_downloads['a'] = { + 'id': 'a', 'filename': 'f', 'username': 'torrent', + 'state': 'InProgress, Downloading', 'progress': 50.0, + 'size': 100, 'transferred': 50, 'speed': 1000, + 'file_path': None, + } + statuses = _run(plugin.get_all_downloads()) + assert len(statuses) == 1 + assert statuses[0].id == 'a' + assert statuses[0].progress == 50.0 + + +# --------------------------------------------------------------------------- +# Usenet plugin — projection +# --------------------------------------------------------------------------- + + +def _make_usenet_result(**overrides) -> ProwlarrSearchResult: + base = dict( + guid='guid-u', title='Some Artist - Some Album', indexer_id=5, + indexer_name='UsenetIndexer', protocol='usenet', + download_url='https://x/y.nzb', magnet_uri=None, + info_url=None, size=400_000_000, seeders=None, leechers=None, + grabs=42, publish_date='2026-01-01', categories=[3010], + raw={}, + ) + base.update(overrides) + return ProwlarrSearchResult(**base) + + +def test_usenet_project_drops_torrent_protocol() -> None: + plugin = UsenetDownloadPlugin() + results = [_make_usenet_result(), _make_usenet_result(protocol='torrent', title='T')] + tracks, albums = plugin._project_results(results) + assert len(tracks) == 1 + assert tracks[0].username == 'usenet' + + +def test_usenet_project_drops_results_without_download_url() -> None: + """Usenet plugins reject magnet-only results entirely — NZBs + don't have a magnet equivalent.""" + plugin = UsenetDownloadPlugin() + results = [_make_usenet_result(download_url=None)] + tracks, _ = plugin._project_results(results) + assert tracks == [] + + +def test_usenet_project_encodes_url_in_filename() -> None: + plugin = UsenetDownloadPlugin() + tracks, _ = plugin._project_results([_make_usenet_result()]) + url, display = _decode_filename(tracks[0].filename) + assert url == 'https://x/y.nzb' + assert display == 'Some Artist - Some Album' + # Artist + title should be parsed out, not auto-extracted from filename. + assert tracks[0].artist == 'Some Artist' + assert tracks[0].title == 'Some Album' + + +def test_usenet_finalize_picks_first_audio_file(tmp_path: Path) -> None: + """Same finalize contract as torrent — sanity check the shared + helper path works for usenet too.""" + plugin = UsenetDownloadPlugin() + plugin.active_downloads['u-1'] = { + 'id': 'u-1', 'filename': 'x', 'username': 'usenet', + 'display_name': 'X', 'state': 'InProgress, Downloading', + 'progress': 50.0, 'size': 0, 'transferred': 0, 'speed': 0, + 'file_path': None, 'job_id': 'j1', 'error': None, + } + (tmp_path / 'track1.flac').write_bytes(b'fLaC') + plugin._finalize_download('u-1', str(tmp_path)) + assert plugin.active_downloads['u-1']['state'] == 'Completed, Succeeded' + assert plugin.active_downloads['u-1']['file_path'].endswith('track1.flac') + + +def test_usenet_is_configured_requires_both_sides() -> None: + plugin = UsenetDownloadPlugin() + fake_adapter = MagicMock() + fake_adapter.is_configured.return_value = True + with patch.object(plugin._prowlarr, 'is_configured', return_value=False), \ + patch('core.download_plugins.usenet.get_active_usenet_adapter', return_value=fake_adapter): + assert plugin.is_configured() is False + with patch.object(plugin._prowlarr, 'is_configured', return_value=True), \ + patch('core.download_plugins.usenet.get_active_usenet_adapter', return_value=None): + assert plugin.is_configured() is False + with patch.object(plugin._prowlarr, 'is_configured', return_value=True), \ + patch('core.download_plugins.usenet.get_active_usenet_adapter', return_value=fake_adapter): + assert plugin.is_configured() is True + + +# --------------------------------------------------------------------------- +# Plugin conformance — both must satisfy the DownloadSourcePlugin Protocol +# --------------------------------------------------------------------------- + + +def test_plugins_conform_to_protocol() -> None: + from core.download_plugins.base import DownloadSourcePlugin + assert isinstance(TorrentDownloadPlugin(), DownloadSourcePlugin) + assert isinstance(UsenetDownloadPlugin(), DownloadSourcePlugin) + + +# --------------------------------------------------------------------------- +# Registry — both should register cleanly +# --------------------------------------------------------------------------- + + +def test_torrent_album_pick_prefers_seeded_flac(tmp_path: Path) -> None: + """Album bundle picker prefers high-seeded FLAC over low-seeded MP3 + of comparable size — protects against picking a dead torrent.""" + from core.download_plugins.album_bundle import pick_best_album_release + from core.download_plugins.torrent import _guess_quality_from_title + flac = _make_torrent_result(title='Kendrick Lamar - GNX [FLAC]', size=400_000_000, seeders=120) + mp3 = _make_torrent_result(title='Kendrick Lamar - GNX [MP3 320]', size=120_000_000, seeders=5, guid='guid-2') + picked = pick_best_album_release([flac, mp3], _guess_quality_from_title) + assert picked is flac + + +def test_torrent_album_pick_drops_too_small() -> None: + """Single-track torrents (~10 MB) shouldn't be picked when the user + is downloading a whole album — the size floor (40 MB) catches them.""" + from core.download_plugins.album_bundle import pick_best_album_release + from core.download_plugins.torrent import _guess_quality_from_title + single = _make_torrent_result(title='Kendrick Lamar - HUMBLE', size=10_000_000, seeders=500) + album = _make_torrent_result(title='Kendrick Lamar - DAMN [MP3]', size=120_000_000, seeders=50, guid='guid-2') + picked = pick_best_album_release([single, album], _guess_quality_from_title) + assert picked is album + + +def test_torrent_album_pick_falls_back_when_all_outside_size_range() -> None: + """If every candidate is below the floor (e.g. all results are + singles), pick the most-seeded one rather than returning None — + user still wants a download even if it's a track torrent.""" + from core.download_plugins.album_bundle import pick_best_album_release + from core.download_plugins.torrent import _guess_quality_from_title + small_a = _make_torrent_result(title='X [MP3]', size=8_000_000, seeders=5) + small_b = _make_torrent_result(title='Y [MP3]', size=9_000_000, seeders=80, guid='guid-2') + picked = pick_best_album_release([small_a, small_b], _guess_quality_from_title) + assert picked is small_b + + +def test_unique_staging_path_handles_collision(tmp_path: Path) -> None: + from core.download_plugins.album_bundle import unique_staging_path + src = tmp_path / 'src' / 'track.flac' + src.parent.mkdir() + src.write_bytes(b'fLaC') + dest_dir = tmp_path / 'staging' + dest_dir.mkdir() + # First call returns the natural name. + first = unique_staging_path(dest_dir, src) + assert first == dest_dir / 'track.flac' + first.write_bytes(b'fLaC') + # Second call picks a non-colliding suffix. + second = unique_staging_path(dest_dir, src) + assert second == dest_dir / 'track_1.flac' + + +def test_torrent_album_to_staging_short_circuits_when_not_configured() -> None: + """The gate must refuse to operate when Prowlarr isn't set up — + every later call would hit the network with empty creds.""" + plugin = TorrentDownloadPlugin() + with patch.object(plugin, 'is_configured', return_value=False): + outcome = plugin.download_album_to_staging('GNX', 'Kendrick Lamar', '/tmp/staging') + assert outcome['success'] is False + assert 'not configured' in outcome['error'].lower() + + +def test_torrent_album_to_staging_ignores_candidates_without_download_url(tmp_path: Path) -> None: + plugin = TorrentDownloadPlugin() + fake_adapter = MagicMock() + fake_adapter.is_configured.return_value = True + with patch.object(plugin, 'is_configured', return_value=True), \ + patch.object(plugin._prowlarr, 'search', new=AsyncMock(return_value=[ + _make_torrent_result(download_url=None, magnet_uri=None), + ])), \ + patch('core.download_plugins.torrent.get_active_torrent_adapter', return_value=fake_adapter): + outcome = plugin.download_album_to_staging('GNX', 'Kendrick Lamar', str(tmp_path)) + + assert outcome['success'] is False + assert 'No torrent results' in outcome['error'] + fake_adapter.add_torrent.assert_not_called() + + +def test_registry_includes_torrent_and_usenet() -> None: + """The registry decides what shows up in the orchestrator's + iteration helpers. If we forget to register a new plugin the + download source dropdown will silently no-op.""" + from core.download_plugins.registry import build_default_registry + registry = build_default_registry() + names = registry.names() + assert 'torrent' in names + assert 'usenet' in names diff --git a/tests/test_worker_utils_album_track_count.py b/tests/test_worker_utils_album_track_count.py index 4b988d70..cf96b535 100644 --- a/tests/test_worker_utils_album_track_count.py +++ b/tests/test_worker_utils_album_track_count.py @@ -1,6 +1,8 @@ """Tests for `worker_utils.set_album_api_track_count` — the shared helper enrichment workers call to cache authoritative track counts.""" +import sqlite3 + from core.worker_utils import set_album_api_track_count @@ -114,3 +116,21 @@ def test_swallows_cursor_execute_errors(): cursor = _BrokenCursor() # Should not raise. set_album_api_track_count(cursor, "album-z", 10) + + +def test_repairs_missing_api_track_count_column(): + conn = sqlite3.connect(":memory:") + cursor = conn.cursor() + cursor.execute("CREATE TABLE albums (id TEXT PRIMARY KEY, title TEXT)") + cursor.execute("INSERT INTO albums (id, title) VALUES ('album-z', 'Album')") + + set_album_api_track_count(cursor, "album-z", 10) + + cursor.execute("PRAGMA table_info(albums)") + cols = {row[1] for row in cursor.fetchall()} + cursor.execute("SELECT api_track_count FROM albums WHERE id = 'album-z'") + row = cursor.fetchone() + conn.close() + + assert 'api_track_count' in cols + assert row[0] == 10 diff --git a/web_server.py b/web_server.py index ddf3ebe4..3966f93c 100644 --- a/web_server.py +++ b/web_server.py @@ -40,7 +40,7 @@ logger = setup_logging(_log_level, _log_path) # App version — single source of truth for backup metadata, system-info, update check, etc. # Semver: MAJOR.MINOR.PATCH. Bump at each dev→main release. -_SOULSYNC_BASE_VERSION = "2.5.8" +_SOULSYNC_BASE_VERSION = "2.5.9" def _build_version_string(): """Append short commit hash to version when available (e.g. 2.35+abc1234).""" @@ -2115,10 +2115,15 @@ def get_status(): _status_cache_timestamps['media_server'] = current_time # else: use cached value - # Test Soulseek - only if it's the active source or in the hybrid order - if current_time - _status_cache_timestamps['soulseek'] > STATUS_CACHE_TTL: - download_mode = config_manager.get('download_source.mode', 'hybrid') - hybrid_order = config_manager.get('download_source.hybrid_order', ['hifi', 'youtube', 'soulseek']) + download_mode = config_manager.get('download_source.mode', 'hybrid') + hybrid_order = config_manager.get('download_source.hybrid_order', ['hifi', 'youtube', 'soulseek']) + if isinstance(hybrid_order, str): + hybrid_order = [hybrid_order] + source_cache_key = f"{download_mode}:{','.join(str(s) for s in (hybrid_order or []))}" + + # Test Soulseek/download source only when the cached source selection is still current. + if (current_time - _status_cache_timestamps['soulseek'] > STATUS_CACHE_TTL or + _status_cache['soulseek'].get('source_cache_key') != source_cache_key): soulseek_relevant = (download_mode == 'soulseek' or (download_mode == 'hybrid' and 'soulseek' in hybrid_order)) @@ -2130,11 +2135,24 @@ def get_status(): is_serverless = (download_mode in serverless_sources or (download_mode == 'hybrid' and hybrid_order and any(s in serverless_sources for s in hybrid_order))) + external_client_sources = ('torrent', 'usenet') + external_client_relevant = ( + download_mode in external_client_sources or + (download_mode == 'hybrid' and + hybrid_order and any(s in external_client_sources for s in hybrid_order)) + ) # Serverless check first — avoids slow slskd timeout when YouTube/HiFi are in hybrid order if is_serverless: soulseek_status = True soulseek_response_time = 0 + elif external_client_relevant and download_orchestrator: + soulseek_start = time.time() + try: + soulseek_status = run_async(download_orchestrator.check_connection()) + except Exception: + soulseek_status = False + soulseek_response_time = (time.time() - soulseek_start) * 1000 elif soulseek_relevant and download_orchestrator: soulseek_start = time.time() try: @@ -2148,13 +2166,17 @@ def get_status(): _status_cache['soulseek'] = { 'connected': soulseek_status, - 'response_time': round(soulseek_response_time, 1) + 'response_time': round(soulseek_response_time, 1), + 'source_cache_key': source_cache_key, } _status_cache_timestamps['soulseek'] = current_time # Include download source mode so frontend can update labels - download_mode = config_manager.get('download_source.mode', 'hybrid') _status_cache['soulseek']['source'] = download_mode + soulseek_data = { + key: value for key, value in _status_cache['soulseek'].items() + if key != 'source_cache_key' + } # Count active downloads for nav badge active_dl_count = 0 @@ -2167,7 +2189,7 @@ def get_status(): 'metadata_source': metadata_status['metadata_source'], 'spotify': metadata_status['spotify'], 'media_server': _status_cache['media_server'], - 'soulseek': _status_cache['soulseek'], + 'soulseek': soulseek_data, 'active_media_server': active_server, 'enrichment': _get_enrichment_status(), 'active_downloads': active_dl_count, @@ -5887,6 +5909,14 @@ def _find_completed_file_robust(download_dir, api_filename, transfer_dir=None): from difflib import SequenceMatcher from unidecode import unidecode + audio_extensions = { + '.mp3', '.flac', '.m4a', '.aac', '.ogg', '.opus', '.wav', '.wma', '.alac', + '.aiff', '.aif', '.dsf', '.dff', '.ape', + } + + def _is_audio_candidate(path): + return os.path.splitext(str(path or ''))[1].lower() in audio_extensions + # YOUTUBE/TIDAL SUPPORT: Handle encoded filename format "id||title" # Extract just the title part for file matching if '||' in api_filename: @@ -5921,9 +5951,12 @@ def _find_completed_file_robust(download_dir, api_filename, transfer_dir=None): # Skip quarantine folder — contains known-wrong files from AcoustID verification dirs[:] = [d for d in dirs if d != 'ss_quarantine'] for file in files: + file_path = os.path.join(root, file) + if not _is_audio_candidate(file_path): + continue + # Direct basename match if os.path.basename(file) == target_basename: - file_path = os.path.join(root, file) # Fast path: if path aligns with expected directory structure, return now if api_dir_parts and _path_matches_api_dirs(file_path): logger.info(f"Found path-confirmed match in {location_name}: {file_path}") @@ -5940,7 +5973,6 @@ def _find_completed_file_robust(download_dir, api_filename, transfer_dir=None): file_stem, file_ext_part = os.path.splitext(file) stripped_stem = re.sub(r'_\d{10,}$', '', file_stem) if stripped_stem != file_stem and stripped_stem + file_ext_part == target_basename: - file_path = os.path.join(root, file) if api_dir_parts and _path_matches_api_dirs(file_path): logger.info(f"Found path-confirmed dedup match in {location_name}: {file_path}") return file_path, 1.0 @@ -5956,7 +5988,7 @@ def _find_completed_file_robust(download_dir, api_filename, transfer_dir=None): if similarity > highest_fuzzy_similarity: highest_fuzzy_similarity = similarity - best_fuzzy_path = os.path.join(root, file) + best_fuzzy_path = file_path # Return best exact match (disambiguated by path), or fall back to fuzzy if exact_matches: @@ -16463,9 +16495,10 @@ def _get_staging_file_cache(batch_id): """Scan staging folder once per batch and cache the results.""" with _staging_cache_lock: if batch_id in _staging_cache: - return _staging_cache[batch_id] + cached = _staging_cache[batch_id] + return [f for f in cached if os.path.exists(f.get('full_path', ''))] - staging_path = get_staging_path() + staging_path = _get_album_bundle_staging_path(batch_id) or get_staging_path() if not os.path.isdir(staging_path): with _staging_cache_lock: _staging_cache[batch_id] = [] @@ -16496,10 +16529,32 @@ def _get_staging_file_cache(batch_id): return files +def _get_album_bundle_staging_path(batch_id): + """Return the private album-bundle staging path for this batch, if any.""" + if not batch_id: + return None + with tasks_lock: + row = download_batches.get(batch_id) + if isinstance(row, dict) and row.get('album_bundle_private_staging'): + return row.get('album_bundle_staging_path') + return None + + # Staging-folder match shortcut lives in core/downloads/staging.py. from core.downloads import staging as _downloads_staging +def _staging_get_batch_field(batch_id, field): + """Accessor injected into StagingDeps so the staging-match helper + can read an album-bundle provenance override off the batch state + without importing ``download_batches`` directly.""" + with tasks_lock: + row = download_batches.get(batch_id) + if isinstance(row, dict): + return row.get(field) + return None + + def _build_staging_deps(): """Build the StagingDeps bundle from web_server.py globals on each call.""" return _downloads_staging.StagingDeps( @@ -16508,6 +16563,7 @@ def _build_staging_deps(): get_staging_file_cache=_get_staging_file_cache, docker_resolve_path=docker_resolve_path, post_process_matched_download_with_verification=_post_process_matched_download_with_verification, + get_batch_field=_staging_get_batch_field, ) @@ -16647,7 +16703,7 @@ def _store_batch_source(batch_id, username, filename): """Browse the successful download's folder and store results on the batch for reuse.""" _sr = source_reuse_logger _sr.info(f"_store_batch_source called: batch={batch_id}, user={username}, file={filename}") - if not batch_id or username in ('youtube', 'tidal', 'qobuz', 'hifi', 'deezer_dl', 'lidarr', 'soundcloud', 'amazon'): + if not batch_id or username in ('youtube', 'tidal', 'qobuz', 'hifi', 'deezer_dl', 'lidarr', 'soundcloud', 'amazon', 'torrent', 'usenet'): _sr.info(f"Skipped — no batch_id or streaming source ({username})") return @@ -19723,39 +19779,12 @@ def soundcloud_status(): @app.route('/api/hifi/instances', methods=['GET']) def hifi_instances(): """Check availability of all HiFi API instances.""" - import requests as req try: hifi = download_orchestrator.client("hifi") instances = list(hifi._instances) results = [] for url in instances: - entry = {'url': url, 'status': 'unknown', 'version': None, 'can_search': False, 'can_download': False} - try: - # Check root for version - r = req.get(f'{url}/', timeout=5, headers={'Accept': 'application/json'}) - if r.ok: - data = r.json() - entry['version'] = data.get('version') - entry['status'] = 'online' - # Check search - sr = req.get(f'{url}/search', params={'s': 'test', 'limit': 1}, timeout=5) - entry['can_search'] = sr.ok - # Check track (download capability) - tr = req.get(f'{url}/track', params={'id': '1550546', 'quality': 'LOSSLESS'}, timeout=5) - entry['can_download'] = tr.ok - if not tr.ok: - entry['download_error'] = f'HTTP {tr.status_code}' - else: - entry['status'] = f'error (HTTP {r.status_code})' - except req.exceptions.SSLError: - entry['status'] = 'ssl_error' - except req.exceptions.ConnectTimeout: - entry['status'] = 'timeout' - except req.exceptions.ConnectionError: - entry['status'] = 'offline' - except Exception as e: - entry['status'] = f'error ({type(e).__name__})' - results.append(entry) + results.append(hifi.check_instance_capabilities(url)) return jsonify({'instances': results, 'active': hifi._get_instance()}) except Exception as e: return jsonify({'error': str(e)}), 500 diff --git a/webui/index.html b/webui/index.html index f6b0cb1b..41393284 100644 --- a/webui/index.html +++ b/webui/index.html @@ -4328,6 +4328,8 @@ + +
@@ -4735,6 +4737,15 @@
+ + + `; }).join(''); } else { - tracksHtml = '
No tracks loaded
'; + tracksHtml = batch.phase === 'album_downloading' + ? '
Downloading one release first. Track matching starts after staging.
' + : '
No tracks loaded
'; } } diff --git a/webui/static/settings.js b/webui/static/settings.js index abe02287..0d3888c1 100644 --- a/webui/static/settings.js +++ b/webui/static/settings.js @@ -597,10 +597,12 @@ const HYBRID_SOURCES = [ { id: 'amazon', name: 'Amazon Music', icon: null, emoji: '🛒' }, { id: 'lidarr', name: 'Lidarr', icon: null, emoji: '📦' }, { id: 'soundcloud', name: 'SoundCloud', icon: 'https://www.svgrepo.com/show/452219/soundcloud.svg', emoji: '☁️' }, + { id: 'torrent', name: 'Torrent', icon: null, emoji: '🧲' }, + { id: 'usenet', name: 'Usenet', icon: null, emoji: '📰' }, ]; let _hybridSourceOrder = ['soulseek', 'youtube']; -let _hybridSourceEnabled = { soulseek: true, youtube: true, tidal: false, qobuz: false, hifi: false, deezer_dl: false, amazon: false, lidarr: false, soundcloud: false }; +let _hybridSourceEnabled = { soulseek: true, youtube: true, tidal: false, qobuz: false, hifi: false, deezer_dl: false, amazon: false, lidarr: false, soundcloud: false, torrent: false, usenet: false }; let _hybridVisualOrder = null; // Full visual order including disabled sources function buildHybridSourceList() { @@ -1536,6 +1538,11 @@ function updateDownloadSourceUI() { if (amazonContainer) amazonContainer.style.display = activeSources.has('amazon') ? 'block' : 'none'; if (lidarrContainer) lidarrContainer.style.display = activeSources.has('lidarr') ? 'block' : 'none'; if (soundcloudContainer) soundcloudContainer.style.display = activeSources.has('soundcloud') ? 'block' : 'none'; + const prowlarrRedirect = document.getElementById('prowlarr-source-redirect'); + if (prowlarrRedirect) { + const showProwlarr = activeSources.has('torrent') || activeSources.has('usenet'); + prowlarrRedirect.style.display = showProwlarr ? 'block' : 'none'; + } // Quality profile is Soulseek-only and downloads-tab-only const qualityProfileSection = document.getElementById('quality-profile-section'); diff --git a/webui/static/shared-helpers.js b/webui/static/shared-helpers.js index 9840fea3..05ab4fa2 100644 --- a/webui/static/shared-helpers.js +++ b/webui/static/shared-helpers.js @@ -3377,8 +3377,8 @@ function updateServiceStatus(service, statusData, spotifyStatus = null) { // Update download source title on dashboard card if (service === 'soulseek' && statusData.source) { - const sourceNames = { soulseek: 'Soulseek', youtube: 'YouTube', tidal: 'Tidal', qobuz: 'Qobuz', hifi: 'HiFi', deezer_dl: 'Deezer', lidarr: 'Lidarr', soundcloud: 'SoundCloud', hybrid: 'Hybrid' }; - const displayName = sourceNames[statusData.source] || 'Soulseek'; + const sourceNames = { soulseek: 'Soulseek', youtube: 'YouTube', tidal: 'Tidal', qobuz: 'Qobuz', hifi: 'HiFi', deezer_dl: 'Deezer', amazon: 'Amazon', lidarr: 'Lidarr', soundcloud: 'SoundCloud', torrent: 'Torrent', usenet: 'Usenet', hybrid: 'Hybrid' }; + const displayName = sourceNames[statusData.source] || 'Download Source'; const titleEl = document.getElementById('download-source-title'); if (titleEl) titleEl.textContent = displayName; } @@ -3422,8 +3422,8 @@ function updateSidebarServiceStatus(service, statusData, spotifyStatus = null) { // Update download source name based on configured mode if (service === 'soulseek' && statusData.source) { - const sourceNames = { soulseek: 'Soulseek', youtube: 'YouTube', tidal: 'Tidal', qobuz: 'Qobuz', hifi: 'HiFi', deezer_dl: 'Deezer', lidarr: 'Lidarr', soundcloud: 'SoundCloud', hybrid: 'Hybrid' }; - const displayName = sourceNames[statusData.source] || 'Soulseek'; + const sourceNames = { soulseek: 'Soulseek', youtube: 'YouTube', tidal: 'Tidal', qobuz: 'Qobuz', hifi: 'HiFi', deezer_dl: 'Deezer', amazon: 'Amazon', lidarr: 'Lidarr', soundcloud: 'SoundCloud', torrent: 'Torrent', usenet: 'Usenet', hybrid: 'Hybrid' }; + const displayName = sourceNames[statusData.source] || 'Download Source'; const sidebarName = document.getElementById('download-source-name'); if (sidebarName) sidebarName.textContent = displayName; } diff --git a/webui/static/style.css b/webui/static/style.css index 6ba8a24d..d6c531ed 100644 --- a/webui/static/style.css +++ b/webui/static/style.css @@ -3316,6 +3316,45 @@ body.helper-mode-active #dashboard-activity-feed:hover { line-height: 1.55; } +.ind-hero-warning { + margin-top: 14px; + padding: 12px 14px; + background: rgba(255, 165, 0, 0.06); + border: 1px solid rgba(255, 165, 0, 0.2); + border-radius: 10px; +} + +.ind-hero-warning-title { + font-size: 12px; + font-weight: 700; + color: #ffb155; + letter-spacing: 0.4px; + margin-bottom: 6px; +} + +.ind-hero-warning-body { + font-size: 12.5px; + color: rgba(255, 255, 255, 0.72); + line-height: 1.55; +} + +.ind-hero-warning-body ul { + margin: 6px 0 0 18px; + padding: 0; +} + +.ind-hero-warning-body li { + margin: 3px 0; +} + +.ind-hero-warning-body code { + font-family: 'SF Mono', Monaco, monospace; + font-size: 11.5px; + background: rgba(0, 0, 0, 0.3); + padding: 1px 5px; + border-radius: 3px; +} + /* Section-header status dots — Lidarr-style green/red/grey indicator */ .ind-status-dot { width: 9px; @@ -10294,6 +10333,25 @@ body.helper-mode-active #dashboard-activity-feed:hover { border: 1px solid rgba(100, 200, 130, 0.2); } +.library-history-badge.download.source-torrent { + color: #5dade2; + background: rgba(93, 173, 226, 0.1); + border-color: rgba(93, 173, 226, 0.25); +} + +.library-history-badge.download.source-usenet { + color: #a78bfa; + background: rgba(167, 139, 250, 0.1); + border-color: rgba(167, 139, 250, 0.25); +} + +.library-history-badge.download.source-staging, +.library-history-badge.download.source-auto-import { + color: rgba(255, 255, 255, 0.55); + background: rgba(255, 255, 255, 0.05); + border-color: rgba(255, 255, 255, 0.12); +} + .library-history-badge.import { color: rgba(120, 160, 230, 0.9); background: rgba(120, 160, 230, 0.1); @@ -19846,6 +19904,11 @@ body.helper-mode-active #dashboard-activity-feed:hover { inset 0 1px 2px rgba(255, 255, 255, 0.3); } +.album-bundle-waiting { + color: rgba(255, 255, 255, 0.55); + font-style: italic; +} + /* Track Table Section */ .download-tracks-section { /* Premium glassmorphic card matching other sections */ @@ -58173,6 +58236,9 @@ body.reduce-effects *::after { font-size: 0.7rem; color: rgba(255, 255, 255, 0.4); margin-top: 2px; + white-space: nowrap; + overflow: hidden; + text-overflow: ellipsis; } .adl-batch-card-source { @@ -58315,6 +58381,13 @@ body.reduce-effects *::after { .adl-batch-track-status.failed { color: #ef4444; } .adl-batch-track-status.queued { color: rgba(255, 255, 255, 0.3); } +.adl-batch-release-note { + padding: 6px 0; + color: rgba(255, 255, 255, 0.38); + font-size: 0.7rem; + line-height: 1.4; +} + /* Mini progress bar per track */ .adl-batch-track-progress { width: 100%; diff --git a/webui/static/wishlist-tools.js b/webui/static/wishlist-tools.js index 1a50bf19..fd8eee6b 100644 --- a/webui/static/wishlist-tools.js +++ b/webui/static/wishlist-tools.js @@ -3406,7 +3406,7 @@ async function loadLibraryHistory() { const sc = data.stats?.source_counts || {}; const srcEntries = Object.entries(sc).sort((a, b) => b[1] - a[1]); if (srcEntries.length > 0 && tab === 'download') { - const _srcColors = { Soulseek: '#4caf50', Tidal: '#000', YouTube: '#ff0000', Qobuz: '#4285f4', HiFi: '#00bcd4', Deezer: '#a238ff' }; + const _srcColors = { Soulseek: '#4caf50', Tidal: '#000', YouTube: '#ff0000', Qobuz: '#4285f4', HiFi: '#00bcd4', Deezer: '#a238ff', Lidarr: '#5dade2', Amazon: '#ff9900', SoundCloud: '#ff7700', Torrent: '#5dade2', Usenet: '#a78bfa', Staging: '#888', 'Auto-Import': '#888' }; sourceBar.innerHTML = srcEntries.map(([src, cnt]) => `${src}: ${cnt}` ).join(''); @@ -3449,7 +3449,10 @@ function renderHistoryEntry(entry) { const parts = []; if (entry.download_source) parts.push(entry.download_source); if (entry.quality) parts.push(entry.quality); - badge = parts.map(p => `${escapeHtml(p)}`).join(''); + badge = parts.map(p => { + const cls = String(p || '').toLowerCase().replace(/[^a-z0-9]+/g, '-').replace(/^-|-$/g, ''); + return `${escapeHtml(p)}`; + }).join(''); } else if (entry.event_type === 'import' && entry.server_source) { const sourceName = { plex: 'Plex', jellyfin: 'Jellyfin', navidrome: 'Navidrome' }[entry.server_source] || entry.server_source; badge = `${escapeHtml(sourceName)}`;