From 5f126584f9883baa72e8a6696d38f3fa4be38493 Mon Sep 17 00:00:00 2001 From: Broque Thomas <26755000+Nezreka@users.noreply.github.com> Date: Wed, 20 May 2026 17:05:00 -0700 Subject: [PATCH 01/25] feat(downloads): add archive_pipeline module for torrent/usenet downloads MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Shared helper the upcoming torrent and usenet download plugins both compose against. Narrow surface — no matching, no tagging, no library import. Just walks audio files and extracts archives when needed. Why a separate module: usenet downloaders (SABnzbd, NZBGet) already auto-extract by default, and Lidarr's import pipeline extracts before SoulSync sees the files. The only client that sometimes leaves an archive behind is a torrent client when the album was packed as a .rar — most music torrents ship loose but not all. Centralising the walk + extract logic means both new plugins can do the same thing, and a future direct-archive source (zip download from a private site, etc.) plugs in for free. - core/archive_pipeline.py: - AUDIO_EXTENSIONS / ARCHIVE_EXTENSIONS constants (audio set matches core/imports/file_ops.py quality_tiers). - is_archive(path) handles compound extensions (.tar.gz etc). - walk_audio_files(directory) — recursive, case-insensitive. - find_archives_in_dir(directory) — top-level only (don't surprise-extract sample / proof folders inside a torrent). - extract_archive(archive_path, extract_to=None) — handles .zip, .tar variants, .rar (optional rarfile dep), .7z (optional py7zr dep). Optional deps warn-and-skip if absent. - extract_all_in_dir + collect_audio_after_extraction — the one-shot helpers the download plugins call after a download completes. - Path-traversal protection: every archive member's resolved path must stay inside the destination — first violator aborts the extract without writing anything. Applies to zip, tar, and rar. - tests/test_archive_pipeline.py: 21 tests covering the walker (nested dirs, case-insensitive, ignores non-audio), archive detection (compound extensions, missing files), zip extraction + path-traversal rejection, tar.gz + tar path-traversal, multi-archive directory, mixed-loose-and-archived collection. --- core/archive_pipeline.py | 232 ++++++++++++++++++++++++++++++++ tests/test_archive_pipeline.py | 238 +++++++++++++++++++++++++++++++++ webui/static/helper.js | 1 + 3 files changed, 471 insertions(+) create mode 100644 core/archive_pipeline.py create mode 100644 tests/test_archive_pipeline.py diff --git a/core/archive_pipeline.py b/core/archive_pipeline.py new file mode 100644 index 00000000..12248e93 --- /dev/null +++ b/core/archive_pipeline.py @@ -0,0 +1,232 @@ +"""Archive extraction + audio-file discovery for torrent / usenet downloads. + +The torrent and usenet download plugins need a uniform way to: + +1. Walk the downloader's save directory and find every audio file in it. +2. If the directory contains an archive (``.zip`` / ``.rar`` / ``.tar`` / + ``.7z``), extract it first so the audio files inside become walkable. + +This module is intentionally narrow — no matching, no tagging, no +import. The download plugin layer composes this with the existing +post-processing / matching pipeline. Lidarr does NOT use this module: +Lidarr extracts archives in its own import step before SoulSync sees +the files at all. Usenet downloaders (SABnzbd, NZBGet) also auto- +extract by default. Torrents are the main case where SoulSync may +need to do the extract step itself — most music torrents ship loose, +but some bundle the album in a ``.rar`` archive. + +``rarfile`` is an optional dependency. If it isn't installed, archives +with ``.rar`` content are skipped with a single warning rather than +crashing the download. +""" + +from __future__ import annotations + +import tarfile +import zipfile +from pathlib import Path +from typing import List, Optional + +from utils.logging_config import get_logger + +logger = get_logger("archive_pipeline") + + +# Same audio-extension set as ``core/imports/file_ops.py`` ``quality_tiers``. +# Keep them in sync — if a new format is added to file_ops, add it here too +# or the walker will skip it and the download plugin will mark the download +# failed even when files arrived. +AUDIO_EXTENSIONS = frozenset([ + # lossless + '.flac', '.ape', '.wav', '.alac', '.dsf', '.dff', '.aiff', '.aif', + # high lossy + '.opus', '.ogg', + # standard lossy + '.m4a', '.aac', + # low lossy + '.mp3', '.wma', +]) + +ARCHIVE_EXTENSIONS = frozenset(['.zip', '.rar', '.tar', '.tar.gz', '.tgz', '.7z']) + + +def is_archive(path: Path) -> bool: + """True if the file extension looks like a supported archive. + + Compound extensions (``.tar.gz``, ``.tar.bz2``) are detected by + checking the last two suffixes joined together — Path.suffix + only returns the final suffix. + """ + if not path.is_file(): + return False + name = path.name.lower() + if name.endswith(('.tar.gz', '.tar.bz2', '.tar.xz')): + return True + return path.suffix.lower() in ARCHIVE_EXTENSIONS + + +def walk_audio_files(directory: Path) -> List[Path]: + """Recursively scan ``directory`` for audio files. Returns + a sorted list of absolute paths. Empty list if the directory + doesn't exist or contains no audio. + """ + if not directory or not directory.exists() or not directory.is_dir(): + return [] + out: List[Path] = [] + for child in directory.rglob('*'): + if not child.is_file(): + continue + if child.suffix.lower() in AUDIO_EXTENSIONS: + out.append(child.resolve()) + out.sort() + return out + + +def find_archives_in_dir(directory: Path) -> List[Path]: + """Find every archive file directly inside ``directory`` (one + level deep — torrents normally put the archive at the root of + their folder; we don't search nested dirs to avoid extracting + something we shouldn't). + """ + if not directory or not directory.exists() or not directory.is_dir(): + return [] + return sorted(p for p in directory.iterdir() if is_archive(p)) + + +def extract_archive(archive_path: Path, extract_to: Optional[Path] = None) -> Optional[Path]: + """Extract a single archive in-place (or to ``extract_to`` if + given). Returns the directory the archive was extracted into, + or ``None`` on failure. + + Supports ``.zip``, ``.tar``/``.tar.gz``/``.tar.bz2``/``.tar.xz``, + and ``.rar`` (only when the optional ``rarfile`` library is + installed). ``.7z`` is recognised but extraction requires + ``py7zr``; without it, the call logs and returns None. + """ + if not archive_path or not archive_path.exists(): + logger.warning("archive_pipeline: %s does not exist", archive_path) + return None + dest = extract_to or archive_path.parent + dest.mkdir(parents=True, exist_ok=True) + + name = archive_path.name.lower() + try: + if name.endswith('.zip'): + with zipfile.ZipFile(archive_path) as zf: + _safe_extract_zip(zf, dest) + return dest + if name.endswith(('.tar', '.tar.gz', '.tgz', '.tar.bz2', '.tar.xz')): + with tarfile.open(archive_path) as tf: + _safe_extract_tar(tf, dest) + return dest + if name.endswith('.rar'): + return _extract_rar(archive_path, dest) + if name.endswith('.7z'): + return _extract_7z(archive_path, dest) + except (zipfile.BadZipFile, tarfile.TarError, OSError) as e: + logger.error("archive_pipeline: failed to extract %s: %s", archive_path, e) + return None + logger.warning("archive_pipeline: unknown archive type for %s", archive_path) + return None + + +def extract_all_in_dir(directory: Path) -> List[Path]: + """Find every archive in ``directory`` and extract each in place. + Returns the list of directories archives were extracted into + (usually all the same — ``directory`` itself). Archives that + failed to extract are skipped silently after a warning. + """ + out: List[Path] = [] + for archive in find_archives_in_dir(directory): + result = extract_archive(archive) + if result is not None: + out.append(result) + return out + + +def collect_audio_after_extraction(directory: Path) -> List[Path]: + """One-shot helper for the download plugins: extract any archives + in the directory, then return the walked audio file list. This is + the common pattern — torrent / usenet plugin gets a save_path, + calls this, hands the resulting files to the matching pipeline. + """ + extract_all_in_dir(directory) + return walk_audio_files(directory) + + +# --------------------------------------------------------------------------- +# Safety helpers +# --------------------------------------------------------------------------- + + +def _safe_extract_zip(zf: zipfile.ZipFile, dest: Path) -> None: + """Extract a zipfile after rejecting any member whose resolved + path escapes ``dest`` (path traversal protection). + """ + dest = dest.resolve() + for member in zf.namelist(): + target = (dest / member).resolve() + if dest not in target.parents and target != dest: + logger.error("archive_pipeline: refusing path-traversal member %r", member) + return + zf.extractall(dest) + + +def _safe_extract_tar(tf: tarfile.TarFile, dest: Path) -> None: + """Same path-traversal protection for tarfiles.""" + dest = dest.resolve() + for member in tf.getmembers(): + target = (dest / member.name).resolve() + if dest not in target.parents and target != dest: + logger.error("archive_pipeline: refusing path-traversal member %r", member.name) + return + # ``filter='data'`` is the Python 3.12+ safe extractor; fall back + # to the legacy call on older runtimes. + try: + tf.extractall(dest, filter='data') # type: ignore[call-arg] + except TypeError: + tf.extractall(dest) + + +def _extract_rar(archive_path: Path, dest: Path) -> Optional[Path]: + try: + import rarfile # type: ignore[import-untyped] + except ImportError: + logger.warning( + "archive_pipeline: cannot extract %s — rarfile library not installed. " + "Install with: pip install rarfile (and ensure unrar is on PATH).", + archive_path, + ) + return None + try: + with rarfile.RarFile(archive_path) as rf: + dest_resolved = dest.resolve() + for name in rf.namelist(): + target = (dest_resolved / name).resolve() + if dest_resolved not in target.parents and target != dest_resolved: + logger.error("archive_pipeline: refusing path-traversal rar member %r", name) + return None + rf.extractall(dest) + return dest + except Exception as e: + logger.error("archive_pipeline: rar extract failed for %s: %s", archive_path, e) + return None + + +def _extract_7z(archive_path: Path, dest: Path) -> Optional[Path]: + try: + import py7zr # type: ignore[import-untyped] + except ImportError: + logger.warning( + "archive_pipeline: cannot extract %s — py7zr library not installed. " + "Install with: pip install py7zr.", + archive_path, + ) + return None + try: + with py7zr.SevenZipFile(archive_path, 'r') as sz: + sz.extractall(path=dest) + return dest + except Exception as e: + logger.error("archive_pipeline: 7z extract failed for %s: %s", archive_path, e) + return None diff --git a/tests/test_archive_pipeline.py b/tests/test_archive_pipeline.py new file mode 100644 index 00000000..053c200a --- /dev/null +++ b/tests/test_archive_pipeline.py @@ -0,0 +1,238 @@ +"""Tests for ``core/archive_pipeline.py``. + +Covers the audio-file walker, archive detector, and zip / tar +extraction (rar / 7z paths use optional deps so they're only +exercised when the libs are present in the test environment). +Path-traversal protection gets explicit coverage — a malicious +archive must not escape the extraction directory. +""" + +from __future__ import annotations + +import os +import tarfile +import zipfile +from pathlib import Path + +import pytest + +from core.archive_pipeline import ( + AUDIO_EXTENSIONS, + ARCHIVE_EXTENSIONS, + collect_audio_after_extraction, + extract_archive, + extract_all_in_dir, + find_archives_in_dir, + is_archive, + walk_audio_files, +) + + +# --------------------------------------------------------------------------- +# Constants +# --------------------------------------------------------------------------- + + +def test_audio_extensions_cover_common_formats() -> None: + for ext in ('.flac', '.mp3', '.m4a', '.ogg', '.opus', '.wav', '.aac', '.wma'): + assert ext in AUDIO_EXTENSIONS + + +def test_archive_extensions_cover_common_formats() -> None: + for ext in ('.zip', '.rar', '.tar', '.7z'): + assert ext in ARCHIVE_EXTENSIONS + + +# --------------------------------------------------------------------------- +# is_archive +# --------------------------------------------------------------------------- + + +def test_is_archive_detects_simple_extensions(tmp_path: Path) -> None: + zip_path = tmp_path / 'x.zip' + zip_path.write_bytes(b'PK\x03\x04') # minimal — is_archive doesn't validate content + assert is_archive(zip_path) is True + + +def test_is_archive_detects_compound_tar_extensions(tmp_path: Path) -> None: + """``.tar.gz`` etc. — Path.suffix only catches the last suffix, + so the detector has to special-case compound extensions.""" + targz = tmp_path / 'x.tar.gz' + targz.write_bytes(b'\x1f\x8b') + assert is_archive(targz) is True + + +def test_is_archive_returns_false_for_audio(tmp_path: Path) -> None: + flac = tmp_path / 'song.flac' + flac.write_bytes(b'fLaC') + assert is_archive(flac) is False + + +def test_is_archive_returns_false_for_missing_file(tmp_path: Path) -> None: + assert is_archive(tmp_path / 'does-not-exist.zip') is False + + +# --------------------------------------------------------------------------- +# walk_audio_files +# --------------------------------------------------------------------------- + + +def test_walk_audio_files_finds_nested(tmp_path: Path) -> None: + # Layout: root/album/disc1/track.flac + root/album/disc2/track.mp3 + (tmp_path / 'album' / 'disc1').mkdir(parents=True) + (tmp_path / 'album' / 'disc2').mkdir(parents=True) + (tmp_path / 'album' / 'disc1' / 'track1.flac').write_bytes(b'fLaC') + (tmp_path / 'album' / 'disc2' / 'track1.mp3').write_bytes(b'ID3') + (tmp_path / 'album' / 'cover.jpg').write_bytes(b'\xff\xd8') + found = walk_audio_files(tmp_path) + names = sorted(p.name for p in found) + assert names == ['track1.flac', 'track1.mp3'] + + +def test_walk_audio_files_returns_empty_for_missing(tmp_path: Path) -> None: + assert walk_audio_files(tmp_path / 'does-not-exist') == [] + + +def test_walk_audio_files_ignores_non_audio(tmp_path: Path) -> None: + (tmp_path / 'readme.txt').write_text('hi') + (tmp_path / 'cover.png').write_bytes(b'\x89PNG') + assert walk_audio_files(tmp_path) == [] + + +def test_walk_audio_files_case_insensitive_extension(tmp_path: Path) -> None: + """Lots of torrents have uppercase extensions (.MP3, .FLAC) — + the walker must catch those too.""" + (tmp_path / 'TRACK.MP3').write_bytes(b'ID3') + (tmp_path / 'TRACK.FLAC').write_bytes(b'fLaC') + found = walk_audio_files(tmp_path) + assert len(found) == 2 + + +# --------------------------------------------------------------------------- +# find_archives_in_dir +# --------------------------------------------------------------------------- + + +def test_find_archives_in_dir_only_top_level(tmp_path: Path) -> None: + """find_archives doesn't recurse — torrents put the archive at + the top of the dir; deeper search risks extracting unrelated + archives that ship inside a sample folder, etc.""" + (tmp_path / 'album.zip').write_bytes(b'PK\x03\x04') + nested = tmp_path / 'subdir' + nested.mkdir() + (nested / 'nested.zip').write_bytes(b'PK\x03\x04') + found = find_archives_in_dir(tmp_path) + assert [p.name for p in found] == ['album.zip'] + + +def test_find_archives_in_dir_empty(tmp_path: Path) -> None: + assert find_archives_in_dir(tmp_path) == [] + assert find_archives_in_dir(tmp_path / 'missing') == [] + + +# --------------------------------------------------------------------------- +# extract_archive — zip +# --------------------------------------------------------------------------- + + +def test_extract_zip_writes_files(tmp_path: Path) -> None: + zip_path = tmp_path / 'album.zip' + with zipfile.ZipFile(zip_path, 'w') as zf: + zf.writestr('track1.mp3', b'ID3 track1 data') + zf.writestr('track2.flac', b'fLaC data') + result = extract_archive(zip_path) + assert result == zip_path.parent + assert (tmp_path / 'track1.mp3').exists() + assert (tmp_path / 'track2.flac').exists() + + +def test_extract_zip_rejects_path_traversal(tmp_path: Path) -> None: + """A malicious archive trying to write ``../../etc/passwd`` must + be refused without extracting anything.""" + zip_path = tmp_path / 'evil.zip' + extract_dest = tmp_path / 'staging' + extract_dest.mkdir() + with zipfile.ZipFile(zip_path, 'w') as zf: + zf.writestr('../escaped.txt', b'evil') + zf.writestr('safe.mp3', b'ID3') + extract_archive(zip_path, extract_to=extract_dest) + # Neither file should have landed — extraction aborts on the first + # traversal attempt. + assert not (extract_dest / 'safe.mp3').exists() + assert not (tmp_path / 'escaped.txt').exists() + + +def test_extract_zip_returns_none_for_bad_zip(tmp_path: Path) -> None: + bad = tmp_path / 'not-a-zip.zip' + bad.write_bytes(b'this is not a zip') + assert extract_archive(bad) is None + + +def test_extract_archive_missing_file_returns_none(tmp_path: Path) -> None: + assert extract_archive(tmp_path / 'does-not-exist.zip') is None + + +# --------------------------------------------------------------------------- +# extract_archive — tar +# --------------------------------------------------------------------------- + + +def test_extract_tar_gz_writes_files(tmp_path: Path) -> None: + payload = tmp_path / 'track.mp3' + payload.write_bytes(b'ID3 track') + tar_path = tmp_path / 'album.tar.gz' + with tarfile.open(tar_path, 'w:gz') as tf: + tf.add(payload, arcname='track.mp3') + payload.unlink() # remove the source so we can verify the extract recreated it + extract_archive(tar_path) + assert (tmp_path / 'track.mp3').exists() + + +def test_extract_tar_rejects_path_traversal(tmp_path: Path) -> None: + extract_dest = tmp_path / 'staging' + extract_dest.mkdir() + tar_path = tmp_path / 'evil.tar' + payload = tmp_path / 'src.txt' + payload.write_bytes(b'evil') + with tarfile.open(tar_path, 'w') as tf: + info = tf.gettarinfo(str(payload), arcname='../escaped.txt') + with payload.open('rb') as fh: + tf.addfile(info, fh) + extract_archive(tar_path, extract_to=extract_dest) + assert not (tmp_path / 'escaped.txt').exists() + + +# --------------------------------------------------------------------------- +# extract_all_in_dir + collect_audio_after_extraction +# --------------------------------------------------------------------------- + + +def test_extract_all_in_dir_handles_multiple_archives(tmp_path: Path) -> None: + (tmp_path / 'one.zip') # placeholder + z1 = tmp_path / 'one.zip' + z2 = tmp_path / 'two.zip' + with zipfile.ZipFile(z1, 'w') as zf: + zf.writestr('a.mp3', b'a') + with zipfile.ZipFile(z2, 'w') as zf: + zf.writestr('b.mp3', b'b') + out = extract_all_in_dir(tmp_path) + assert len(out) == 2 + assert (tmp_path / 'a.mp3').exists() + assert (tmp_path / 'b.mp3').exists() + + +def test_collect_audio_after_extraction_combines_loose_and_extracted(tmp_path: Path) -> None: + """The typical mixed case: torrent dropped a .zip and also some + loose .mp3 files alongside it. The collector returns BOTH.""" + (tmp_path / 'bonus.mp3').write_bytes(b'ID3') + zip_path = tmp_path / 'main.zip' + with zipfile.ZipFile(zip_path, 'w') as zf: + zf.writestr('track1.flac', b'fLaC') + zf.writestr('track2.flac', b'fLaC') + found = collect_audio_after_extraction(tmp_path) + names = sorted(p.name for p in found) + assert names == ['bonus.mp3', 'track1.flac', 'track2.flac'] + + +def test_collect_audio_after_extraction_no_archives_no_audio(tmp_path: Path) -> None: + assert collect_audio_after_extraction(tmp_path) == [] diff --git a/webui/static/helper.js b/webui/static/helper.js index f60b0527..5919d795 100644 --- a/webui/static/helper.js +++ b/webui/static/helper.js @@ -3415,6 +3415,7 @@ function closeHelperSearch() { const WHATS_NEW = { '2.6.0': [ { unreleased: true }, + { title: 'Archive pipeline module (groundwork for torrent / usenet downloads)', desc: 'new core/archive_pipeline.py — walks a directory for audio files (recursive, case-insensitive extensions), extracts zip / tar / tar.gz / rar / 7z archives in-place (rar and 7z are optional deps that warn but don\'t crash if absent), and rejects any archive member trying to escape the destination via path traversal. shared helper the upcoming torrent and usenet download plugins both consume — usenet downloaders usually auto-extract, but the occasional torrent ships an album in a .rar and SoulSync handles it now. 21 unit tests cover the walker + zip + tar extraction + path-traversal protection.' }, { title: 'Indexers & Downloaders tab restyled with collapsible sections', desc: 'tab now opens with an intro hero card explaining the flow (indexers find releases → downloader fetches them → SoulSync imports) and folds the rest into three collapsible sections: Indexers, Torrent Client, Usenet Client. each section gets a per-service color accent (Prowlarr orange, torrent sky-blue, usenet violet), a status dot in the header (green when Test Connection succeeds, red on failure, grey before testing), and Lidarr-style indexer cards with protocol badges instead of the inline emoji list. Indexers section is open by default; the downloader sections start collapsed since not everyone uses both protocols.' }, { title: 'Regression tests for the new indexer + downloader plumbing', desc: 'mocked unit tests for Prowlarr + all five downloader adapters (qBittorrent, Transmission, Deluge, SABnzbd, NZBGet). 54 tests pin the state-mapping tables, the parse logic, and the protocol quirks each client needs handled (qBit Referer header, Transmission session-id renegotiation, Deluge magnet-vs-URL method split, SAB queue+history merge, NZBGet 64-bit size fields). next time anyone touches one of these adapters, CI catches breakage before it hits a real downloader.' }, { title: 'Usenet client adapters (SABnzbd, NZBGet)', desc: 'third commit in the torrent + usenet rollout. SoulSync now talks to SABnzbd and NZBGet through a sibling adapter contract that mirrors the torrent adapter set — pick one downloader in Settings → Indexers & Downloaders, fill in its API key (SABnzbd) or username + password (NZBGet), and Test Connection confirms the link. all three layers are now stood up: Prowlarr finds releases, the torrent adapter and the usenet adapter each know how to ship work to the underlying client. next commit wires Prowlarr search → adapter dispatch → archive extraction so the new sources actually download. job state mapping covers SABnzbd queue + history and NZBGet groups + history, including the verify/repair/unpack phases that are unique to usenet.' }, From 080b1aa1b46a29c3ee3a7f98ceb00c31e7aa70f6 Mon Sep 17 00:00:00 2001 From: Broque Thomas <26755000+Nezreka@users.noreply.github.com> Date: Wed, 20 May 2026 17:22:19 -0700 Subject: [PATCH 02/25] feat(downloads): wire torrent + usenet as live download sources MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The payoff for the previous five commits. Two new download sources slot into the existing DownloadSourcePlugin contract, backed by Prowlarr (search) + the torrent or usenet client adapter (transfer) + archive_pipeline (post-extract walk). They appear in the Download Source dropdown next to Soulseek / Tidal / Lidarr / etc. and also participate in hybrid mode. Pipeline (both plugins, mirror shape): 1. search(query) → ProwlarrClient.search filtered to the right protocol, projected into TrackResult / AlbumResult shapes the existing search UI already speaks. Filename field encodes the indexer's download URL (or magnet URI for torrents) so download() can recover it later. 2. download() → decodes URL, hands it to the active adapter (qBittorrent / Transmission / Deluge for torrent; SABnzbd / NZBGet for usenet), spawns a background poll thread that tracks progress + reports the adapter-reported save_path. 3. On 'seeding' / 'completed' → archive_pipeline walks the save directory, extracts any archives the downloader didn't already unpack, picks the first audio file as the canonical file_path. Matches the Lidarr client's single-track-pick contract — picking which specific track to import happens in post-processing. - core/download_plugins/torrent.py: TorrentDownloadPlugin + module-level helpers (_decode_filename, _guess_quality_from_title, _parse_indexer_id_filter, _adapter_state_to_display, _row_to_status). Uses get_active_torrent_adapter() so a settings change to the client type takes effect without restart. - core/download_plugins/usenet.py: UsenetDownloadPlugin — parallel shape, reuses the torrent module's helpers. Different enough states (no seeding, no magnet) to warrant its own class but cheap to keep in lockstep. - core/download_plugins/registry.py: register 'torrent' and 'usenet' plugins. Per the registry docstring this is the only wiring point needed — the orchestrator picks them up automatically via the iteration helpers. - webui/index.html: 'Torrent Only (via Prowlarr)' + 'Usenet Only (via Prowlarr)' added to the Download Source dropdown. New redirect card (#prowlarr-source-redirect) explains that the actual config lives on the Indexers & Downloaders tab — shown whenever torrent or usenet is in the active source set. - webui/static/settings.js: HYBRID_SOURCES gets two new entries so hybrid mode can pick them up. updateDownloadSourceUI now toggles the redirect card based on active sources. - tests/test_torrent_usenet_plugins.py: 23 tests covering pure helpers (filename encode/decode round-trip incl. magnet URIs, quality guesser, state mapping), search projection logic (protocol filter, drops without URLs, magnet-preferred-over-URL, filename encoding, neutralised soulseek-specific score fields), is_configured (both prowlarr + adapter required), finalize (picks first audio file, errors on empty dir / missing save_path), clear/get_all lifecycle, DownloadSourcePlugin protocol conformance, and registry membership. --- core/download_plugins/registry.py | 4 + core/download_plugins/torrent.py | 472 +++++++++++++++++++++++++++ core/download_plugins/usenet.py | 332 +++++++++++++++++++ tests/test_torrent_usenet_plugins.py | 346 ++++++++++++++++++++ webui/index.html | 11 + webui/static/helper.js | 14 + webui/static/settings.js | 9 +- 7 files changed, 1187 insertions(+), 1 deletion(-) create mode 100644 core/download_plugins/torrent.py create mode 100644 core/download_plugins/usenet.py create mode 100644 tests/test_torrent_usenet_plugins.py diff --git a/core/download_plugins/registry.py b/core/download_plugins/registry.py index ad01c2e9..2ca5ebb8 100644 --- a/core/download_plugins/registry.py +++ b/core/download_plugins/registry.py @@ -40,6 +40,8 @@ from core.download_plugins.base import DownloadSourcePlugin # orchestrator did. from core.amazon_download_client import AmazonDownloadClient from core.deezer_download_client import DeezerDownloadClient +from core.download_plugins.torrent import TorrentDownloadPlugin +from core.download_plugins.usenet import UsenetDownloadPlugin from core.hifi_client import HiFiClient from core.lidarr_download_client import LidarrDownloadClient from core.qobuz_client import QobuzClient @@ -190,5 +192,7 @@ def build_default_registry() -> DownloadPluginRegistry: aliases=('deezer_dl',))) registry.register(PluginSpec(name='lidarr', factory=LidarrDownloadClient, display_name='Lidarr')) registry.register(PluginSpec(name='soundcloud',factory=SoundcloudClient, display_name='SoundCloud')) + registry.register(PluginSpec(name='torrent', factory=TorrentDownloadPlugin, display_name='Torrent (Prowlarr)')) + registry.register(PluginSpec(name='usenet', factory=UsenetDownloadPlugin, display_name='Usenet (Prowlarr)')) return registry diff --git a/core/download_plugins/torrent.py b/core/download_plugins/torrent.py new file mode 100644 index 00000000..a5487422 --- /dev/null +++ b/core/download_plugins/torrent.py @@ -0,0 +1,472 @@ +"""TorrentDownloadPlugin — composes Prowlarr search + torrent client +adapter + archive_pipeline into a uniform download source. + +Pipeline: +1. ``search(query)`` calls ``ProwlarrClient.search`` filtered to + ``protocol='torrent'`` results, projects releases into + ``TrackResult`` / ``AlbumResult`` shaped objects the existing + search UI already understands. Encodes the indexer's + ``downloadUrl`` (or magnet URI) into the filename so + ``download()`` can recover it. +2. ``download(username, filename, ...)`` decodes the URL, asks the + active torrent adapter (qBittorrent, Transmission, or Deluge per + user's settings) to add it, spawns a background thread that + polls the adapter for completion. +3. On completion the thread walks the adapter-reported save path + via ``archive_pipeline.collect_audio_after_extraction`` and + marks the download succeeded with the first audio file as the + primary ``file_path`` (matches Lidarr's single-track-pick + contract — picking which specific track to import happens in + post-processing, not here). + +Limitations: +- ``save_path`` is the torrent client's view of the disk. If + SoulSync runs on a different host than qBit / Trans / Deluge, + the post-processing pipeline can't see those files. The plugin + works fine for the all-on-one-box case (most users); remote + setups will need a future sync step (rclone / SMB / Docker + bind mount). +- Track-level metadata isn't available until after download. + Search results carry only the release title + indexer metadata; + individual track names are populated when the matching pipeline + walks the extracted audio files. +""" + +from __future__ import annotations + +import asyncio +import re +import threading +import time +import uuid +from pathlib import Path +from typing import Any, Dict, List, Optional, Tuple + +from config.settings import config_manager +from core.archive_pipeline import collect_audio_after_extraction +from core.download_plugins.base import DownloadSourcePlugin +from core.download_plugins.types import AlbumResult, DownloadStatus, TrackResult +from core.prowlarr_client import ( + DEFAULT_MUSIC_CATEGORIES, + ProwlarrClient, + ProwlarrSearchResult, +) +from core.torrent_clients import get_active_adapter as get_active_torrent_adapter +from utils.async_helpers import run_async +from utils.logging_config import get_logger + +logger = get_logger("download_plugins.torrent") + + +# Separator used to encode the download URL inside the filename +# field. Same convention Lidarr / YouTube use for embedding their +# own opaque identifiers — ``||``. +_FILENAME_SEP = '||' + +# Adapter states that count as the download being on-disk and +# safe to walk. ``seeding`` and ``completed`` both mean the +# bits are there; the user can pause seeding manually if they +# don't want to keep sharing. +_COMPLETE_STATES = frozenset(['seeding', 'completed']) + +# Max seconds the poll thread keeps watching one download before +# giving up. 6 hours covers slow private trackers without leaking +# threads forever on dead torrents. +_POLL_TIMEOUT_SECONDS = 6 * 60 * 60 + +# Poll cadence — torrent state changes slowly; no point hammering +# the WebUI more than once a second. +_POLL_INTERVAL_SECONDS = 2.0 + + +class TorrentDownloadPlugin(DownloadSourcePlugin): + """Torrent download source backed by Prowlarr + an active + torrent client adapter.""" + + def __init__(self) -> None: + self._prowlarr = ProwlarrClient() + # Track every download we've kicked off. Keyed by our own + # uuid — NOT the adapter's hash — because the orchestrator + # owns the lifecycle and we need a stable id even before + # the adapter has assigned one. + self.active_downloads: Dict[str, Dict[str, Any]] = {} + self._lock = threading.Lock() + self.shutdown_check = None + + def set_shutdown_check(self, check_callable): + self.shutdown_check = check_callable + + def reload_settings(self) -> None: + self._prowlarr.reload_settings() + + # ------------------------------------------------------------------ + # Lifecycle + # ------------------------------------------------------------------ + + def is_configured(self) -> bool: + if not self._prowlarr.is_configured(): + return False + adapter = get_active_torrent_adapter() + return bool(adapter and adapter.is_configured()) + + async def check_connection(self) -> bool: + if not self._prowlarr.is_configured(): + return False + adapter = get_active_torrent_adapter() + if not adapter or not adapter.is_configured(): + return False + # Probe both sides. A torrent download is useless if either + # the indexer or the downloader is unreachable. + prowlarr_ok = await self._prowlarr.check_connection() + if not prowlarr_ok: + return False + return await adapter.check_connection() + + # ------------------------------------------------------------------ + # Search + # ------------------------------------------------------------------ + + async def search( + self, + query: str, + timeout: Optional[int] = None, + progress_callback=None, + ) -> Tuple[List[TrackResult], List[AlbumResult]]: + if not self._prowlarr.is_configured(): + return ([], []) + try: + indexer_ids = _parse_indexer_id_filter() + results = await self._prowlarr.search( + query, + categories=DEFAULT_MUSIC_CATEGORIES, + indexer_ids=indexer_ids, + ) + except Exception as e: + logger.error("Torrent plugin search failed: %s", e) + return ([], []) + return self._project_results(results) + + def _project_results( + self, results: List[ProwlarrSearchResult] + ) -> Tuple[List[TrackResult], List[AlbumResult]]: + """Turn Prowlarr releases into TrackResult / AlbumResult + shaped objects. One TrackResult + one AlbumResult per + release — Prowlarr search hits are at the release level, + not the track level, so we can't synthesise track listings + without downloading the actual torrent.""" + tracks: List[TrackResult] = [] + albums: List[AlbumResult] = [] + for result in results: + if result.protocol != 'torrent': + continue + download_url = result.magnet_uri or result.download_url + if not download_url: + continue + filename = f"{download_url}{_FILENAME_SEP}{result.title}" + quality = _guess_quality_from_title(result.title) + tr = TrackResult( + username='torrent', + filename=filename, + size=result.size, + bitrate=None, + duration=None, + quality=quality, + # Torrent results don't have per-uploader slot / queue + # data the way Soulseek does. Fill with neutral values + # so the quality_score doesn't punish them artificially. + free_upload_slots=max(1, result.seeders or 0), + upload_speed=0, + queue_length=0, + artist=None, + title=result.title, + album=None, + track_number=None, + _source_metadata={ + 'indexer': result.indexer_name, + 'indexer_id': result.indexer_id, + 'seeders': result.seeders, + 'leechers': result.leechers, + 'grabs': result.grabs, + 'protocol': 'torrent', + }, + ) + tracks.append(tr) + albums.append(AlbumResult( + username='torrent', + album_path=f"torrent/{result.guid}", + album_title=result.title, + artist=None, + track_count=1, # unknown until download finishes + total_size=result.size, + tracks=[tr], + dominant_quality=quality, + year=None, + )) + return tracks, albums + + # ------------------------------------------------------------------ + # Download + # ------------------------------------------------------------------ + + async def download( + self, + username: str, + filename: str, + file_size: int = 0, + ) -> Optional[str]: + if not self.is_configured(): + return None + download_url, display_name = _decode_filename(filename) + if not download_url: + logger.error("Torrent download missing URL in filename: %r", filename) + return None + + download_id = str(uuid.uuid4()) + with self._lock: + self.active_downloads[download_id] = { + 'id': download_id, + 'filename': filename, + 'username': 'torrent', + 'display_name': display_name, + 'state': 'Initializing', + 'progress': 0.0, + 'size': file_size, + 'transferred': 0, + 'speed': 0, + 'file_path': None, + 'torrent_hash': None, + 'error': None, + } + + thread = threading.Thread( + target=self._download_thread, + args=(download_id, download_url, display_name), + daemon=True, + name=f'torrent-dl-{download_id[:8]}', + ) + thread.start() + return download_id + + def _download_thread(self, download_id: str, download_url: str, display_name: str) -> None: + """Background worker: hand the URL to the active adapter, + poll until done, then walk the resulting directory.""" + adapter = get_active_torrent_adapter() + if adapter is None or not adapter.is_configured(): + self._mark_error(download_id, "No torrent client configured") + return + + try: + torrent_hash = run_async(adapter.add_torrent(download_url)) + except Exception as e: + self._mark_error(download_id, f"add_torrent failed: {e}") + return + if not torrent_hash: + self._mark_error(download_id, "Torrent client refused the URL") + return + + with self._lock: + row = self.active_downloads.get(download_id) + if row is not None: + row['torrent_hash'] = torrent_hash + row['state'] = 'InProgress, Downloading' + + deadline = time.monotonic() + _POLL_TIMEOUT_SECONDS + last_save_path: Optional[str] = None + while time.monotonic() < deadline: + if self.shutdown_check and self.shutdown_check(): + return + try: + status = run_async(adapter.get_status(torrent_hash)) + except Exception as e: + logger.warning("Torrent poll error for %s: %s", torrent_hash, e) + status = None + + if status is None: + # Adapter forgot about the torrent — probably user-removed. + self._mark_error(download_id, "Torrent disappeared from client") + return + + with self._lock: + row = self.active_downloads.get(download_id) + if row is not None: + row['progress'] = status.progress * 100.0 + row['transferred'] = status.downloaded + row['speed'] = status.download_speed + row['size'] = status.size or row.get('size', 0) + row['state'] = _adapter_state_to_display(status.state) + row['error'] = status.error + if status.save_path: + last_save_path = status.save_path + + if status.state in _COMPLETE_STATES: + self._finalize_download(download_id, last_save_path) + return + if status.state == 'error': + self._mark_error(download_id, status.error or "Torrent client reported error") + return + + time.sleep(_POLL_INTERVAL_SECONDS) + + self._mark_error(download_id, "Torrent download timed out") + + def _finalize_download(self, download_id: str, save_path: Optional[str]) -> None: + """Adapter said complete. Walk the directory + pick the + first audio file as the canonical ``file_path``.""" + if not save_path: + self._mark_error(download_id, "Torrent completed but no save_path reported") + return + try: + audio_files = collect_audio_after_extraction(Path(save_path)) + except Exception as e: + self._mark_error(download_id, f"Post-extract walk failed: {e}") + return + if not audio_files: + self._mark_error(download_id, f"No audio files found in {save_path}") + return + primary = audio_files[0] + with self._lock: + row = self.active_downloads.get(download_id) + if row is not None: + row['state'] = 'Completed, Succeeded' + row['progress'] = 100.0 + row['file_path'] = str(primary) + logger.info("Torrent download complete: %s -> %s (%d audio files)", + download_id[:8], primary.name, len(audio_files)) + + def _mark_error(self, download_id: str, message: str) -> None: + logger.error("Torrent download %s failed: %s", download_id[:8], message) + with self._lock: + row = self.active_downloads.get(download_id) + if row is not None: + row['state'] = 'Completed, Errored' + row['error'] = message + + # ------------------------------------------------------------------ + # Status / lifecycle + # ------------------------------------------------------------------ + + async def get_all_downloads(self) -> List[DownloadStatus]: + with self._lock: + rows = list(self.active_downloads.values()) + return [_row_to_status(r) for r in rows] + + async def get_download_status(self, download_id: str) -> Optional[DownloadStatus]: + with self._lock: + row = self.active_downloads.get(download_id) + if row is None: + return None + return _row_to_status(row) + + async def cancel_download( + self, + download_id: str, + username: Optional[str] = None, + remove: bool = False, + ) -> bool: + adapter = get_active_torrent_adapter() + with self._lock: + row = self.active_downloads.get(download_id) + torrent_hash = row.get('torrent_hash') if row else None + if adapter and torrent_hash: + try: + await adapter.remove(torrent_hash, delete_files=remove) + except Exception as e: + logger.warning("Torrent cancel via adapter failed: %s", e) + with self._lock: + if remove: + self.active_downloads.pop(download_id, None) + else: + row = self.active_downloads.get(download_id) + if row is not None: + row['state'] = 'Cancelled' + return True + + async def clear_all_completed_downloads(self) -> bool: + with self._lock: + for did in list(self.active_downloads.keys()): + state = self.active_downloads[did].get('state', '') + if state.startswith('Completed') or state == 'Cancelled': + self.active_downloads.pop(did, None) + return True + + +# --------------------------------------------------------------------------- +# Module-level helpers (pure functions — easy to unit-test) +# --------------------------------------------------------------------------- + + +def _decode_filename(filename: str) -> Tuple[Optional[str], str]: + """Pull the encoded download URL out of the ``filename`` string. + Returns ``(url, display_name)``. ``url`` is None when the string + has no separator.""" + if not filename or _FILENAME_SEP not in filename: + return (None, filename or '') + url, display = filename.split(_FILENAME_SEP, 1) + return (url, display) + + +def _guess_quality_from_title(title: str) -> str: + """Read the quality hint from a release title — most music + torrents put the encoding right in the name (FLAC, MP3 320, + etc.). Falls back to ``'mp3'`` so quality_score doesn't crash.""" + if not title: + return 'mp3' + lower = title.lower() + if 'flac' in lower: + return 'flac' + if re.search(r'\b24[\s-]?bit\b', lower) or 'hi-?res' in lower: + return 'flac' + if 'aac' in lower: + return 'aac' + if 'ogg' in lower: + return 'ogg' + return 'mp3' + + +def _parse_indexer_id_filter() -> List[int]: + """Read the comma-separated indexer-ID allowlist from config. + Empty list = search every enabled indexer.""" + raw = (config_manager.get('prowlarr.indexer_ids', '') or '').strip() + if not raw: + return [] + out: List[int] = [] + for chunk in raw.split(','): + chunk = chunk.strip() + if not chunk: + continue + try: + out.append(int(chunk)) + except ValueError: + continue + return out + + +def _adapter_state_to_display(state: str) -> str: + """Translate the adapter-uniform state strings into the + ``'InProgress, Downloading'`` / ``'Completed, Succeeded'`` + style the existing UI expects (matches Soulseek + Lidarr).""" + mapping = { + 'queued': 'Queued', + 'downloading': 'InProgress, Downloading', + 'stalled': 'InProgress, Stalled', + 'seeding': 'Completed, Succeeded', + 'completed': 'Completed, Succeeded', + 'paused': 'Paused', + 'error': 'Completed, Errored', + } + return mapping.get(state, state.title()) + + +def _row_to_status(row: Dict[str, Any]) -> DownloadStatus: + return DownloadStatus( + id=row['id'], + filename=row['filename'], + username=row['username'], + state=row.get('state', 'Unknown'), + progress=float(row.get('progress', 0.0)), + size=int(row.get('size', 0)), + transferred=int(row.get('transferred', 0)), + speed=int(row.get('speed', 0)), + time_remaining=None, + file_path=row.get('file_path'), + ) diff --git a/core/download_plugins/usenet.py b/core/download_plugins/usenet.py new file mode 100644 index 00000000..fca9b01d --- /dev/null +++ b/core/download_plugins/usenet.py @@ -0,0 +1,332 @@ +"""UsenetDownloadPlugin — composes Prowlarr search + usenet client +adapter + archive_pipeline into a uniform download source. + +Mirrors ``TorrentDownloadPlugin`` in shape and lifecycle (see that +module's docstring for the full pipeline rationale). Differences: + +- Search filters Prowlarr results to ``protocol='usenet'``. +- ``add_nzb`` replaces ``add_torrent``; for NZBs we usually have + a direct HTTP URL the indexer exposes via Prowlarr. +- Usenet clients (SABnzbd, NZBGet) typically auto-extract during + post-processing, so ``archive_pipeline.collect_audio_after_extraction`` + usually has nothing to extract and just walks loose files. +""" + +from __future__ import annotations + +import threading +import time +import uuid +from pathlib import Path +from typing import Any, Dict, List, Optional, Tuple + +from core.archive_pipeline import collect_audio_after_extraction +from core.download_plugins.base import DownloadSourcePlugin +from core.download_plugins.torrent import ( + _adapter_state_to_display, + _decode_filename, + _guess_quality_from_title, + _parse_indexer_id_filter, + _row_to_status, + _COMPLETE_STATES, + _FILENAME_SEP, + _POLL_INTERVAL_SECONDS, + _POLL_TIMEOUT_SECONDS, +) +from core.download_plugins.types import AlbumResult, DownloadStatus, TrackResult +from core.prowlarr_client import ( + DEFAULT_MUSIC_CATEGORIES, + ProwlarrClient, + ProwlarrSearchResult, +) +from core.usenet_clients import get_active_adapter as get_active_usenet_adapter +from utils.async_helpers import run_async +from utils.logging_config import get_logger + +logger = get_logger("download_plugins.usenet") + + +class UsenetDownloadPlugin(DownloadSourcePlugin): + """Usenet download source backed by Prowlarr + an active usenet + client adapter (SABnzbd or NZBGet).""" + + def __init__(self) -> None: + self._prowlarr = ProwlarrClient() + self.active_downloads: Dict[str, Dict[str, Any]] = {} + self._lock = threading.Lock() + self.shutdown_check = None + + def set_shutdown_check(self, check_callable): + self.shutdown_check = check_callable + + def reload_settings(self) -> None: + self._prowlarr.reload_settings() + + def is_configured(self) -> bool: + if not self._prowlarr.is_configured(): + return False + adapter = get_active_usenet_adapter() + return bool(adapter and adapter.is_configured()) + + async def check_connection(self) -> bool: + if not self._prowlarr.is_configured(): + return False + adapter = get_active_usenet_adapter() + if not adapter or not adapter.is_configured(): + return False + if not await self._prowlarr.check_connection(): + return False + return await adapter.check_connection() + + # ------------------------------------------------------------------ + # Search + # ------------------------------------------------------------------ + + async def search( + self, + query: str, + timeout: Optional[int] = None, + progress_callback=None, + ) -> Tuple[List[TrackResult], List[AlbumResult]]: + if not self._prowlarr.is_configured(): + return ([], []) + try: + indexer_ids = _parse_indexer_id_filter() + results = await self._prowlarr.search( + query, + categories=DEFAULT_MUSIC_CATEGORIES, + indexer_ids=indexer_ids, + ) + except Exception as e: + logger.error("Usenet plugin search failed: %s", e) + return ([], []) + return self._project_results(results) + + def _project_results( + self, results: List[ProwlarrSearchResult] + ) -> Tuple[List[TrackResult], List[AlbumResult]]: + tracks: List[TrackResult] = [] + albums: List[AlbumResult] = [] + for result in results: + if result.protocol != 'usenet': + continue + if not result.download_url: + continue + filename = f"{result.download_url}{_FILENAME_SEP}{result.title}" + quality = _guess_quality_from_title(result.title) + tr = TrackResult( + username='usenet', + filename=filename, + size=result.size, + bitrate=None, + duration=None, + quality=quality, + # Usenet doesn't expose per-uploader concurrency the way + # Soulseek does; fill in neutral non-punishing values. + free_upload_slots=1, + upload_speed=0, + queue_length=0, + artist=None, + title=result.title, + album=None, + track_number=None, + _source_metadata={ + 'indexer': result.indexer_name, + 'indexer_id': result.indexer_id, + 'grabs': result.grabs, + 'protocol': 'usenet', + }, + ) + tracks.append(tr) + albums.append(AlbumResult( + username='usenet', + album_path=f"usenet/{result.guid}", + album_title=result.title, + artist=None, + track_count=1, + total_size=result.size, + tracks=[tr], + dominant_quality=quality, + year=None, + )) + return tracks, albums + + # ------------------------------------------------------------------ + # Download + # ------------------------------------------------------------------ + + async def download( + self, + username: str, + filename: str, + file_size: int = 0, + ) -> Optional[str]: + if not self.is_configured(): + return None + nzb_url, display_name = _decode_filename(filename) + if not nzb_url: + logger.error("Usenet download missing URL in filename: %r", filename) + return None + + download_id = str(uuid.uuid4()) + with self._lock: + self.active_downloads[download_id] = { + 'id': download_id, + 'filename': filename, + 'username': 'usenet', + 'display_name': display_name, + 'state': 'Initializing', + 'progress': 0.0, + 'size': file_size, + 'transferred': 0, + 'speed': 0, + 'file_path': None, + 'job_id': None, + 'error': None, + } + + thread = threading.Thread( + target=self._download_thread, + args=(download_id, nzb_url), + daemon=True, + name=f'usenet-dl-{download_id[:8]}', + ) + thread.start() + return download_id + + def _download_thread(self, download_id: str, nzb_url: str) -> None: + adapter = get_active_usenet_adapter() + if adapter is None or not adapter.is_configured(): + self._mark_error(download_id, "No usenet client configured") + return + + try: + job_id = run_async(adapter.add_nzb(nzb_url)) + except Exception as e: + self._mark_error(download_id, f"add_nzb failed: {e}") + return + if not job_id: + self._mark_error(download_id, "Usenet client refused the NZB") + return + + with self._lock: + row = self.active_downloads.get(download_id) + if row is not None: + row['job_id'] = job_id + row['state'] = 'InProgress, Downloading' + + deadline = time.monotonic() + _POLL_TIMEOUT_SECONDS + last_save_path: Optional[str] = None + while time.monotonic() < deadline: + if self.shutdown_check and self.shutdown_check(): + return + try: + status = run_async(adapter.get_status(job_id)) + except Exception as e: + logger.warning("Usenet poll error for %s: %s", job_id, e) + status = None + + if status is None: + self._mark_error(download_id, "Usenet job disappeared from client") + return + + with self._lock: + row = self.active_downloads.get(download_id) + if row is not None: + row['progress'] = status.progress * 100.0 + row['transferred'] = status.downloaded + row['speed'] = status.download_speed + row['size'] = status.size or row.get('size', 0) + row['state'] = _adapter_state_to_display(status.state) + row['error'] = status.error + if status.save_path: + last_save_path = status.save_path + + if status.state in _COMPLETE_STATES: + self._finalize_download(download_id, last_save_path) + return + if status.state == 'failed': + self._mark_error(download_id, status.error or "Usenet client reported failure") + return + + time.sleep(_POLL_INTERVAL_SECONDS) + + self._mark_error(download_id, "Usenet download timed out") + + def _finalize_download(self, download_id: str, save_path: Optional[str]) -> None: + if not save_path: + self._mark_error(download_id, "Usenet job completed but no save_path reported") + return + try: + audio_files = collect_audio_after_extraction(Path(save_path)) + except Exception as e: + self._mark_error(download_id, f"Post-extract walk failed: {e}") + return + if not audio_files: + self._mark_error(download_id, f"No audio files found in {save_path}") + return + primary = audio_files[0] + with self._lock: + row = self.active_downloads.get(download_id) + if row is not None: + row['state'] = 'Completed, Succeeded' + row['progress'] = 100.0 + row['file_path'] = str(primary) + logger.info("Usenet download complete: %s -> %s (%d audio files)", + download_id[:8], primary.name, len(audio_files)) + + def _mark_error(self, download_id: str, message: str) -> None: + logger.error("Usenet download %s failed: %s", download_id[:8], message) + with self._lock: + row = self.active_downloads.get(download_id) + if row is not None: + row['state'] = 'Completed, Errored' + row['error'] = message + + # ------------------------------------------------------------------ + # Status / lifecycle + # ------------------------------------------------------------------ + + async def get_all_downloads(self) -> List[DownloadStatus]: + with self._lock: + rows = list(self.active_downloads.values()) + return [_row_to_status(r) for r in rows] + + async def get_download_status(self, download_id: str) -> Optional[DownloadStatus]: + with self._lock: + row = self.active_downloads.get(download_id) + if row is None: + return None + return _row_to_status(row) + + async def cancel_download( + self, + download_id: str, + username: Optional[str] = None, + remove: bool = False, + ) -> bool: + adapter = get_active_usenet_adapter() + with self._lock: + row = self.active_downloads.get(download_id) + job_id = row.get('job_id') if row else None + if adapter and job_id: + try: + await adapter.remove(job_id, delete_files=remove) + except Exception as e: + logger.warning("Usenet cancel via adapter failed: %s", e) + with self._lock: + if remove: + self.active_downloads.pop(download_id, None) + else: + row = self.active_downloads.get(download_id) + if row is not None: + row['state'] = 'Cancelled' + return True + + async def clear_all_completed_downloads(self) -> bool: + with self._lock: + for did in list(self.active_downloads.keys()): + state = self.active_downloads[did].get('state', '') + if state.startswith('Completed') or state == 'Cancelled': + self.active_downloads.pop(did, None) + return True diff --git a/tests/test_torrent_usenet_plugins.py b/tests/test_torrent_usenet_plugins.py new file mode 100644 index 00000000..4a9818a2 --- /dev/null +++ b/tests/test_torrent_usenet_plugins.py @@ -0,0 +1,346 @@ +"""Tests for ``core/download_plugins/torrent.py`` and ``usenet.py``. + +Both plugins compose a Prowlarr client + an adapter + the archive +pipeline. The tests mock the Prowlarr client and the active adapter +factory so we can pin the projection logic, filename encoding / +decoding, finalize path, and the cancel / clear lifecycle without +touching the network or filesystem (beyond ``tmp_path``). +""" + +from __future__ import annotations + +import asyncio +from pathlib import Path +from typing import Optional +from unittest.mock import MagicMock, patch + +import pytest + +from core.download_plugins.torrent import ( + TorrentDownloadPlugin, + _adapter_state_to_display, + _decode_filename, + _FILENAME_SEP, + _guess_quality_from_title, +) +from core.download_plugins.usenet import UsenetDownloadPlugin +from core.prowlarr_client import ProwlarrSearchResult +from core.torrent_clients.base import TorrentStatus +from core.usenet_clients.base import UsenetStatus + + +def _run(coro): + return asyncio.new_event_loop().run_until_complete(coro) + + +# --------------------------------------------------------------------------- +# Pure helpers +# --------------------------------------------------------------------------- + + +def test_decode_filename_splits_on_separator() -> None: + url, display = _decode_filename(f"https://x/y.torrent{_FILENAME_SEP}Album Name") + assert url == 'https://x/y.torrent' + assert display == 'Album Name' + + +def test_decode_filename_without_separator_returns_none_url() -> None: + url, display = _decode_filename('just a name') + assert url is None + assert display == 'just a name' + + +def test_decode_filename_handles_magnet_with_embedded_separators() -> None: + """Magnet URIs contain ``=`` and ``&`` but no ``||`` — so a + magnet must round-trip cleanly through the encoder.""" + magnet = 'magnet:?xt=urn:btih:abc123&dn=Album+Name' + encoded = f"{magnet}{_FILENAME_SEP}Display" + url, display = _decode_filename(encoded) + assert url == magnet + assert display == 'Display' + + +def test_guess_quality_from_title() -> None: + assert _guess_quality_from_title('Album [FLAC]') == 'flac' + assert _guess_quality_from_title('Album 24-bit Hi-Res') == 'flac' + assert _guess_quality_from_title('Album [MP3 320]') == 'mp3' + assert _guess_quality_from_title('Album [AAC 256]') == 'aac' + assert _guess_quality_from_title('Album [OGG]') == 'ogg' + # Default fallback so quality_score doesn't crash on bare titles. + assert _guess_quality_from_title('Just A Title') == 'mp3' + assert _guess_quality_from_title('') == 'mp3' + + +def test_adapter_state_mapping_covers_complete_states() -> None: + assert _adapter_state_to_display('downloading') == 'InProgress, Downloading' + assert _adapter_state_to_display('seeding') == 'Completed, Succeeded' + assert _adapter_state_to_display('completed') == 'Completed, Succeeded' + assert _adapter_state_to_display('error') == 'Completed, Errored' + assert _adapter_state_to_display('stalled') == 'InProgress, Stalled' + # Unknown state falls through with title-casing rather than crashing. + assert _adapter_state_to_display('weird') == 'Weird' + + +# --------------------------------------------------------------------------- +# Torrent plugin — search projection +# --------------------------------------------------------------------------- + + +def _make_torrent_result(**overrides) -> ProwlarrSearchResult: + base = dict( + guid='guid-1', title='Some Album FLAC', indexer_id=3, + indexer_name='Indexer', protocol='torrent', + download_url='https://x/y.torrent', magnet_uri=None, + info_url=None, size=500_000_000, seeders=12, leechers=3, + grabs=100, publish_date='2026-01-01', categories=[3040], + raw={}, + ) + base.update(overrides) + return ProwlarrSearchResult(**base) + + +def test_torrent_project_results_drops_non_torrent_protocol() -> None: + plugin = TorrentDownloadPlugin() + results = [ + _make_torrent_result(), + _make_torrent_result(protocol='usenet', title='Usenet Album'), + ] + tracks, albums = plugin._project_results(results) + assert len(tracks) == 1 + assert tracks[0].title == 'Some Album FLAC' + assert len(albums) == 1 + + +def test_torrent_project_results_drops_releases_without_download_url() -> None: + plugin = TorrentDownloadPlugin() + results = [_make_torrent_result(download_url=None, magnet_uri=None)] + tracks, albums = plugin._project_results(results) + assert tracks == [] + assert albums == [] + + +def test_torrent_project_results_prefers_magnet_when_available() -> None: + plugin = TorrentDownloadPlugin() + magnet = 'magnet:?xt=urn:btih:abc' + results = [_make_torrent_result(magnet_uri=magnet, download_url='https://x/y.torrent')] + tracks, _ = plugin._project_results(results) + url, _ = _decode_filename(tracks[0].filename) + assert url == magnet + + +def test_torrent_project_results_encodes_url_and_title_in_filename() -> None: + plugin = TorrentDownloadPlugin() + tracks, _ = plugin._project_results([_make_torrent_result()]) + url, display = _decode_filename(tracks[0].filename) + assert url == 'https://x/y.torrent' + assert display == 'Some Album FLAC' + + +def test_torrent_project_results_neutralizes_soulseek_specific_fields() -> None: + """TrackResult.quality_score punishes results with no upload + slots; torrent results don't have that concept so the + projection has to fill in non-punishing neutral values.""" + plugin = TorrentDownloadPlugin() + tracks, _ = plugin._project_results([_make_torrent_result(seeders=0)]) + # seeders=0 means we should still hand the picker something + # usable. free_upload_slots floors at 1 to avoid the 0-slot + # penalty applied to dead Soulseek peers. + assert tracks[0].free_upload_slots >= 1 + + +# --------------------------------------------------------------------------- +# Torrent plugin — is_configured / check_connection +# --------------------------------------------------------------------------- + + +def test_torrent_is_configured_requires_both_sides() -> None: + plugin = TorrentDownloadPlugin() + with patch.object(plugin._prowlarr, 'is_configured', return_value=False), \ + patch('core.download_plugins.torrent.get_active_torrent_adapter', return_value=None): + assert plugin.is_configured() is False + fake_adapter = MagicMock() + fake_adapter.is_configured.return_value = False + with patch.object(plugin._prowlarr, 'is_configured', return_value=True), \ + patch('core.download_plugins.torrent.get_active_torrent_adapter', return_value=fake_adapter): + assert plugin.is_configured() is False + fake_adapter.is_configured.return_value = True + with patch.object(plugin._prowlarr, 'is_configured', return_value=True), \ + patch('core.download_plugins.torrent.get_active_torrent_adapter', return_value=fake_adapter): + assert plugin.is_configured() is True + + +# --------------------------------------------------------------------------- +# Torrent plugin — finalize / cancel / clear +# --------------------------------------------------------------------------- + + +def test_torrent_finalize_picks_first_audio_file(tmp_path: Path) -> None: + plugin = TorrentDownloadPlugin() + # Seed an in-flight download row + plugin.active_downloads['dl-1'] = { + 'id': 'dl-1', 'filename': 'x', 'username': 'torrent', + 'display_name': 'X', 'state': 'InProgress, Downloading', + 'progress': 50.0, 'size': 0, 'transferred': 0, 'speed': 0, + 'file_path': None, 'torrent_hash': 'h1', 'error': None, + } + # Drop two audio files in the save dir + (tmp_path / 'b.flac').write_bytes(b'fLaC') + (tmp_path / 'a.mp3').write_bytes(b'ID3') + plugin._finalize_download('dl-1', str(tmp_path)) + row = plugin.active_downloads['dl-1'] + assert row['state'] == 'Completed, Succeeded' + assert row['progress'] == 100.0 + # Walker sorts → 'a.mp3' wins as first. + assert row['file_path'].endswith('a.mp3') + + +def test_torrent_finalize_marks_error_when_no_audio(tmp_path: Path) -> None: + plugin = TorrentDownloadPlugin() + plugin.active_downloads['dl-1'] = { + 'id': 'dl-1', 'filename': 'x', 'username': 'torrent', + 'display_name': 'X', 'state': 'InProgress, Downloading', + 'progress': 50.0, 'size': 0, 'transferred': 0, 'speed': 0, + 'file_path': None, 'torrent_hash': 'h1', 'error': None, + } + # tmp_path has no audio files + plugin._finalize_download('dl-1', str(tmp_path)) + assert plugin.active_downloads['dl-1']['state'] == 'Completed, Errored' + assert 'No audio files' in plugin.active_downloads['dl-1']['error'] + + +def test_torrent_finalize_marks_error_when_save_path_missing() -> None: + plugin = TorrentDownloadPlugin() + plugin.active_downloads['dl-1'] = { + 'id': 'dl-1', 'filename': 'x', 'username': 'torrent', + 'display_name': 'X', 'state': 'InProgress, Downloading', + 'progress': 50.0, 'size': 0, 'transferred': 0, 'speed': 0, + 'file_path': None, 'torrent_hash': 'h1', 'error': None, + } + plugin._finalize_download('dl-1', None) + assert plugin.active_downloads['dl-1']['state'] == 'Completed, Errored' + assert 'no save_path' in plugin.active_downloads['dl-1']['error'].lower() + + +def test_torrent_clear_completed_drops_only_done_rows() -> None: + plugin = TorrentDownloadPlugin() + plugin.active_downloads['a'] = {'id': 'a', 'state': 'InProgress, Downloading'} + plugin.active_downloads['b'] = {'id': 'b', 'state': 'Completed, Succeeded'} + plugin.active_downloads['c'] = {'id': 'c', 'state': 'Completed, Errored'} + plugin.active_downloads['d'] = {'id': 'd', 'state': 'Cancelled'} + _run(plugin.clear_all_completed_downloads()) + assert list(plugin.active_downloads.keys()) == ['a'] + + +def test_torrent_get_all_returns_status_objects() -> None: + plugin = TorrentDownloadPlugin() + plugin.active_downloads['a'] = { + 'id': 'a', 'filename': 'f', 'username': 'torrent', + 'state': 'InProgress, Downloading', 'progress': 50.0, + 'size': 100, 'transferred': 50, 'speed': 1000, + 'file_path': None, + } + statuses = _run(plugin.get_all_downloads()) + assert len(statuses) == 1 + assert statuses[0].id == 'a' + assert statuses[0].progress == 50.0 + + +# --------------------------------------------------------------------------- +# Usenet plugin — projection +# --------------------------------------------------------------------------- + + +def _make_usenet_result(**overrides) -> ProwlarrSearchResult: + base = dict( + guid='guid-u', title='Some Album', indexer_id=5, + indexer_name='UsenetIndexer', protocol='usenet', + download_url='https://x/y.nzb', magnet_uri=None, + info_url=None, size=400_000_000, seeders=None, leechers=None, + grabs=42, publish_date='2026-01-01', categories=[3010], + raw={}, + ) + base.update(overrides) + return ProwlarrSearchResult(**base) + + +def test_usenet_project_drops_torrent_protocol() -> None: + plugin = UsenetDownloadPlugin() + results = [_make_usenet_result(), _make_usenet_result(protocol='torrent', title='T')] + tracks, albums = plugin._project_results(results) + assert len(tracks) == 1 + assert tracks[0].username == 'usenet' + + +def test_usenet_project_drops_results_without_download_url() -> None: + """Usenet plugins reject magnet-only results entirely — NZBs + don't have a magnet equivalent.""" + plugin = UsenetDownloadPlugin() + results = [_make_usenet_result(download_url=None)] + tracks, _ = plugin._project_results(results) + assert tracks == [] + + +def test_usenet_project_encodes_url_in_filename() -> None: + plugin = UsenetDownloadPlugin() + tracks, _ = plugin._project_results([_make_usenet_result()]) + url, display = _decode_filename(tracks[0].filename) + assert url == 'https://x/y.nzb' + assert display == 'Some Album' + + +def test_usenet_finalize_picks_first_audio_file(tmp_path: Path) -> None: + """Same finalize contract as torrent — sanity check the shared + helper path works for usenet too.""" + plugin = UsenetDownloadPlugin() + plugin.active_downloads['u-1'] = { + 'id': 'u-1', 'filename': 'x', 'username': 'usenet', + 'display_name': 'X', 'state': 'InProgress, Downloading', + 'progress': 50.0, 'size': 0, 'transferred': 0, 'speed': 0, + 'file_path': None, 'job_id': 'j1', 'error': None, + } + (tmp_path / 'track1.flac').write_bytes(b'fLaC') + plugin._finalize_download('u-1', str(tmp_path)) + assert plugin.active_downloads['u-1']['state'] == 'Completed, Succeeded' + assert plugin.active_downloads['u-1']['file_path'].endswith('track1.flac') + + +def test_usenet_is_configured_requires_both_sides() -> None: + plugin = UsenetDownloadPlugin() + fake_adapter = MagicMock() + fake_adapter.is_configured.return_value = True + with patch.object(plugin._prowlarr, 'is_configured', return_value=False), \ + patch('core.download_plugins.usenet.get_active_usenet_adapter', return_value=fake_adapter): + assert plugin.is_configured() is False + with patch.object(plugin._prowlarr, 'is_configured', return_value=True), \ + patch('core.download_plugins.usenet.get_active_usenet_adapter', return_value=None): + assert plugin.is_configured() is False + with patch.object(plugin._prowlarr, 'is_configured', return_value=True), \ + patch('core.download_plugins.usenet.get_active_usenet_adapter', return_value=fake_adapter): + assert plugin.is_configured() is True + + +# --------------------------------------------------------------------------- +# Plugin conformance — both must satisfy the DownloadSourcePlugin Protocol +# --------------------------------------------------------------------------- + + +def test_plugins_conform_to_protocol() -> None: + from core.download_plugins.base import DownloadSourcePlugin + assert isinstance(TorrentDownloadPlugin(), DownloadSourcePlugin) + assert isinstance(UsenetDownloadPlugin(), DownloadSourcePlugin) + + +# --------------------------------------------------------------------------- +# Registry — both should register cleanly +# --------------------------------------------------------------------------- + + +def test_registry_includes_torrent_and_usenet() -> None: + """The registry decides what shows up in the orchestrator's + iteration helpers. If we forget to register a new plugin the + download source dropdown will silently no-op.""" + from core.download_plugins.registry import build_default_registry + registry = build_default_registry() + names = registry.names() + assert 'torrent' in names + assert 'usenet' in names diff --git a/webui/index.html b/webui/index.html index f6b0cb1b..35147361 100644 --- a/webui/index.html +++ b/webui/index.html @@ -4328,6 +4328,8 @@ + +
@@ -4735,6 +4737,15 @@
+ + +
-
⚠️ Filesystem access required
+
💡 Filesystem access — easiest setup
- Your torrent / usenet client downloads to its own folder (not Soulseek's). SoulSync needs read access to that folder to import the files. + Torrent and usenet clients write files to their own download folders, not Soulseek's. SoulSync needs read access to those folders to import the files. Simplest fix: point all your downloaders (Soulseek, qBittorrent, SABnzbd / NZBGet) at the same download folder. One folder, one path to worry about, everything just works.
    -
  • Bare-metal (everything on one PC): works out of the box — SoulSync reads the path directly.
    • -
  • Docker: map the client's save folder into the SoulSync container at the same path the client reports. Commented examples are in docker-compose.yml — uncomment and edit the host paths.
    • +
  • Bare-metal: just set every client's download path to the same folder (e.g. your existing slskd folder). Done.
    • +
  • Docker: easiest is to reuse the ./downloads mount you already have for slskd — point qBit and SAB at /downloads inside their containers too. Or use the commented placeholders in docker-compose.yml if you want separate folders.
  • Remote client (NAS / different host): mount the remote folder locally (SMB / NFS / rclone) before SoulSync can see it.
diff --git a/webui/static/helper.js b/webui/static/helper.js index 1a9d6913..8ae56bc0 100644 --- a/webui/static/helper.js +++ b/webui/static/helper.js @@ -3415,7 +3415,7 @@ function closeHelperSearch() { const WHATS_NEW = { '2.6.0': [ { unreleased: true }, - { title: 'Docker mount heads-up for torrent / usenet sources', desc: 'added an orange warning card on the Indexers & Downloaders tab explaining the filesystem-access requirement up front. torrent and usenet clients each download to their own folders (not Soulseek\'s), so SoulSync needs read access to those paths — works out of the box on bare-metal, needs a volume mount on Docker, needs SMB / NFS / rclone on remote-NAS setups. docker-compose.yml gained commented placeholder mounts for /downloads/torrents and /downloads/usenet that users can uncomment + edit to point at their qBit / SAB save folders.' }, + { title: 'Filesystem-access heads-up for torrent / usenet sources', desc: 'new advisory card on the Indexers & Downloaders tab explaining the cleanest setup: point ALL your downloaders (Soulseek, qBittorrent, SABnzbd / NZBGet) at the same download folder. One folder, one mount, everything just works. Bare-metal needs no change; Docker users can reuse the existing ./downloads mount and just configure each client to write there. docker-compose.yml updated to call this out as the easiest path, with optional commented placeholders for users who prefer separate folders per protocol.' }, { title: 'Torrent and Usenet downloads', desc: 'two new download sources live in the Download Source dropdown: Torrent Only (via Prowlarr) and Usenet Only (via Prowlarr). they reuse the Prowlarr + torrent client + usenet client you set up on the Indexers & Downloaders tab. searches go through Prowlarr filtered by protocol, picked releases get handed to your torrent client or usenet client, and the resulting files get walked through archive_pipeline (extracts .zip / .rar / .tar when the client didn\'t already do it) and handed to the matching pipeline. both sources are also available in hybrid mode alongside soulseek / youtube / tidal / etc. one caveat: SoulSync needs read access to the torrent / usenet client\'s save_path — works out of the box for everything-on-one-box setups, but remote downloader hosts will need a future sync step.' }, { title: 'Archive pipeline module (groundwork for torrent / usenet downloads)', desc: 'new core/archive_pipeline.py — walks a directory for audio files (recursive, case-insensitive extensions), extracts zip / tar / tar.gz / rar / 7z archives in-place (rar and 7z are optional deps that warn but don\'t crash if absent), and rejects any archive member trying to escape the destination via path traversal. shared helper the upcoming torrent and usenet download plugins both consume — usenet downloaders usually auto-extract, but the occasional torrent ships an album in a .rar and SoulSync handles it now. 21 unit tests cover the walker + zip + tar extraction + path-traversal protection.' }, { title: 'Indexers & Downloaders tab restyled with collapsible sections', desc: 'tab now opens with an intro hero card explaining the flow (indexers find releases → downloader fetches them → SoulSync imports) and folds the rest into three collapsible sections: Indexers, Torrent Client, Usenet Client. each section gets a per-service color accent (Prowlarr orange, torrent sky-blue, usenet violet), a status dot in the header (green when Test Connection succeeds, red on failure, grey before testing), and Lidarr-style indexer cards with protocol badges instead of the inline emoji list. Indexers section is open by default; the downloader sections start collapsed since not everyone uses both protocols.' }, From 478fd25dd69a194c6081b88d2a4137464a2dcb80 Mon Sep 17 00:00:00 2001 From: Broque Thomas <26755000+Nezreka@users.noreply.github.com> Date: Wed, 20 May 2026 18:06:23 -0700 Subject: [PATCH 05/25] fix(downloads): pre-fill artist/title so search UI doesn't show download URL MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Real-world test surfaced the bug — torrent results displayed 'by download?apikey=c15d6f69...&link=...' as the uploader / artist in the basic search UI. The cause is TrackResult.__post_init__: when artist is None it runs parse_filename_metadata on the bare filename, and our filename starts with the indexer's download URL (needed so download() can recover the URL later). The auto-parser treats the URL as 'artist' and ships it to the UI. Fix: - core/download_plugins/torrent.py: new _parse_release_title() splits 'Artist - Title' / 'Artist - Album' out of the release title and strips trailing [FLAC] / (2016) tags. Falls back to ('', cleaned_title) when no dash is found, and explicitly rejects URL-looking strings as an extra defence. The projection pre-fills both artist and title on TrackResult, so __post_init__ skips the auto-parse entirely. When the release title has no dash, artist defaults to the indexer name so the UI shows 'by Indexer' instead of a URL. - core/download_plugins/usenet.py: imports the new helper and applies the same fix. - tests/test_torrent_usenet_plugins.py: 5 tests for the new helper (dash split, trailing-tag stripping, no-dash fallback, multiple-dash preservation, URL-prefix rejection). Existing projection tests updated to assert artist + title come through parsed correctly, plus a new test pinning the indexer-name fallback for titles without a dash so the URL-leak regression can't return. --- core/download_plugins/torrent.py | 46 ++++++++++++++++--- core/download_plugins/usenet.py | 15 ++++--- tests/test_torrent_usenet_plugins.py | 67 +++++++++++++++++++++++++--- 3 files changed, 113 insertions(+), 15 deletions(-) diff --git a/core/download_plugins/torrent.py b/core/download_plugins/torrent.py index a5487422..414b31d5 100644 --- a/core/download_plugins/torrent.py +++ b/core/download_plugins/torrent.py @@ -164,6 +164,7 @@ class TorrentDownloadPlugin(DownloadSourcePlugin): continue filename = f"{download_url}{_FILENAME_SEP}{result.title}" quality = _guess_quality_from_title(result.title) + parsed_artist, parsed_title = _parse_release_title(result.title) tr = TrackResult( username='torrent', filename=filename, @@ -177,9 +178,13 @@ class TorrentDownloadPlugin(DownloadSourcePlugin): free_upload_slots=max(1, result.seeders or 0), upload_speed=0, queue_length=0, - artist=None, - title=result.title, - album=None, + # Pre-fill artist + title so TrackResult.__post_init__ + # doesn't auto-parse the filename — our filename starts + # with the indexer download URL, which would otherwise + # show up as "by download?apikey=..." in the UI. + artist=parsed_artist or result.indexer_name or 'Torrent', + title=parsed_title or result.title, + album=parsed_title or None, track_number=None, _source_metadata={ 'indexer': result.indexer_name, @@ -194,8 +199,8 @@ class TorrentDownloadPlugin(DownloadSourcePlugin): albums.append(AlbumResult( username='torrent', album_path=f"torrent/{result.guid}", - album_title=result.title, - artist=None, + album_title=parsed_title or result.title, + artist=parsed_artist or None, track_count=1, # unknown until download finishes total_size=result.size, tracks=[tr], @@ -405,6 +410,37 @@ def _decode_filename(filename: str) -> Tuple[Optional[str], str]: return (url, display) +def _parse_release_title(title: str) -> Tuple[str, str]: + """Split a release title into ``(artist, title)`` using the + ``Artist - Title`` / ``Artist - Album`` convention almost every + indexer follows. Returns ``('', title)`` when no dash is found. + + Without this, ``TrackResult.__post_init__`` runs the bare + filename through ``parse_filename_metadata`` — and our filename + starts with the indexer's download URL, so the auto-parser + extracts garbage like ``download?apikey=...`` as the artist + and shows it in the search-result UI's "by" line. Pre-filling + the artist field short-circuits the auto-parse. + """ + if not title: + return ('', '') + # Strip common quality / format tags so the dash split doesn't + # eat them — "Artist - Album [FLAC] (2020)" → "Artist", "Album". + cleaned = re.sub(r'\s*[\[\(][^\]\)]*[\]\)]\s*$', '', title.strip()) + # Look for the FIRST " - " (or "-" surrounded by content). Some + # release titles have multiple dashes (subtitle dashes); the + # first split is the artist/work boundary. + parts = re.split(r'\s+-\s+|\s+-(?=\S)|(?<=\S)-\s+', cleaned, maxsplit=1) + if len(parts) == 2: + artist = parts[0].strip() + rest = parts[1].strip() + # Reject obvious non-artist prefixes (URLs, hashes, single + # punctuation) so we don't propagate garbage. + if artist and not re.match(r'^https?:|^[a-f0-9]{32,}$', artist): + return (artist, rest or cleaned) + return ('', cleaned) + + def _guess_quality_from_title(title: str) -> str: """Read the quality hint from a release title — most music torrents put the encoding right in the name (FLAC, MP3 320, diff --git a/core/download_plugins/usenet.py b/core/download_plugins/usenet.py index fca9b01d..8442db43 100644 --- a/core/download_plugins/usenet.py +++ b/core/download_plugins/usenet.py @@ -27,6 +27,7 @@ from core.download_plugins.torrent import ( _decode_filename, _guess_quality_from_title, _parse_indexer_id_filter, + _parse_release_title, _row_to_status, _COMPLETE_STATES, _FILENAME_SEP, @@ -114,6 +115,7 @@ class UsenetDownloadPlugin(DownloadSourcePlugin): continue filename = f"{result.download_url}{_FILENAME_SEP}{result.title}" quality = _guess_quality_from_title(result.title) + parsed_artist, parsed_title = _parse_release_title(result.title) tr = TrackResult( username='usenet', filename=filename, @@ -126,9 +128,12 @@ class UsenetDownloadPlugin(DownloadSourcePlugin): free_upload_slots=1, upload_speed=0, queue_length=0, - artist=None, - title=result.title, - album=None, + # Pre-fill artist + title so TrackResult.__post_init__ + # doesn't auto-parse the filename — same URL-in-filename + # gotcha as the torrent plugin. + artist=parsed_artist or result.indexer_name or 'Usenet', + title=parsed_title or result.title, + album=parsed_title or None, track_number=None, _source_metadata={ 'indexer': result.indexer_name, @@ -141,8 +146,8 @@ class UsenetDownloadPlugin(DownloadSourcePlugin): albums.append(AlbumResult( username='usenet', album_path=f"usenet/{result.guid}", - album_title=result.title, - artist=None, + album_title=parsed_title or result.title, + artist=parsed_artist or None, track_count=1, total_size=result.size, tracks=[tr], diff --git a/tests/test_torrent_usenet_plugins.py b/tests/test_torrent_usenet_plugins.py index 4a9818a2..bf2f856b 100644 --- a/tests/test_torrent_usenet_plugins.py +++ b/tests/test_torrent_usenet_plugins.py @@ -22,6 +22,7 @@ from core.download_plugins.torrent import ( _decode_filename, _FILENAME_SEP, _guess_quality_from_title, + _parse_release_title, ) from core.download_plugins.usenet import UsenetDownloadPlugin from core.prowlarr_client import ProwlarrSearchResult @@ -71,6 +72,45 @@ def test_guess_quality_from_title() -> None: assert _guess_quality_from_title('') == 'mp3' +def test_parse_release_title_splits_artist_dash_title() -> None: + """Most release titles follow 'Artist - Title' / 'Artist - Album'.""" + assert _parse_release_title('Danny Brown - Atrocity Exhibition') == ('Danny Brown', 'Atrocity Exhibition') + assert _parse_release_title('Kendrick Lamar - DAMN.') == ('Kendrick Lamar', 'DAMN.') + + +def test_parse_release_title_strips_trailing_tags() -> None: + """Quality / year tags at the end shouldn't pollute the title.""" + artist, title = _parse_release_title('Danny Brown - Atrocity Exhibition [FLAC]') + assert artist == 'Danny Brown' + assert title == 'Atrocity Exhibition' + artist, title = _parse_release_title('Danny Brown - Atrocity Exhibition (2016)') + assert artist == 'Danny Brown' + assert title == 'Atrocity Exhibition' + + +def test_parse_release_title_handles_no_dash() -> None: + """Some indexers post bare titles. Caller should fall back to + the indexer name as the 'artist' field.""" + artist, title = _parse_release_title('JustATitle') + assert artist == '' + assert title == 'JustATitle' + + +def test_parse_release_title_handles_dashes_in_title() -> None: + """Track titles can themselves contain dashes — only split on + the FIRST one so subtitles survive.""" + artist, title = _parse_release_title('Artist - Title - Live Version') + assert artist == 'Artist' + assert title == 'Title - Live Version' + + +def test_parse_release_title_rejects_url_prefix() -> None: + """Defensive: if a URL somehow lands in the title field, refuse + to call it an artist.""" + artist, title = _parse_release_title('https://example.com/x - Album') + assert artist == '' + + def test_adapter_state_mapping_covers_complete_states() -> None: assert _adapter_state_to_display('downloading') == 'InProgress, Downloading' assert _adapter_state_to_display('seeding') == 'Completed, Succeeded' @@ -88,7 +128,7 @@ def test_adapter_state_mapping_covers_complete_states() -> None: def _make_torrent_result(**overrides) -> ProwlarrSearchResult: base = dict( - guid='guid-1', title='Some Album FLAC', indexer_id=3, + guid='guid-1', title='Danny Brown - Atrocity Exhibition [FLAC]', indexer_id=3, indexer_name='Indexer', protocol='torrent', download_url='https://x/y.torrent', magnet_uri=None, info_url=None, size=500_000_000, seeders=12, leechers=3, @@ -107,7 +147,8 @@ def test_torrent_project_results_drops_non_torrent_protocol() -> None: ] tracks, albums = plugin._project_results(results) assert len(tracks) == 1 - assert tracks[0].title == 'Some Album FLAC' + assert tracks[0].title == 'Atrocity Exhibition' + assert tracks[0].artist == 'Danny Brown' assert len(albums) == 1 @@ -133,7 +174,20 @@ def test_torrent_project_results_encodes_url_and_title_in_filename() -> None: tracks, _ = plugin._project_results([_make_torrent_result()]) url, display = _decode_filename(tracks[0].filename) assert url == 'https://x/y.torrent' - assert display == 'Some Album FLAC' + assert display == 'Danny Brown - Atrocity Exhibition [FLAC]' + + +def test_torrent_project_falls_back_to_indexer_name_when_title_lacks_dash() -> None: + """When the title has no 'Artist -' prefix we'd auto-parse the + filename (which starts with the indexer download URL) and end + up showing the URL in the UI's 'by' field. Pre-filling artist + with the indexer name avoids that.""" + plugin = TorrentDownloadPlugin() + tracks, _ = plugin._project_results([_make_torrent_result(title='JustATitle')]) + assert tracks[0].artist == 'Indexer' + # And the URL is definitely not the artist. + assert 'http' not in tracks[0].artist + assert '||' not in tracks[0].artist def test_torrent_project_results_neutralizes_soulseek_specific_fields() -> None: @@ -252,7 +306,7 @@ def test_torrent_get_all_returns_status_objects() -> None: def _make_usenet_result(**overrides) -> ProwlarrSearchResult: base = dict( - guid='guid-u', title='Some Album', indexer_id=5, + guid='guid-u', title='Some Artist - Some Album', indexer_id=5, indexer_name='UsenetIndexer', protocol='usenet', download_url='https://x/y.nzb', magnet_uri=None, info_url=None, size=400_000_000, seeders=None, leechers=None, @@ -285,7 +339,10 @@ def test_usenet_project_encodes_url_in_filename() -> None: tracks, _ = plugin._project_results([_make_usenet_result()]) url, display = _decode_filename(tracks[0].filename) assert url == 'https://x/y.nzb' - assert display == 'Some Album' + assert display == 'Some Artist - Some Album' + # Artist + title should be parsed out, not auto-extracted from filename. + assert tracks[0].artist == 'Some Artist' + assert tracks[0].title == 'Some Album' def test_usenet_finalize_picks_first_audio_file(tmp_path: Path) -> None: From e83b66147192a22891e81931419802780d018f50 Mon Sep 17 00:00:00 2001 From: Broque Thomas <26755000+Nezreka@users.noreply.github.com> Date: Wed, 20 May 2026 18:11:30 -0700 Subject: [PATCH 06/25] fix(torrent): use before/after diff to recover qBit info-hash MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Live testing surfaced: every download attempt failed with 'Torrent client refused the URL', but qBit was actually accepting the add request fine. The bug was in our hash-lookup strategy. qBittorrent's /api/v2/torrents/add returns 200 'Ok.' regardless of whether the URL was actually valid / accepted / registered. The previous code then queried /torrents/info?category=soulsync to find the just-added torrent — but qBit hadn't categorised the new torrent yet on the first poll, AND a fresh install has no 'soulsync' category configured, so the lookup returned empty and the adapter reported failure for every working download. New strategy: - Snapshot every torrent hash qBit currently tracks BEFORE posting to /add. - POST /add, accept its (uninformative) 200 OK. - Poll the all-torrents list for up to 5 seconds, looking for a hash that wasn't present in the before-snapshot. First new hash wins. The diff strategy works the same for /add with urls= (HTTP URL / magnet) and /add with files= (raw .torrent upload), so both paths now share the same _all_hashes + _poll_for_new_hash helpers. Adds a warning log when qBit returns an unexpected body and an error log when no new hash appears (so future investigation has breadcrumbs). --- core/torrent_clients/qbittorrent.py | 62 +++++++++++++++++++++++------ 1 file changed, 49 insertions(+), 13 deletions(-) diff --git a/core/torrent_clients/qbittorrent.py b/core/torrent_clients/qbittorrent.py index 3373086e..579c8ec8 100644 --- a/core/torrent_clients/qbittorrent.py +++ b/core/torrent_clients/qbittorrent.py @@ -164,28 +164,60 @@ class QBittorrentAdapter: category: str, save_path: Optional[str], ) -> Optional[str]: - data = {'urls': url_or_magnet, 'category': category or self._category} + cat = category or self._category + # Snapshot the current set of torrent hashes BEFORE adding — + # qBittorrent's /add endpoint returns 200 "Ok." regardless of + # whether the URL was actually accepted or registered, and + # category-filtered lookups race the add (qBit hasn't + # categorised the new torrent yet on the first poll). Diffing + # before / after is the only reliable way to recover the hash. + before = self._all_hashes() + if before is None: + return None + data = {'urls': url_or_magnet, 'category': cat} if save_path or self._save_path: data['savepath'] = save_path or self._save_path resp = self._call('POST', '/api/v2/torrents/add', data=data) if not resp or not resp.ok: + logger.warning("qBittorrent /torrents/add returned HTTP %s body=%r", + resp.status_code if resp else 'no-response', + (resp.text[:200] if resp else '')) return None - # qBittorrent's /add endpoint returns 200 "Ok." on success but - # NOT the resulting info-hash. We have to look it up via - # /torrents/info filtered by category — the just-added torrent - # will be the newest entry in that category. - return self._lookup_latest_hash(category or self._category) + if resp.text and resp.text.strip() and resp.text.strip() != 'Ok.': + logger.warning("qBittorrent /torrents/add unexpected body: %r", resp.text[:200]) + return None + new_hash = self._poll_for_new_hash(before) + if not new_hash: + logger.error("qBittorrent accepted the request but no new torrent appeared — " + "URL may have been rejected (bad magnet, unreachable HTTPS, " + "duplicate hash, etc.)") + return new_hash - def _lookup_latest_hash(self, category: str) -> Optional[str]: - resp = self._call('GET', '/api/v2/torrents/info', params={'category': category, 'sort': 'added_on', 'reverse': 'true', 'limit': 1}) + def _all_hashes(self) -> Optional[set]: + """Return the set of every torrent hash qBit currently tracks, + or None on lookup failure.""" + resp = self._call('GET', '/api/v2/torrents/info') if not resp or not resp.ok: return None try: - items = resp.json() - if items: - return items[0].get('hash') + return {item.get('hash') for item in resp.json() if item.get('hash')} except Exception as e: logger.error("qBittorrent /torrents/info parse failed: %s", e) + return None + + def _poll_for_new_hash(self, before: set) -> Optional[str]: + """Poll up to ~5s for a new torrent to appear (qBit takes a + moment to fetch the .torrent file from the URL and register + it). Returns the new hash, or None if nothing showed up.""" + import time as _time + for _ in range(10): + _time.sleep(0.5) + current = self._all_hashes() + if current is None: + continue + new = current - before + if new: + return next(iter(new)) return None async def add_torrent_file( @@ -205,14 +237,18 @@ class QBittorrentAdapter: category: str, save_path: Optional[str], ) -> Optional[str]: - data = {'category': category or self._category} + cat = category or self._category + before = self._all_hashes() + if before is None: + return None + data = {'category': cat} if save_path or self._save_path: data['savepath'] = save_path or self._save_path files = {'torrents': ('soulsync.torrent', file_bytes, 'application/x-bittorrent')} resp = self._call('POST', '/api/v2/torrents/add', data=data, files=files) if not resp or not resp.ok: return None - return self._lookup_latest_hash(category or self._category) + return self._poll_for_new_hash(before) async def get_status(self, torrent_id: str) -> Optional[TorrentStatus]: loop = asyncio.get_event_loop() From a2db5382bb492b48beb2c2de0a2d975493844557 Mon Sep 17 00:00:00 2001 From: Broque Thomas <26755000+Nezreka@users.noreply.github.com> Date: Wed, 20 May 2026 18:27:14 -0700 Subject: [PATCH 07/25] fix(downloads): route torrent/usenet through streaming-result validation path MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Live-test bug: Spotify-flow downloads with Torrent Only as the active source produced 'download_failed' for every track. Searches hit Prowlarr fine but no candidate ever got picked. Root cause was in core/downloads/validation.py's get_valid_candidates: - The streaming-source allowlist for the structured-metadata path didn't include 'torrent' / 'usenet', so torrent results fell into the Soulseek matching branch. - Soulseek matching parses ``candidate.filename`` as a slskd-style ``Artist/Album/Track.flac`` path. Torrent / usenet filenames are encoded as ``||`` so the orchestrator can recover the URL — splitting that string on slashes produced garbage path segments that never matched the expected artist, every candidate failed the artist-folder gate, returned [], track status flipped to 'not_found'. Fixes: - _streaming_sources now includes 'torrent' and 'usenet'. They take the structured-metadata scoring path that reads r.title / r.artist directly (the projection layer pre-fills both correctly). - Artist gate skipped for torrent/usenet, same as YouTube. Album- level releases legitimately don't expose per-track artist — the projection falls back to the indexer name as the 'artist' field, which would otherwise fail the gate against every Spotify artist. - New album-name fallback scoring: for torrent/usenet only, the candidate title is ALSO scored against the wanted track's spotify_track.album field, and the max of (track-title score, album-title score) wins. This makes a candidate titled "GNX (2024) [FLAC]" match every track on the GNX album rather than scoring near zero against a specific track title like "Luther (with SZA)". match_type 'album_release' for visibility. All 9 existing validation tests still pass. --- core/downloads/validation.py | 36 ++++++++++++++++++++++++++++++++---- 1 file changed, 32 insertions(+), 4 deletions(-) diff --git a/core/downloads/validation.py b/core/downloads/validation.py index 3ac38751..337cde73 100644 --- a/core/downloads/validation.py +++ b/core/downloads/validation.py @@ -97,8 +97,12 @@ def get_valid_candidates(results, spotify_track, query): return [] # Streaming sources (YouTube, Tidal, Qobuz, HiFi, Deezer, SoundCloud) return structured API results - # with proper artist/title metadata — score using the same matching engine as Soulseek - _streaming_sources = ("youtube", "tidal", "qobuz", "hifi", "deezer_dl", "soundcloud", "amazon") + # with proper artist/title metadata — score using the same matching engine as Soulseek. + # Torrent / usenet results also belong here: their filename field is a download URL, not + # a slskd-style ``Artist/Album/Track.flac`` path, so the Soulseek matcher would extract + # garbage segments from it. Routing them through the streaming path means score_track_match + # reads ``r.title`` and ``r.artist`` directly (which the torrent/usenet projections pre-fill). + _streaming_sources = ("youtube", "tidal", "qobuz", "hifi", "deezer_dl", "soundcloud", "amazon", "torrent", "usenet") if results[0].username in _streaming_sources: source_label = results[0].username.replace('_dl', '').title() expected_artists = spotify_track.artists if spotify_track else [] @@ -142,6 +146,27 @@ def get_valid_candidates(results, spotify_track, query): candidate_duration_ms=r.duration or 0, ) + # Torrent / usenet results are typically release-level (album torrents). + # Looking for "Luther (with SZA)" against a candidate titled + # "GNX (2024) [FLAC]" scores ~0 on track-title alone, even though + # the album torrent does in fact contain the wanted track. Score + # the candidate title against the wanted track's ALBUM name too + # and take the max, so album-level releases match every track on + # them. The album_track_count bonus only kicks in when we have + # a non-empty album string to compare against. + if r.username in ('torrent', 'usenet') and spotify_track and spotify_track.album: + album_conf, _ = matching_engine.score_track_match( + source_title=spotify_track.album, + source_artists=expected_artists, + source_duration_ms=0, # albums don't have one duration + candidate_title=r.title or '', + candidate_artists=[r.artist] if r.artist else [], + candidate_duration_ms=0, + ) + if album_conf > confidence: + confidence = album_conf + match_type = 'album_release' + # Version detection penalty — reject live/remix/acoustic when expecting original r_title_lower = (r.title or '').lower() is_wrong_version = False @@ -162,8 +187,11 @@ def get_valid_candidates(results, spotify_track, query): # Artist gate — streaming APIs (Tidal/Qobuz/HiFi/Deezer) have reliable metadata, # so "My Will" by "B. Starr" should never match expected "B小町". - # Skip for YouTube — artist is parsed from video titles and often unreliable. - if r.username != 'youtube': + # Skip for YouTube (video-title parsing is unreliable) and torrent/usenet + # (album-level releases legitimately don't expose per-track artist — + # the projection layer fills artist with the indexer name as a fallback, + # which would otherwise fail the gate against every Spotify artist). + if r.username not in ('youtube', 'torrent', 'usenet'): from difflib import SequenceMatcher import re as _re _cand_artist_raw = r.artist or '' From c990ce079d43fbf5e509347b6ad5ffa14a8eaa2e Mon Sep 17 00:00:00 2001 From: Broque Thomas <26755000+Nezreka@users.noreply.github.com> Date: Wed, 20 May 2026 18:48:48 -0700 Subject: [PATCH 08/25] feat(downloads): album-bundle flow for torrent/usenet single-source mode MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fixes the core architectural mismatch between indexer-based sources and the per-track search-and-pick contract every other download plugin satisfies. Prowlarr returns release-level torrents and NZBs; searching for "Luther (with SZA)" against the GNX album torrent scores near-zero on track-title similarity. Per-track candidate validation rejects every result, every track in the batch flips to not_found. The album-name fallback added in an earlier commit papers over it for some cases but doesn't fix the fundamental behavior: the user wanted the whole album. New album-bundle flow does what the user actually wanted: 1. Gate fires inside core/downloads/master.py BEFORE the per-track analysis loop, strictly when the batch has an album context AND download_source.mode is 'torrent' or 'usenet' (single-source — hybrid stays per-track to preserve fallback to Soulseek / etc.). 2. Plugin's new download_album_to_staging method searches Prowlarr ONCE for the album as a whole (' '), filters to the right protocol, runs results through _pick_best_album_release. 3. Picker prefers seeded FLAC over low-seeded MP3, drops single- track torrents that snuck in via the 40 MB size floor (single tracks are typically ~10 MB), falls back to most-seeded when every candidate is below the floor. 4. Picked release goes to the active adapter (qBit / Transmission / Deluge for torrent; SAB / NZBGet for usenet). Polls until complete with progress mirrored into the batch state so the Downloads page can show meaningful status. 5. On completion the existing archive_pipeline walks the save dir (extracting archives if any), every audio file gets copied into the staging folder via _unique_staging_path so concurrent batches don't collide. 6. Gate exits, master worker continues into the normal per-track flow. Each track task hits try_staging_match early in the worker and finds its file by fuzzy title match — no Prowlarr search ever fires per-track, no candidate rejection, files flow through the existing post-processing pipeline (tags, AcoustID, library import). Gate is strictly opt-in. Three orthogonal conditions must all hold: batch_is_album, mode in ('torrent', 'usenet'), and the plugin must expose download_album_to_staging. Any other source / hybrid mode / non-album batch flows through the master worker unchanged. The existing per-track torrent path still works for basic-search single-track grabs. - core/download_plugins/torrent.py: download_album_to_staging plus _pick_best_album_release and _unique_staging_path helpers (shared with the usenet plugin). _poll_album_download mirrors the existing poll loop with progress callback emission. - core/download_plugins/usenet.py: parallel implementation reusing the picker + staging helpers. Different state set ('failed' vs 'error') from the usenet adapter contract. - core/downloads/master.py: ~90-line gate right after batch context loading. Mirrors plugin lifecycle into batch state under ``album_bundle_*`` keys so the Downloads page can render progress while the torrent/usenet job runs (per-track tasks don't exist yet during this phase). Failed bundle download fails the batch with a meaningful error; missing plugin / context falls back to the per-track flow with a warning. - tests/test_torrent_usenet_plugins.py: 5 new tests pinning the album picker preferences (FLAC over MP3 with comparable size + better seeders, size floor drops singles, fallback when all small), staging-path collision suffix, and the not-configured short-circuit. --- core/download_plugins/torrent.py | 217 ++++++++++++++++++++++++++- core/download_plugins/usenet.py | 130 ++++++++++++++++ core/downloads/master.py | 94 ++++++++++++ tests/test_torrent_usenet_plugins.py | 57 +++++++ webui/static/helper.js | 1 + 5 files changed, 495 insertions(+), 4 deletions(-) diff --git a/core/download_plugins/torrent.py b/core/download_plugins/torrent.py index 414b31d5..faa38c89 100644 --- a/core/download_plugins/torrent.py +++ b/core/download_plugins/torrent.py @@ -1,7 +1,9 @@ """TorrentDownloadPlugin — composes Prowlarr search + torrent client adapter + archive_pipeline into a uniform download source. -Pipeline: +Two flows: + +**Per-track flow** (basic search, single-track wishlist) — 1. ``search(query)`` calls ``ProwlarrClient.search`` filtered to ``protocol='torrent'`` results, projects releases into ``TrackResult`` / ``AlbumResult`` shaped objects the existing @@ -15,9 +17,20 @@ Pipeline: 3. On completion the thread walks the adapter-reported save path via ``archive_pipeline.collect_audio_after_extraction`` and marks the download succeeded with the first audio file as the - primary ``file_path`` (matches Lidarr's single-track-pick - contract — picking which specific track to import happens in - post-processing, not here). + primary ``file_path``. + +**Album-bundle flow** (album-context batch downloads — wired in +``core/downloads/master.py``) — +4. ``download_album_to_staging(album, artist, staging_dir)`` does + ONE Prowlarr search for the whole release, picks the best + torrent (prefers FLAC, decent seeders, reasonable size), + downloads it, extracts archives if needed, copies every audio + file into the staging directory. The existing per-track + ``try_staging_match`` flow then finds + imports each track by + fuzzy title match against the staged files. Per-track Prowlarr + queries never fire — track titles like "Luther (with SZA)" + would match album torrents like "GNX (2024) [FLAC]" at near- + zero confidence and break the per-track dispatch. Limitations: - ``save_path`` is the torrent client's view of the disk. If @@ -36,6 +49,7 @@ from __future__ import annotations import asyncio import re +import shutil import threading import time import uuid @@ -394,6 +408,201 @@ class TorrentDownloadPlugin(DownloadSourcePlugin): self.active_downloads.pop(did, None) return True + # ------------------------------------------------------------------ + # Album-bundle flow + # ------------------------------------------------------------------ + + def download_album_to_staging( + self, + album_name: str, + artist_name: str, + staging_dir: str, + progress_callback=None, + ) -> Dict[str, Any]: + """One-shot album download: search Prowlarr for the whole + release, pick the best torrent, fetch it, extract if needed, + copy every audio file into ``staging_dir`` so the existing + ``try_staging_match`` flow can hand each track off to the + post-processing pipeline. + + ``progress_callback`` is called with a dict on each state + change so the batch UI can show download progress without + waiting for the whole thing. + + Returns ``{'success': bool, 'files': [paths], 'error': str|None}``. + """ + result: Dict[str, Any] = {'success': False, 'files': [], 'error': None} + if not self.is_configured(): + result['error'] = 'Torrent source not configured' + return result + + adapter = get_active_torrent_adapter() + if adapter is None or not adapter.is_configured(): + result['error'] = 'No active torrent client' + return result + + def _emit(state: str, **extra) -> None: + if progress_callback: + payload = {'state': state, **extra} + try: + progress_callback(payload) + except Exception as cb_exc: + logger.debug("[Torrent album] progress callback failed: %s", cb_exc) + + # Phase 1: search Prowlarr for the album. + query = f"{artist_name} {album_name}".strip() + _emit('searching', query=query) + try: + search_results = run_async(self._prowlarr.search( + query, categories=DEFAULT_MUSIC_CATEGORIES, + indexer_ids=_parse_indexer_id_filter(), + )) + except Exception as e: + result['error'] = f'Prowlarr search failed: {e}' + return result + + candidates = [r for r in search_results if r.protocol == 'torrent'] + if not candidates: + result['error'] = f'No torrent results found for "{query}"' + return result + + picked = _pick_best_album_release(candidates) + if picked is None: + result['error'] = 'No suitable torrent candidate after filtering' + return result + + download_url = picked.magnet_uri or picked.download_url + logger.info("[Torrent album] Picked '%s' (size=%.1fMB seeders=%s indexer=%s)", + picked.title, picked.size / 1_048_576, picked.seeders, picked.indexer_name) + _emit('queued', release=picked.title, size=picked.size, seeders=picked.seeders) + + # Phase 2: hand to adapter. + try: + torrent_id = run_async(adapter.add_torrent(download_url)) + except Exception as e: + result['error'] = f'Torrent client refused the release: {e}' + return result + if not torrent_id: + result['error'] = 'Torrent client refused the release' + return result + + # Phase 3: poll until complete. + _emit('downloading', release=picked.title) + save_path = self._poll_album_download(adapter, torrent_id, picked.title, _emit) + if save_path is None: + result['error'] = 'Torrent download failed or timed out' + return result + + # Phase 4: extract + walk + copy to staging. + _emit('staging', release=picked.title) + try: + audio_files = collect_audio_after_extraction(Path(save_path)) + except Exception as e: + result['error'] = f'Failed to walk audio files: {e}' + return result + if not audio_files: + result['error'] = f'No audio files found in {save_path}' + return result + + staging_path = Path(staging_dir) + staging_path.mkdir(parents=True, exist_ok=True) + copied: List[str] = [] + for src in audio_files: + dst = _unique_staging_path(staging_path, src) + try: + shutil.copy2(src, dst) + copied.append(str(dst)) + except Exception as e: + logger.warning("[Torrent album] Failed to copy %s -> %s: %s", src, dst, e) + if not copied: + result['error'] = 'No audio files copied to staging' + return result + logger.info("[Torrent album] Staged %d audio files for '%s'", len(copied), album_name) + _emit('staged', count=len(copied)) + result['success'] = True + result['files'] = copied + return result + + def _poll_album_download(self, adapter, torrent_id, title, emit) -> Optional[str]: + """Poll the adapter until the torrent is complete. Returns + the save path or ``None`` on timeout / failure.""" + deadline = time.monotonic() + _POLL_TIMEOUT_SECONDS + last_save_path: Optional[str] = None + while time.monotonic() < deadline: + if self.shutdown_check and self.shutdown_check(): + return None + try: + status = run_async(adapter.get_status(torrent_id)) + except Exception as e: + logger.warning("[Torrent album] Poll error: %s", e) + status = None + if status is None: + logger.error("[Torrent album] '%s' disappeared from client", title) + return None + emit('downloading', progress=status.progress, downloaded=status.downloaded, + speed=status.download_speed) + if status.save_path: + last_save_path = status.save_path + if status.state in _COMPLETE_STATES: + return last_save_path + if status.state == 'error': + logger.error("[Torrent album] '%s' errored: %s", title, status.error) + return None + time.sleep(_POLL_INTERVAL_SECONDS) + logger.error("[Torrent album] '%s' timed out", title) + return None + + +# --------------------------------------------------------------------------- +# Album-pick helpers +# --------------------------------------------------------------------------- + + +_QUALITY_SCORE = {'flac': 4, 'ogg': 3, 'aac': 2, 'mp3': 1} + + +def _pick_best_album_release(candidates) -> Optional[Any]: + """Pick the single best torrent for an album-bundle download. + + Heuristic, in priority order: + 1. Reasonable album-ish size (40 MB – 3 GB) — drops single-track + releases that snuck in and quarantines suspicious giants. + 2. Higher seeders > lower (dead torrents = dead downloads). + 3. Higher quality (FLAC > AAC > MP3) inferred from title. + 4. Larger size as tiebreaker (often = higher bitrate). + """ + MIN_BYTES = 40 * 1024 * 1024 + MAX_BYTES = 3 * 1024 * 1024 * 1024 + + sized = [c for c in candidates if MIN_BYTES <= (c.size or 0) <= MAX_BYTES] + pool = sized or candidates + if not pool: + return None + + def _score(c) -> tuple: + seeders = c.seeders or 0 + quality = _QUALITY_SCORE.get(_guess_quality_from_title(c.title), 0) + size = c.size or 0 + return (seeders, quality, size) + + return max(pool, key=_score) + + +def _unique_staging_path(staging_dir: Path, src: Path) -> Path: + """Return a destination path inside ``staging_dir`` that doesn't + collide with an existing file. Appends ``_1``, ``_2``, etc. before + the extension when needed.""" + dest = staging_dir / src.name + if not dest.exists(): + return dest + stem = dest.stem + suffix = dest.suffix + for i in range(1, 1000): + candidate = staging_dir / f"{stem}_{i}{suffix}" + if not candidate.exists(): + return candidate + return dest # give up — overwrite + # --------------------------------------------------------------------------- # Module-level helpers (pure functions — easy to unit-test) diff --git a/core/download_plugins/usenet.py b/core/download_plugins/usenet.py index 8442db43..92792305 100644 --- a/core/download_plugins/usenet.py +++ b/core/download_plugins/usenet.py @@ -14,6 +14,7 @@ module's docstring for the full pipeline rationale). Differences: from __future__ import annotations +import shutil import threading import time import uuid @@ -28,7 +29,9 @@ from core.download_plugins.torrent import ( _guess_quality_from_title, _parse_indexer_id_filter, _parse_release_title, + _pick_best_album_release, _row_to_status, + _unique_staging_path, _COMPLETE_STATES, _FILENAME_SEP, _POLL_INTERVAL_SECONDS, @@ -335,3 +338,130 @@ class UsenetDownloadPlugin(DownloadSourcePlugin): if state.startswith('Completed') or state == 'Cancelled': self.active_downloads.pop(did, None) return True + + # ------------------------------------------------------------------ + # Album-bundle flow + # ------------------------------------------------------------------ + + def download_album_to_staging( + self, + album_name: str, + artist_name: str, + staging_dir: str, + progress_callback=None, + ) -> Dict[str, Any]: + """Usenet sibling of ``TorrentDownloadPlugin.download_album_to_staging``. + See that method's docstring for the contract.""" + result: Dict[str, Any] = {'success': False, 'files': [], 'error': None} + if not self.is_configured(): + result['error'] = 'Usenet source not configured' + return result + + adapter = get_active_usenet_adapter() + if adapter is None or not adapter.is_configured(): + result['error'] = 'No active usenet client' + return result + + def _emit(state: str, **extra) -> None: + if progress_callback: + try: + progress_callback({'state': state, **extra}) + except Exception as cb_exc: + logger.debug("[Usenet album] progress callback failed: %s", cb_exc) + + query = f"{artist_name} {album_name}".strip() + _emit('searching', query=query) + try: + search_results = run_async(self._prowlarr.search( + query, categories=DEFAULT_MUSIC_CATEGORIES, + indexer_ids=_parse_indexer_id_filter(), + )) + except Exception as e: + result['error'] = f'Prowlarr search failed: {e}' + return result + + candidates = [r for r in search_results + if r.protocol == 'usenet' and r.download_url] + if not candidates: + result['error'] = f'No usenet results found for "{query}"' + return result + + picked = _pick_best_album_release(candidates) + if picked is None: + result['error'] = 'No suitable NZB candidate after filtering' + return result + + logger.info("[Usenet album] Picked '%s' (size=%.1fMB grabs=%s indexer=%s)", + picked.title, picked.size / 1_048_576, picked.grabs, picked.indexer_name) + _emit('queued', release=picked.title, size=picked.size, grabs=picked.grabs) + + try: + job_id = run_async(adapter.add_nzb(picked.download_url)) + except Exception as e: + result['error'] = f'Usenet client refused the NZB: {e}' + return result + if not job_id: + result['error'] = 'Usenet client refused the NZB' + return result + + _emit('downloading', release=picked.title) + save_path = self._poll_album_download(adapter, job_id, picked.title, _emit) + if save_path is None: + result['error'] = 'Usenet download failed or timed out' + return result + + _emit('staging', release=picked.title) + try: + audio_files = collect_audio_after_extraction(Path(save_path)) + except Exception as e: + result['error'] = f'Failed to walk audio files: {e}' + return result + if not audio_files: + result['error'] = f'No audio files found in {save_path}' + return result + + staging_path = Path(staging_dir) + staging_path.mkdir(parents=True, exist_ok=True) + copied: List[str] = [] + for src in audio_files: + dst = _unique_staging_path(staging_path, src) + try: + shutil.copy2(src, dst) + copied.append(str(dst)) + except Exception as e: + logger.warning("[Usenet album] Failed to copy %s -> %s: %s", src, dst, e) + if not copied: + result['error'] = 'No audio files copied to staging' + return result + logger.info("[Usenet album] Staged %d audio files for '%s'", len(copied), album_name) + _emit('staged', count=len(copied)) + result['success'] = True + result['files'] = copied + return result + + def _poll_album_download(self, adapter, job_id, title, emit) -> Optional[str]: + deadline = time.monotonic() + _POLL_TIMEOUT_SECONDS + last_save_path: Optional[str] = None + while time.monotonic() < deadline: + if self.shutdown_check and self.shutdown_check(): + return None + try: + status = run_async(adapter.get_status(job_id)) + except Exception as e: + logger.warning("[Usenet album] Poll error: %s", e) + status = None + if status is None: + logger.error("[Usenet album] '%s' disappeared from client", title) + return None + emit('downloading', progress=status.progress, downloaded=status.downloaded, + speed=status.download_speed) + if status.save_path: + last_save_path = status.save_path + if status.state in _COMPLETE_STATES: + return last_save_path + if status.state == 'failed': + logger.error("[Usenet album] '%s' failed: %s", title, status.error) + return None + time.sleep(_POLL_INTERVAL_SECONDS) + logger.error("[Usenet album] '%s' timed out", title) + return None diff --git a/core/downloads/master.py b/core/downloads/master.py index 118db159..518797b9 100644 --- a/core/downloads/master.py +++ b/core/downloads/master.py @@ -311,6 +311,100 @@ def run_full_missing_tracks_process(batch_id, playlist_id, tracks_json, deps: Ma if force_download_all: logger.warning(f"[Force Download] Force download mode enabled for batch {batch_id} - treating all tracks as missing") + # ════════════════════════════════════════════════════════════════ + # ALBUM-BUNDLE GATE for torrent / usenet single-source mode. + # + # Indexer-based sources (torrent / usenet) are release-level — a + # Prowlarr search for "Luther (with SZA)" returns the GNX album + # torrent at near-zero confidence against the track title. The + # per-track search loop fails for every track on the album. + # + # Workaround: when the user is downloading an album AND has + # torrent / usenet selected as the SINGLE active source (not + # hybrid — hybrid stays per-track to preserve fallback to + # Soulseek / streaming sources), do ONE Prowlarr search for the + # whole album, hand the picked release to the torrent / usenet + # client, walk the resulting audio files, and drop them into the + # staging folder. Each per-track task then hits the existing + # ``try_staging_match`` early-return in the per-track worker + # before any Prowlarr search fires, and the normal post- + # processing pipeline imports each matched file. + # + # Gate intentionally narrow: hybrid mode, non-album batches, and + # plugins without ``download_album_to_staging`` (i.e. every + # source other than torrent / usenet) all bypass this branch + # untouched. + # ════════════════════════════════════════════════════════════════ + _album_bundle_mode = (deps.config_manager.get('download_source.mode', 'soulseek') or 'soulseek').lower() + _is_torrent_or_usenet = _album_bundle_mode in ('torrent', 'usenet') + if batch_is_album and _is_torrent_or_usenet and batch_album_context and batch_artist_context: + _bundle_album = (batch_album_context.get('name') or '').strip() + _bundle_artist = (batch_artist_context.get('name') or '').strip() + _bundle_plugin = None + try: + _bundle_plugin = deps.download_orchestrator.client(_album_bundle_mode) + except Exception as _exc: + logger.warning("[Album Bundle] Could not resolve %s plugin: %s", _album_bundle_mode, _exc) + if _bundle_album and _bundle_artist and _bundle_plugin and hasattr(_bundle_plugin, 'download_album_to_staging'): + _staging_dir = deps.config_manager.get('import.staging_path', './Staging') or './Staging' + logger.info( + "[Album Bundle] Engaging %s album flow for '%s' by '%s' -> %s", + _album_bundle_mode, _bundle_album, _bundle_artist, _staging_dir, + ) + with tasks_lock: + if batch_id in download_batches: + download_batches[batch_id]['phase'] = 'album_downloading' + download_batches[batch_id]['album_bundle_state'] = 'searching' + download_batches[batch_id]['album_bundle_source'] = _album_bundle_mode + + def _bundle_emit(payload): + """Mirror the album-download lifecycle into batch state so the + Downloads page can render meaningful status while the torrent / + usenet job runs (the per-track tasks don't exist yet).""" + try: + with tasks_lock: + if batch_id in download_batches: + _row = download_batches[batch_id] + _row['album_bundle_state'] = payload.get('state', '') + for _k in ('progress', 'release', 'speed', 'downloaded', 'size', 'seeders', 'grabs', 'count'): + if _k in payload: + _row[f'album_bundle_{_k}'] = payload[_k] + except Exception as _emit_exc: + logger.debug("[Album Bundle] emit failed: %s", _emit_exc) + + try: + _bundle_outcome = _bundle_plugin.download_album_to_staging( + _bundle_album, _bundle_artist, _staging_dir, _bundle_emit, + ) + except Exception as _bundle_exc: + logger.exception("[Album Bundle] %s plugin raised: %s", _album_bundle_mode, _bundle_exc) + _bundle_outcome = {'success': False, 'error': f'Plugin error: {_bundle_exc}'} + + if not _bundle_outcome.get('success'): + _err = _bundle_outcome.get('error', 'Album bundle download failed') + logger.error("[Album Bundle] %s flow failed for '%s': %s", _album_bundle_mode, _bundle_album, _err) + with tasks_lock: + if batch_id in download_batches: + download_batches[batch_id]['phase'] = 'failed' + download_batches[batch_id]['error'] = _err + download_batches[batch_id]['album_bundle_state'] = 'failed' + return + logger.info( + "[Album Bundle] %s staged %d files for '%s' — handing off to per-track staging matcher", + _album_bundle_mode, len(_bundle_outcome.get('files', [])), _bundle_album, + ) + with tasks_lock: + if batch_id in download_batches: + download_batches[batch_id]['phase'] = 'analysis' + download_batches[batch_id]['album_bundle_state'] = 'staged' + else: + logger.warning( + "[Album Bundle] Gate matched but plugin / context unavailable " + "(mode=%s album=%r artist=%r plugin=%s) — falling back to per-track flow", + _album_bundle_mode, _bundle_album, _bundle_artist, + type(_bundle_plugin).__name__ if _bundle_plugin else None, + ) + # Allow duplicate tracks across albums — when enabled, only skip tracks already # owned in THIS album, not tracks owned in other albums allow_duplicates = deps.config_manager.get('wishlist.allow_duplicate_tracks', True) diff --git a/tests/test_torrent_usenet_plugins.py b/tests/test_torrent_usenet_plugins.py index bf2f856b..f830a7ed 100644 --- a/tests/test_torrent_usenet_plugins.py +++ b/tests/test_torrent_usenet_plugins.py @@ -392,6 +392,63 @@ def test_plugins_conform_to_protocol() -> None: # --------------------------------------------------------------------------- +def test_torrent_album_pick_prefers_seeded_flac(tmp_path: Path) -> None: + """Album bundle picker prefers high-seeded FLAC over low-seeded MP3 + of comparable size — protects against picking a dead torrent.""" + from core.download_plugins.torrent import _pick_best_album_release + flac = _make_torrent_result(title='Kendrick Lamar - GNX [FLAC]', size=400_000_000, seeders=120) + mp3 = _make_torrent_result(title='Kendrick Lamar - GNX [MP3 320]', size=120_000_000, seeders=5, guid='guid-2') + picked = _pick_best_album_release([flac, mp3]) + assert picked is flac + + +def test_torrent_album_pick_drops_too_small() -> None: + """Single-track torrents (~10 MB) shouldn't be picked when the user + is downloading a whole album — the size floor (40 MB) catches them.""" + from core.download_plugins.torrent import _pick_best_album_release + single = _make_torrent_result(title='Kendrick Lamar - HUMBLE', size=10_000_000, seeders=500) + album = _make_torrent_result(title='Kendrick Lamar - DAMN [MP3]', size=120_000_000, seeders=50, guid='guid-2') + picked = _pick_best_album_release([single, album]) + assert picked is album + + +def test_torrent_album_pick_falls_back_when_all_outside_size_range() -> None: + """If every candidate is below the floor (e.g. all results are + singles), pick the most-seeded one rather than returning None — + user still wants a download even if it's a track torrent.""" + from core.download_plugins.torrent import _pick_best_album_release + small_a = _make_torrent_result(title='X [MP3]', size=8_000_000, seeders=5) + small_b = _make_torrent_result(title='Y [MP3]', size=9_000_000, seeders=80, guid='guid-2') + picked = _pick_best_album_release([small_a, small_b]) + assert picked is small_b + + +def test_unique_staging_path_handles_collision(tmp_path: Path) -> None: + from core.download_plugins.torrent import _unique_staging_path + src = tmp_path / 'src' / 'track.flac' + src.parent.mkdir() + src.write_bytes(b'fLaC') + dest_dir = tmp_path / 'staging' + dest_dir.mkdir() + # First call returns the natural name. + first = _unique_staging_path(dest_dir, src) + assert first == dest_dir / 'track.flac' + first.write_bytes(b'fLaC') + # Second call picks a non-colliding suffix. + second = _unique_staging_path(dest_dir, src) + assert second == dest_dir / 'track_1.flac' + + +def test_torrent_album_to_staging_short_circuits_when_not_configured() -> None: + """The gate must refuse to operate when Prowlarr isn't set up — + every later call would hit the network with empty creds.""" + plugin = TorrentDownloadPlugin() + with patch.object(plugin, 'is_configured', return_value=False): + outcome = plugin.download_album_to_staging('GNX', 'Kendrick Lamar', '/tmp/staging') + assert outcome['success'] is False + assert 'not configured' in outcome['error'].lower() + + def test_registry_includes_torrent_and_usenet() -> None: """The registry decides what shows up in the orchestrator's iteration helpers. If we forget to register a new plugin the diff --git a/webui/static/helper.js b/webui/static/helper.js index 8ae56bc0..080b0df7 100644 --- a/webui/static/helper.js +++ b/webui/static/helper.js @@ -3415,6 +3415,7 @@ function closeHelperSearch() { const WHATS_NEW = { '2.6.0': [ { unreleased: true }, + { title: 'Album-bundle flow for torrent / usenet downloads', desc: 'fixes the core architectural problem with indexer-based sources. Prowlarr returns release-level torrents — searching per-track for "Luther (with SZA)" against the GNX album torrent scores near-zero and the orchestrator rejects every candidate. New gated flow: when downloading an album AND torrent or usenet is the single active source (not hybrid), SoulSync now does ONE Prowlarr search for the whole release, picks the best torrent (prefers FLAC, high seeders, reasonable size — drops single-track torrents that snuck in), hands it to your torrent / usenet client, walks the resulting audio files (extracting .zip/.rar/.tar if needed), and drops them all into the staging folder. The existing per-track staging matcher then imports each one to the library by fuzzy title match — same path as the Auto-Import flow. Gate is strictly opt-in: per-track flow is completely untouched for hybrid mode, non-album downloads, and every other source. 5 new tests cover the album picker (seeded-FLAC preference, size floor for single-track torrents, fallback when all candidates are small) and the staging path collision handler.' }, { title: 'Filesystem-access heads-up for torrent / usenet sources', desc: 'new advisory card on the Indexers & Downloaders tab explaining the cleanest setup: point ALL your downloaders (Soulseek, qBittorrent, SABnzbd / NZBGet) at the same download folder. One folder, one mount, everything just works. Bare-metal needs no change; Docker users can reuse the existing ./downloads mount and just configure each client to write there. docker-compose.yml updated to call this out as the easiest path, with optional commented placeholders for users who prefer separate folders per protocol.' }, { title: 'Torrent and Usenet downloads', desc: 'two new download sources live in the Download Source dropdown: Torrent Only (via Prowlarr) and Usenet Only (via Prowlarr). they reuse the Prowlarr + torrent client + usenet client you set up on the Indexers & Downloaders tab. searches go through Prowlarr filtered by protocol, picked releases get handed to your torrent client or usenet client, and the resulting files get walked through archive_pipeline (extracts .zip / .rar / .tar when the client didn\'t already do it) and handed to the matching pipeline. both sources are also available in hybrid mode alongside soulseek / youtube / tidal / etc. one caveat: SoulSync needs read access to the torrent / usenet client\'s save_path — works out of the box for everything-on-one-box setups, but remote downloader hosts will need a future sync step.' }, { title: 'Archive pipeline module (groundwork for torrent / usenet downloads)', desc: 'new core/archive_pipeline.py — walks a directory for audio files (recursive, case-insensitive extensions), extracts zip / tar / tar.gz / rar / 7z archives in-place (rar and 7z are optional deps that warn but don\'t crash if absent), and rejects any archive member trying to escape the destination via path traversal. shared helper the upcoming torrent and usenet download plugins both consume — usenet downloaders usually auto-extract, but the occasional torrent ships an album in a .rar and SoulSync handles it now. 21 unit tests cover the walker + zip + tar extraction + path-traversal protection.' }, From daaed373e71209f9cfa9035a412053d2ea7916e1 Mon Sep 17 00:00:00 2001 From: Broque Thomas <26755000+Nezreka@users.noreply.github.com> Date: Wed, 20 May 2026 19:31:47 -0700 Subject: [PATCH 09/25] fix(provenance): label torrent/usenet/staging downloads correctly in history MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The download history modal was tagging every torrent / usenet album-bundle download as 'Soulseek FLAC 24bit' because: - core/imports/side_effects.py's source_service dict didn't have entries for 'staging', 'torrent', or 'usenet' usernames. The staging matcher in core/downloads/staging.py sets download_tasks[task_id]['username'] = 'staging', which fell through to the dict's default and got recorded as 'soulseek' in the track download provenance row. Same fate for any amazon or other source that wasn't whitelisted. - The album-bundle flow specifically wants to be labeled as 'torrent' or 'usenet' (where the bytes actually came from), not 'staging' (the intermediate). The plugin already stashes the source on the batch state as ``album_bundle_source`` for the Downloads-page status card; provenance recording can read the same field. Fixes: - core/downloads/staging.py: when marking a task post_processing after a staging match, check the batch's album_bundle_source override and use that for username instead of 'staging' when set. Falls back to 'staging' when no override exists (manual file-drop case). - core/imports/side_effects.py: source_service map gets entries for 'staging', 'torrent', 'usenet', and the previously-missing 'amazon' (which was also falling through to 'soulseek'). - webui/static/library.js: the redownload modal's serviceLabels / serviceIcons dicts extended to cover lidarr, amazon, soundcloud, auto_import, staging, torrent, usenet so badges render the correct name instead of either the raw source_service string or no badge at all. - webui/static/wishlist-tools.js: history-source-chip color palette extended for the new source labels (Torrent sky-blue, Usenet violet, Staging / Auto-Import neutral grey). Note: existing tracks in the DB still carry the wrong 'soulseek' label — only NEW downloads after this fix get the right label. A future migration could rewrite historical rows but it's cosmetic and the underlying audio + metadata are correct. --- core/downloads/staging.py | 16 ++++++++++++++-- core/imports/side_effects.py | 11 +++++++++++ webui/static/library.js | 12 ++++++------ webui/static/wishlist-tools.js | 2 +- 4 files changed, 32 insertions(+), 9 deletions(-) diff --git a/core/downloads/staging.py b/core/downloads/staging.py index 12de4a16..a8dda6c3 100644 --- a/core/downloads/staging.py +++ b/core/downloads/staging.py @@ -141,12 +141,24 @@ def try_staging_match(task_id, batch_id, track, deps: StagingDeps): shutil.copy2(best_match['full_path'], dest_path) logger.info(f"[Staging] Copied to transfer: {dest_path}") - # Mark task as completed with staging context + # Mark task as completed with staging context. + # If the batch was populated by the torrent / usenet album-bundle + # flow, prefer that provenance label over generic 'staging' so the + # download history reflects the real source. + _provenance_override = None + try: + from core.runtime_state import download_batches as _db + _batch = _db.get(batch_id) if batch_id else None + if isinstance(_batch, dict): + _provenance_override = _batch.get('album_bundle_source') + except Exception: + _provenance_override = None + _provenance_username = _provenance_override or 'staging' with tasks_lock: if task_id in download_tasks: download_tasks[task_id]['status'] = 'post_processing' download_tasks[task_id]['filename'] = dest_path - download_tasks[task_id]['username'] = 'staging' + download_tasks[task_id]['username'] = _provenance_username download_tasks[task_id]['staging_match'] = True # Run post-processing (tagging, AcoustID verification, path building) diff --git a/core/imports/side_effects.py b/core/imports/side_effects.py index 9b3d9f3a..d1580564 100644 --- a/core/imports/side_effects.py +++ b/core/imports/side_effects.py @@ -276,6 +276,7 @@ def record_download_provenance(context: Dict[str, Any]) -> None: "deezer_dl": "deezer", "lidarr": "lidarr", "soundcloud": "soundcloud", + "amazon": "amazon", # Auto-import: surfaced in provenance so the redownload modal # can tell the user "this came from staging on " instead # of falsely listing soulseek as the source. The underlying @@ -283,6 +284,16 @@ def record_download_provenance(context: Dict[str, Any]) -> None: # separately via the source-aware ID columns on the tracks # row itself. "auto_import": "auto_import", + # Generic staging-match (user dropped files manually OR a + # source we don't have a more specific label for). Better + # than defaulting to 'soulseek' which would falsely tag the + # provenance. + "staging": "staging", + # Torrent / usenet album-bundle flow — the staging matcher + # overrides 'staging' with the bundle source so the history + # shows where the files actually came from. + "torrent": "torrent", + "usenet": "usenet", }.get(username, "soulseek") ti = context.get("track_info") or context.get("search_result") or {} diff --git a/webui/static/library.js b/webui/static/library.js index f8ed0960..cbbcb74e 100644 --- a/webui/static/library.js +++ b/webui/static/library.js @@ -4595,8 +4595,8 @@ async function showTrackSourceInfo(track, anchorEl) { return; } - const serviceIcons = { soulseek: '🔍', youtube: '▶️', tidal: '🌊', qobuz: '🎵', hifi: '🎧', deezer: '💜' }; - const serviceLabels = { soulseek: 'Soulseek', youtube: 'YouTube', tidal: 'Tidal', qobuz: 'Qobuz', hifi: 'HiFi', deezer: 'Deezer' }; + const serviceIcons = { soulseek: '🔍', youtube: '▶️', tidal: '🌊', qobuz: '🎵', hifi: '🎧', deezer: '💜', lidarr: '📦', amazon: '🛒', soundcloud: '☁️', auto_import: '📥', staging: '📥', torrent: '🧲', usenet: '📰' }; + const serviceLabels = { soulseek: 'Soulseek', youtube: 'YouTube', tidal: 'Tidal', qobuz: 'Qobuz', hifi: 'HiFi', deezer: 'Deezer', lidarr: 'Lidarr', amazon: 'Amazon Music', soundcloud: 'SoundCloud', auto_import: 'Auto-Import', staging: 'Staging', torrent: 'Torrent', usenet: 'Usenet' }; const dl = data.downloads[0]; // Most recent download const icon = serviceIcons[dl.source_service] || '📦'; @@ -4926,8 +4926,8 @@ async function _streamRedownloadSources(overlay, track, metadata) { const startBtn = document.getElementById('redownload-start-btn'); if (!columnsEl) return; - const serviceIcons = { soulseek: '🔍', youtube: '▶️', tidal: '🌊', qobuz: '🎵', hifi: '🎧', deezer_dl: '💜', hybrid: '⚡' }; - const serviceLabels = { soulseek: 'Soulseek', youtube: 'YouTube', tidal: 'Tidal', qobuz: 'Qobuz', hifi: 'HiFi', deezer_dl: 'Deezer', hybrid: 'Auto' }; + const serviceIcons = { soulseek: '🔍', youtube: '▶️', tidal: '🌊', qobuz: '🎵', hifi: '🎧', deezer_dl: '💜', hybrid: '⚡', lidarr: '📦', amazon: '🛒', soundcloud: '☁️', torrent: '🧲', usenet: '📰' }; + const serviceLabels = { soulseek: 'Soulseek', youtube: 'YouTube', tidal: 'Tidal', qobuz: 'Qobuz', hifi: 'HiFi', deezer_dl: 'Deezer', hybrid: 'Auto', lidarr: 'Lidarr', amazon: 'Amazon Music', soundcloud: 'SoundCloud', torrent: 'Torrent', usenet: 'Usenet' }; let allCandidates = []; let firstResult = true; @@ -5049,8 +5049,8 @@ async function _streamRedownloadSources(overlay, track, metadata) { /* _renderRedownloadStep2 removed — replaced by _streamRedownloadSources above */ if (false) { - const serviceIcons = { soulseek: '🔍', youtube: '▶️', tidal: '🌊', qobuz: '🎵', hifi: '🎧', deezer_dl: '💜', hybrid: '⚡' }; - const serviceLabels = { soulseek: 'Soulseek', youtube: 'YouTube', tidal: 'Tidal', qobuz: 'Qobuz', hifi: 'HiFi', deezer_dl: 'Deezer', hybrid: 'Auto' }; + const serviceIcons = { soulseek: '🔍', youtube: '▶️', tidal: '🌊', qobuz: '🎵', hifi: '🎧', deezer_dl: '💜', hybrid: '⚡', lidarr: '📦', amazon: '🛒', soundcloud: '☁️', torrent: '🧲', usenet: '📰' }; + const serviceLabels = { soulseek: 'Soulseek', youtube: 'YouTube', tidal: 'Tidal', qobuz: 'Qobuz', hifi: 'HiFi', deezer_dl: 'Deezer', hybrid: 'Auto', lidarr: 'Lidarr', amazon: 'Amazon Music', soundcloud: 'SoundCloud', torrent: 'Torrent', usenet: 'Usenet' }; // Group candidates by source service const grouped = {}; diff --git a/webui/static/wishlist-tools.js b/webui/static/wishlist-tools.js index 1a50bf19..3807102a 100644 --- a/webui/static/wishlist-tools.js +++ b/webui/static/wishlist-tools.js @@ -3406,7 +3406,7 @@ async function loadLibraryHistory() { const sc = data.stats?.source_counts || {}; const srcEntries = Object.entries(sc).sort((a, b) => b[1] - a[1]); if (srcEntries.length > 0 && tab === 'download') { - const _srcColors = { Soulseek: '#4caf50', Tidal: '#000', YouTube: '#ff0000', Qobuz: '#4285f4', HiFi: '#00bcd4', Deezer: '#a238ff' }; + const _srcColors = { Soulseek: '#4caf50', Tidal: '#000', YouTube: '#ff0000', Qobuz: '#4285f4', HiFi: '#00bcd4', Deezer: '#a238ff', Lidarr: '#5dade2', Amazon: '#ff9900', SoundCloud: '#ff7700', Torrent: '#5dade2', Usenet: '#a78bfa', Staging: '#888', 'Auto-Import': '#888' }; sourceBar.innerHTML = srcEntries.map(([src, cnt]) => `${src}: ${cnt}` ).join(''); From 8975031e3a83b659b135dabfa7c1c6edf9efffc2 Mon Sep 17 00:00:00 2001 From: Broque Thomas <26755000+Nezreka@users.noreply.github.com> Date: Wed, 20 May 2026 20:16:14 -0700 Subject: [PATCH 10/25] fix(downloads): skip torrent/usenet in hybrid chain for album batches MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When a user picks Hybrid mode AND downloads an album, the per-track search loop fires once per track. Torrent / usenet are release-level sources — Prowlarr returns album torrents, none of which score meaningfully against an individual track title. Without filtering, every track triggered a redundant Prowlarr search, qBit rejected duplicate hashes after the first, and the run only worked at all because Auto-Import swept Staging behind the scenes. Confusing logs, wasted searches, brittle timing. Fix: thread an optional ``exclude_sources`` parameter through ``DownloadOrchestrator.search``. When the per-track worker detects that the active batch is an album AND mode is hybrid, it passes ``['torrent', 'usenet']`` so the hybrid chain skips them and falls through to per-track-compatible sources (Soulseek / streaming). Gate is narrow on purpose: - Hybrid + album → skip torrent / usenet (THIS fix) - Single-source torrent / usenet + album → album-bundle flow on the master worker (already shipped) - Hybrid + single-track batch (basic search / wishlist / playlist of singles) → torrent / usenet still tried, validation.py's album-name fallback gives them a shot Excluded list logged at INFO when applied so the behavior is visible in logs ("Hybrid search: excluding ['torrent', 'usenet'] for this query"). Default ``exclude_sources=None`` keeps every non-task-worker caller (basic search, stream search, search-and- download-best, automation handlers) on the original code path. --- core/download_orchestrator.py | 31 +++++++++++++++++++++++++++++-- core/downloads/task_worker.py | 24 +++++++++++++++++++++++- webui/static/helper.js | 1 + 3 files changed, 53 insertions(+), 3 deletions(-) diff --git a/core/download_orchestrator.py b/core/download_orchestrator.py index 480ce7d6..879d0f52 100644 --- a/core/download_orchestrator.py +++ b/core/download_orchestrator.py @@ -286,11 +286,25 @@ class DownloadOrchestrator: chain = ['soulseek'] return chain - async def search(self, query: str, timeout: int = None, progress_callback=None) -> Tuple[List[TrackResult], List[AlbumResult]]: + async def search(self, query: str, timeout: int = None, progress_callback=None, + exclude_sources=None) -> Tuple[List[TrackResult], List[AlbumResult]]: """Search for tracks using configured source(s). Single-source modes route directly; hybrid mode delegates to - ``engine.search_with_fallback`` which tries the chain in order.""" + ``engine.search_with_fallback`` which tries the chain in order. + + ``exclude_sources`` (optional) is an iterable of source names + the caller wants filtered out of the hybrid chain. Used by the + per-track download worker to skip torrent / usenet for album- + context batches — those sources are release-level and don't + score meaningfully on per-track titles; the album-bundle flow + on the master worker handles them separately when they're the + single active source.""" if self.mode != 'hybrid': + # Single-source mode is opt-in; honour the user's choice even + # if it's torrent/usenet on an album batch (the master worker + # routes those through the album-bundle flow before per-track + # tasks ever fire, so search() being called here would be a + # non-album / wishlist / basic-search use case). client = self._client(self.mode) if not client: logger.error(f"{self.registry.display_name(self.mode)} client not available (failed to initialize)") @@ -299,6 +313,19 @@ class DownloadOrchestrator: return await client.search(query, timeout, progress_callback) chain = self._resolve_source_chain() + if exclude_sources: + blocked = {s.lower() for s in exclude_sources if s} + filtered = [s for s in chain if s.lower() not in blocked] + if filtered != chain: + logger.info( + "Hybrid search: excluding %s for this query (chain %s -> %s)", + sorted(blocked & {s.lower() for s in chain}), + " → ".join(chain), " → ".join(filtered) if filtered else "(empty)", + ) + chain = filtered + if not chain: + logger.warning("Hybrid search exhausted: no eligible sources after exclusion filter") + return [], [] logger.info(f"Hybrid search ({' → '.join(chain)}): {query}") return await self.engine.search_with_fallback(query, chain, timeout, progress_callback) diff --git a/core/downloads/task_worker.py b/core/downloads/task_worker.py index c5d79d82..8fc42608 100644 --- a/core/downloads/task_worker.py +++ b/core/downloads/task_worker.py @@ -209,8 +209,30 @@ def download_track_worker(task_id: str, batch_id: Optional[str], deps: TaskWorke logger.debug(f"About to call soulseek search for task {task_id}") try: + # Hybrid + album-context batches must skip torrent / usenet during + # the per-track loop — they're release-level sources, can't match + # individual tracks meaningfully, and album-bundle handling only + # fires in single-source mode (see core/downloads/master.py). The + # exclusion lets the hybrid chain fall through to per-track- + # compatible sources (soulseek / streaming) instead of attempting + # N redundant Prowlarr searches that all download the same album + # torrent and rely on the auto-import sweep to clean up. + _exclude_for_hybrid_album = None + try: + _batch_is_album = False + if batch_id: + from core.runtime_state import download_batches as _db + _b = _db.get(batch_id) + if isinstance(_b, dict): + _batch_is_album = bool(_b.get('is_album_download')) + if _batch_is_album and getattr(deps.download_orchestrator, 'mode', '') == 'hybrid': + _exclude_for_hybrid_album = ['torrent', 'usenet'] + except Exception as _exc_filter_err: + logger.debug("[Modal Worker] album-source-exclusion check failed: %s", _exc_filter_err) # Perform search with timeout - tracks_result, _ = deps.run_async(deps.download_orchestrator.search(query, timeout=30)) + tracks_result, _ = deps.run_async(deps.download_orchestrator.search( + query, timeout=30, exclude_sources=_exclude_for_hybrid_album, + )) logger.debug(f"Search completed for task {task_id}, got {len(tracks_result) if tracks_result else 0} results") # CRITICAL: Check cancellation immediately after search returns diff --git a/webui/static/helper.js b/webui/static/helper.js index 080b0df7..9b184462 100644 --- a/webui/static/helper.js +++ b/webui/static/helper.js @@ -3415,6 +3415,7 @@ function closeHelperSearch() { const WHATS_NEW = { '2.6.0': [ { unreleased: true }, + { title: 'Hybrid mode skips torrent / usenet on album batches', desc: 'follow-up to the album-bundle flow. When download mode is set to Hybrid AND the batch is an album, the per-track search loop now silently strips torrent and usenet from the source chain — they\'re release-level sources that can\'t match per-track meaningfully, and the album-bundle handling only fires in single-source mode. Without the skip, hybrid + torrent-first would fire N redundant Prowlarr searches per album and rely on Auto-Import sweeping Staging to recover. Cleaner now: hybrid falls straight through to per-track-compatible sources (Soulseek / streaming) for albums, and torrent / usenet still get a shot for single-track / wishlist / basic-search use cases where per-track makes sense.' }, { title: 'Album-bundle flow for torrent / usenet downloads', desc: 'fixes the core architectural problem with indexer-based sources. Prowlarr returns release-level torrents — searching per-track for "Luther (with SZA)" against the GNX album torrent scores near-zero and the orchestrator rejects every candidate. New gated flow: when downloading an album AND torrent or usenet is the single active source (not hybrid), SoulSync now does ONE Prowlarr search for the whole release, picks the best torrent (prefers FLAC, high seeders, reasonable size — drops single-track torrents that snuck in), hands it to your torrent / usenet client, walks the resulting audio files (extracting .zip/.rar/.tar if needed), and drops them all into the staging folder. The existing per-track staging matcher then imports each one to the library by fuzzy title match — same path as the Auto-Import flow. Gate is strictly opt-in: per-track flow is completely untouched for hybrid mode, non-album downloads, and every other source. 5 new tests cover the album picker (seeded-FLAC preference, size floor for single-track torrents, fallback when all candidates are small) and the staging path collision handler.' }, { title: 'Filesystem-access heads-up for torrent / usenet sources', desc: 'new advisory card on the Indexers & Downloaders tab explaining the cleanest setup: point ALL your downloaders (Soulseek, qBittorrent, SABnzbd / NZBGet) at the same download folder. One folder, one mount, everything just works. Bare-metal needs no change; Docker users can reuse the existing ./downloads mount and just configure each client to write there. docker-compose.yml updated to call this out as the easiest path, with optional commented placeholders for users who prefer separate folders per protocol.' }, { title: 'Torrent and Usenet downloads', desc: 'two new download sources live in the Download Source dropdown: Torrent Only (via Prowlarr) and Usenet Only (via Prowlarr). they reuse the Prowlarr + torrent client + usenet client you set up on the Indexers & Downloaders tab. searches go through Prowlarr filtered by protocol, picked releases get handed to your torrent client or usenet client, and the resulting files get walked through archive_pipeline (extracts .zip / .rar / .tar when the client didn\'t already do it) and handed to the matching pipeline. both sources are also available in hybrid mode alongside soulseek / youtube / tidal / etc. one caveat: SoulSync needs read access to the torrent / usenet client\'s save_path — works out of the box for everything-on-one-box setups, but remote downloader hosts will need a future sync step.' }, From 670a2db95e69ecde1333447b9aec36b64d454414 Mon Sep 17 00:00:00 2001 From: Broque Thomas <26755000+Nezreka@users.noreply.github.com> Date: Wed, 20 May 2026 20:26:30 -0700 Subject: [PATCH 11/25] refactor(downloads): extract album_bundle shared helpers + atomic copy MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Per code review: the album-bundle helpers (release picker + staging collision suffix) were defined as private symbols in torrent.py and imported by usenet.py through ``from core.download_plugins.torrent import _pick_best_album_release, _unique_staging_path``. Sibling plugins shouldn't reach into each other's private surface — leaky module boundary, and the underscore prefix says don't import. Also addressed two latent issues at the same time: - The Auto-Import sweep race: my plugin copied audio files into staging via plain ``shutil.copy2``, which exposes a partial file at the audio extension for the duration of the copy. The Auto- Import worker filters by audio extension when scanning Staging (AUDIO_EXTENSIONS in core/auto_import_worker.py), so a mid-flight scan could pick up a truncated file. Fix: copy to a ``.tmp.`` sidecar first, then atomically rename via ``Path.replace`` (which is ``os.replace`` — atomic on the same filesystem). Auto-Import sees the file either at its final name or not at all. - The 6-hour poll timeout was a hard-coded magic constant. Users with slow private trackers or large box sets would silently time out after 6h. Both the timeout and the poll interval are now read from config (``download_source.album_bundle_timeout_seconds`` / ``..._poll_interval_seconds``) with safe fallback to the existing defaults when unset / non-numeric. - core/download_plugins/album_bundle.py: new module owns the shared surface — ``pick_best_album_release`` (with quality_guess passed in as a parameter to avoid the circular import that would result from this module trying to know about torrent.py's title parser), ``unique_staging_path``, ``atomic_copy_to_staging``, ``copy_audio_files_atomically``, ``get_poll_interval``, ``get_poll_timeout``. Module-level size constants and quality weights live here too. Usenet's grabs-as-popularity-proxy is built into the picker so both plugins get the right behavior without divergent local logic. - core/download_plugins/torrent.py: drops the local helpers + the hard-coded poll constants, imports from album_bundle. Per-track download flow still uses module-level ``_POLL_TIMEOUT_SECONDS`` / ``_POLL_INTERVAL_SECONDS`` aliases (read from config once at import time, same as before from a per-track perspective). - core/download_plugins/usenet.py: drops the imports of the torrent.py private helpers; everything goes through album_bundle now. Stops the cross-plugin private-import leak that started this whole refactor. - tests/test_album_bundle.py: 23 new tests covering the picker heuristic (empty input, singleton drop, FLAC preference, grabs fallback for usenet, size-floor / ceiling boundaries), the collision-suffix logic, the atomic-copy invariant (concurrent scanner thread asserts it never observes a partial audio file during five sequential copies), the failure-skip behavior of the batch copier, and the config-driven poll cadence including garbage-input fallback. - tests/test_torrent_usenet_plugins.py: existing picker tests updated to call the new module-level helpers instead of the former torrent.py privates. --- core/download_plugins/album_bundle.py | 217 +++++++++++++++++++ core/download_plugins/torrent.py | 86 ++------ core/download_plugins/usenet.py | 20 +- tests/test_album_bundle.py | 301 ++++++++++++++++++++++++++ tests/test_torrent_usenet_plugins.py | 21 +- 5 files changed, 551 insertions(+), 94 deletions(-) create mode 100644 core/download_plugins/album_bundle.py create mode 100644 tests/test_album_bundle.py diff --git a/core/download_plugins/album_bundle.py b/core/download_plugins/album_bundle.py new file mode 100644 index 00000000..6a0fb246 --- /dev/null +++ b/core/download_plugins/album_bundle.py @@ -0,0 +1,217 @@ +"""Shared helpers for the album-bundle download flow. + +The torrent and usenet download plugins both implement a +``download_album_to_staging`` method that searches Prowlarr for a +whole release, hands it to the active downloader, walks the +resulting audio files, and copies them into the staging folder. The +two implementations share the same release-picker heuristic and the +same staging-path collision logic. + +Pulled out of ``core/download_plugins/torrent.py`` so the usenet +plugin doesn't have to import private helpers from a sibling +plugin (Cin's "no leaky module boundaries" standard). + +Also exposes ``atomic_copy_to_staging`` — the audio file is copied +to a ``.tmp.`` sidecar first and atomically renamed onto its +final extension. The Auto-Import worker filters by audio extension +so the in-flight ``.tmp`` file is never picked up mid-copy, closing +the race between the album-bundle copy loop and Auto-Import's +folder scan. +""" + +from __future__ import annotations + +import shutil +import time +import uuid +from pathlib import Path +from typing import Iterable, Optional + +from config.settings import config_manager +from utils.logging_config import get_logger + +logger = get_logger("download_plugins.album_bundle") + + +# Album-pick size floor / ceiling. Single-track torrents (~10 MB) +# are rejected when bigger candidates exist; anything past 3 GB is +# treated as suspicious (multi-disc box-set + scans + extras). +ALBUM_PICK_MIN_BYTES = 40 * 1024 * 1024 +ALBUM_PICK_MAX_BYTES = 3 * 1024 * 1024 * 1024 + + +# Quality-score weights for the album-pick heuristic. Mirrors the +# tier order in ``core/imports/file_ops.py``'s ``quality_tiers`` — +# higher number = preferred. +_QUALITY_SCORE = {'flac': 4, 'ogg': 3, 'aac': 2, 'mp3': 1} + + +# Default poll cadence + timeout for the album-download poll loop. +# Both are overridable through config so users with slow trackers +# / large box-sets can extend the deadline without editing code. +DEFAULT_POLL_INTERVAL_SECONDS = 2.0 +DEFAULT_POLL_TIMEOUT_SECONDS = 6 * 60 * 60 + + +def get_poll_interval() -> float: + """Return the per-poll sleep duration (seconds). Configurable via + ``download_source.album_bundle_poll_interval_seconds``.""" + raw = config_manager.get('download_source.album_bundle_poll_interval_seconds', + DEFAULT_POLL_INTERVAL_SECONDS) + try: + value = float(raw) + if value > 0: + return value + except (TypeError, ValueError): + pass + return DEFAULT_POLL_INTERVAL_SECONDS + + +def get_poll_timeout() -> float: + """Return the total deadline for an album-bundle download + (seconds). Configurable via + ``download_source.album_bundle_timeout_seconds``.""" + raw = config_manager.get('download_source.album_bundle_timeout_seconds', + DEFAULT_POLL_TIMEOUT_SECONDS) + try: + value = float(raw) + if value > 0: + return value + except (TypeError, ValueError): + pass + return DEFAULT_POLL_TIMEOUT_SECONDS + + +def quality_score(title: str, quality_guess) -> int: + """Map a release title's inferred quality to a sortable integer. + + ``quality_guess`` is the function from each plugin that maps a + title string to a quality string ('flac' / 'mp3' / etc.) — passed + in so this module doesn't have to import either plugin and risk + a circular import.""" + return _QUALITY_SCORE.get(quality_guess(title) or '', 0) + + +def pick_best_album_release(candidates, quality_guess) -> Optional[object]: + """Pick the single best torrent / NZB for an album-bundle download. + + Heuristic, in priority order: + 1. Reasonable album-ish size (40 MB – 3 GB) — drops single-track + releases that snuck in and quarantines suspicious giants. + 2. Higher seeders > lower (dead torrents = dead downloads). + Usenet releases use ``grabs`` as a popularity proxy when + seeders is None. + 3. Higher quality (FLAC > AAC > MP3) inferred from title. + 4. Larger size as tiebreaker (often = higher bitrate). + """ + if not candidates: + return None + sized = [c for c in candidates + if ALBUM_PICK_MIN_BYTES <= (c.size or 0) <= ALBUM_PICK_MAX_BYTES] + pool = sized or list(candidates) + if not pool: + return None + + def _score(c) -> tuple: + seeders = c.seeders if c.seeders is not None else (c.grabs or 0) + return (seeders, quality_score(c.title or '', quality_guess), c.size or 0) + + return max(pool, key=_score) + + +def unique_staging_path(staging_dir: Path, src: Path) -> Path: + """Return a destination path inside ``staging_dir`` that doesn't + collide with an existing file. Appends ``_1``, ``_2``, ... before + the extension when needed; gives up after 1000 candidates and + returns the unsuffixed path so the caller will overwrite (better + than infinite loop or crash).""" + dest = staging_dir / src.name + if not dest.exists(): + return dest + stem = dest.stem + suffix = dest.suffix + for i in range(1, 1000): + candidate = staging_dir / f"{stem}_{i}{suffix}" + if not candidate.exists(): + return candidate + return dest + + +def atomic_copy_to_staging(src: Path, dest: Path) -> bool: + """Copy ``src`` to ``dest`` without exposing a partial file to + folder scanners. + + The Auto-Import worker filters by audio extension when scanning + Staging — see ``AUDIO_EXTENSIONS`` in ``core/auto_import_worker.py``. + Naming the in-flight file ``.tmp.`` keeps it + invisible until the rename atomically swings it to its final + extension. ``os.replace`` (used by ``Path.rename`` on Python 3.x) + is atomic on the same filesystem, so Auto-Import either sees the + file at its final name (complete) or doesn't see it at all + (in flight). + + Returns True on success, False on copy / rename failure. Caller + is expected to log the failure case so we don't double-log here. + """ + tmp = dest.with_name(f"{dest.name}.tmp.{uuid.uuid4().hex[:8]}") + try: + shutil.copy2(src, tmp) + except Exception: + # Best-effort cleanup of the partial file. If unlink fails + # (locked, permissions) we leave it — Auto-Import ignores it + # anyway because of the .tmp extension. + try: + if tmp.exists(): + tmp.unlink() + except Exception as cleanup_exc: + logger.debug("album_bundle tmp cleanup failed: %s", cleanup_exc) + raise + try: + tmp.replace(dest) + return True + except Exception: + try: + tmp.unlink(missing_ok=True) + except Exception as cleanup_exc: + logger.debug("album_bundle tmp cleanup failed: %s", cleanup_exc) + raise + + +def copy_audio_files_atomically( + sources: Iterable[Path], staging_dir: Path, +) -> list: + """Convenience wrapper: pick a non-colliding staging path for + each source, copy via ``atomic_copy_to_staging``. Returns the + list of final destination paths (as strings). Files that fail + to copy are logged and skipped; the caller decides what to do + with a partial result.""" + staging_dir.mkdir(parents=True, exist_ok=True) + out: list = [] + for src in sources: + dest = unique_staging_path(staging_dir, src) + try: + atomic_copy_to_staging(src, dest) + out.append(str(dest)) + except Exception as e: + logger.warning("[album_bundle] Failed to stage %s -> %s: %s", src, dest, e) + return out + + +# Re-export so callers don't have to remember which module owns +# what. The ``time`` import is kept so plugins can ``from +# core.download_plugins.album_bundle import time`` if they want to, +# avoiding a second std-lib import line for a single use. +__all__ = [ + "ALBUM_PICK_MIN_BYTES", + "ALBUM_PICK_MAX_BYTES", + "DEFAULT_POLL_INTERVAL_SECONDS", + "DEFAULT_POLL_TIMEOUT_SECONDS", + "atomic_copy_to_staging", + "copy_audio_files_atomically", + "get_poll_interval", + "get_poll_timeout", + "pick_best_album_release", + "quality_score", + "time", + "unique_staging_path", +] diff --git a/core/download_plugins/torrent.py b/core/download_plugins/torrent.py index faa38c89..b2e1187e 100644 --- a/core/download_plugins/torrent.py +++ b/core/download_plugins/torrent.py @@ -49,7 +49,6 @@ from __future__ import annotations import asyncio import re -import shutil import threading import time import uuid @@ -58,6 +57,12 @@ from typing import Any, Dict, List, Optional, Tuple from config.settings import config_manager from core.archive_pipeline import collect_audio_after_extraction +from core.download_plugins.album_bundle import ( + copy_audio_files_atomically, + get_poll_interval, + get_poll_timeout, + pick_best_album_release, +) from core.download_plugins.base import DownloadSourcePlugin from core.download_plugins.types import AlbumResult, DownloadStatus, TrackResult from core.prowlarr_client import ( @@ -83,14 +88,13 @@ _FILENAME_SEP = '||' # don't want to keep sharing. _COMPLETE_STATES = frozenset(['seeding', 'completed']) -# Max seconds the poll thread keeps watching one download before -# giving up. 6 hours covers slow private trackers without leaking -# threads forever on dead torrents. -_POLL_TIMEOUT_SECONDS = 6 * 60 * 60 - -# Poll cadence — torrent state changes slowly; no point hammering -# the WebUI more than once a second. -_POLL_INTERVAL_SECONDS = 2.0 +# Poll cadence / timeout — both pull from config via the shared +# album_bundle helpers so users can extend the deadline for slow +# trackers without editing source. Kept as module aliases so the +# per-track flow at the bottom of this file can still import them +# under the legacy names without re-reading config every loop. +_POLL_TIMEOUT_SECONDS = get_poll_timeout() +_POLL_INTERVAL_SECONDS = get_poll_interval() class TorrentDownloadPlugin(DownloadSourcePlugin): @@ -466,7 +470,7 @@ class TorrentDownloadPlugin(DownloadSourcePlugin): result['error'] = f'No torrent results found for "{query}"' return result - picked = _pick_best_album_release(candidates) + picked = pick_best_album_release(candidates, _guess_quality_from_title) if picked is None: result['error'] = 'No suitable torrent candidate after filtering' return result @@ -504,16 +508,7 @@ class TorrentDownloadPlugin(DownloadSourcePlugin): result['error'] = f'No audio files found in {save_path}' return result - staging_path = Path(staging_dir) - staging_path.mkdir(parents=True, exist_ok=True) - copied: List[str] = [] - for src in audio_files: - dst = _unique_staging_path(staging_path, src) - try: - shutil.copy2(src, dst) - copied.append(str(dst)) - except Exception as e: - logger.warning("[Torrent album] Failed to copy %s -> %s: %s", src, dst, e) + copied = copy_audio_files_atomically(audio_files, Path(staging_dir)) if not copied: result['error'] = 'No audio files copied to staging' return result @@ -553,57 +548,6 @@ class TorrentDownloadPlugin(DownloadSourcePlugin): return None -# --------------------------------------------------------------------------- -# Album-pick helpers -# --------------------------------------------------------------------------- - - -_QUALITY_SCORE = {'flac': 4, 'ogg': 3, 'aac': 2, 'mp3': 1} - - -def _pick_best_album_release(candidates) -> Optional[Any]: - """Pick the single best torrent for an album-bundle download. - - Heuristic, in priority order: - 1. Reasonable album-ish size (40 MB – 3 GB) — drops single-track - releases that snuck in and quarantines suspicious giants. - 2. Higher seeders > lower (dead torrents = dead downloads). - 3. Higher quality (FLAC > AAC > MP3) inferred from title. - 4. Larger size as tiebreaker (often = higher bitrate). - """ - MIN_BYTES = 40 * 1024 * 1024 - MAX_BYTES = 3 * 1024 * 1024 * 1024 - - sized = [c for c in candidates if MIN_BYTES <= (c.size or 0) <= MAX_BYTES] - pool = sized or candidates - if not pool: - return None - - def _score(c) -> tuple: - seeders = c.seeders or 0 - quality = _QUALITY_SCORE.get(_guess_quality_from_title(c.title), 0) - size = c.size or 0 - return (seeders, quality, size) - - return max(pool, key=_score) - - -def _unique_staging_path(staging_dir: Path, src: Path) -> Path: - """Return a destination path inside ``staging_dir`` that doesn't - collide with an existing file. Appends ``_1``, ``_2``, etc. before - the extension when needed.""" - dest = staging_dir / src.name - if not dest.exists(): - return dest - stem = dest.stem - suffix = dest.suffix - for i in range(1, 1000): - candidate = staging_dir / f"{stem}_{i}{suffix}" - if not candidate.exists(): - return candidate - return dest # give up — overwrite - - # --------------------------------------------------------------------------- # Module-level helpers (pure functions — easy to unit-test) # --------------------------------------------------------------------------- diff --git a/core/download_plugins/usenet.py b/core/download_plugins/usenet.py index 92792305..ea0f4397 100644 --- a/core/download_plugins/usenet.py +++ b/core/download_plugins/usenet.py @@ -14,7 +14,6 @@ module's docstring for the full pipeline rationale). Differences: from __future__ import annotations -import shutil import threading import time import uuid @@ -22,6 +21,10 @@ from pathlib import Path from typing import Any, Dict, List, Optional, Tuple from core.archive_pipeline import collect_audio_after_extraction +from core.download_plugins.album_bundle import ( + copy_audio_files_atomically, + pick_best_album_release, +) from core.download_plugins.base import DownloadSourcePlugin from core.download_plugins.torrent import ( _adapter_state_to_display, @@ -29,9 +32,7 @@ from core.download_plugins.torrent import ( _guess_quality_from_title, _parse_indexer_id_filter, _parse_release_title, - _pick_best_album_release, _row_to_status, - _unique_staging_path, _COMPLETE_STATES, _FILENAME_SEP, _POLL_INTERVAL_SECONDS, @@ -386,7 +387,7 @@ class UsenetDownloadPlugin(DownloadSourcePlugin): result['error'] = f'No usenet results found for "{query}"' return result - picked = _pick_best_album_release(candidates) + picked = pick_best_album_release(candidates, _guess_quality_from_title) if picked is None: result['error'] = 'No suitable NZB candidate after filtering' return result @@ -420,16 +421,7 @@ class UsenetDownloadPlugin(DownloadSourcePlugin): result['error'] = f'No audio files found in {save_path}' return result - staging_path = Path(staging_dir) - staging_path.mkdir(parents=True, exist_ok=True) - copied: List[str] = [] - for src in audio_files: - dst = _unique_staging_path(staging_path, src) - try: - shutil.copy2(src, dst) - copied.append(str(dst)) - except Exception as e: - logger.warning("[Usenet album] Failed to copy %s -> %s: %s", src, dst, e) + copied = copy_audio_files_atomically(audio_files, Path(staging_dir)) if not copied: result['error'] = 'No audio files copied to staging' return result diff --git a/tests/test_album_bundle.py b/tests/test_album_bundle.py new file mode 100644 index 00000000..48733c31 --- /dev/null +++ b/tests/test_album_bundle.py @@ -0,0 +1,301 @@ +"""Tests for ``core/download_plugins/album_bundle.py``. + +The shared helpers used by both the torrent and usenet album-bundle +flows. Pins the pick heuristic, the atomic-copy invariant +(no partial files ever visible at the audio extension), the +collision-suffix logic, and the config-driven poll cadence so a +future tweak in either plugin can't break the contract. +""" + +from __future__ import annotations + +import os +import threading +import time +from dataclasses import dataclass +from pathlib import Path +from typing import Optional +from unittest.mock import patch + +import pytest + +from core.download_plugins.album_bundle import ( + ALBUM_PICK_MAX_BYTES, + ALBUM_PICK_MIN_BYTES, + DEFAULT_POLL_INTERVAL_SECONDS, + DEFAULT_POLL_TIMEOUT_SECONDS, + atomic_copy_to_staging, + copy_audio_files_atomically, + get_poll_interval, + get_poll_timeout, + pick_best_album_release, + quality_score, + unique_staging_path, +) + + +# Minimal release-result shim — duck-types the fields the picker reads. +@dataclass +class _Release: + title: str + size: int + seeders: Optional[int] = None + grabs: Optional[int] = None + + +def _flac_quality_guess(title: str) -> str: + """Stand-in for the plugin's title→quality function.""" + t = (title or '').lower() + if 'flac' in t: + return 'flac' + if 'aac' in t: + return 'aac' + if 'ogg' in t: + return 'ogg' + return 'mp3' + + +# --------------------------------------------------------------------------- +# pick_best_album_release +# --------------------------------------------------------------------------- + + +def test_picker_returns_none_for_empty_input() -> None: + assert pick_best_album_release([], _flac_quality_guess) is None + + +def test_picker_drops_singletons_when_albums_present() -> None: + """Single-track torrents under 40 MB shouldn't beat an album-sized + candidate even if the single has thousands of seeders.""" + single = _Release(title='Track [MP3]', size=10_000_000, seeders=10_000) + album = _Release(title='Album [MP3]', size=120_000_000, seeders=5) + assert pick_best_album_release([single, album], _flac_quality_guess) is album + + +def test_picker_prefers_flac_when_tied_on_seeders() -> None: + flac = _Release(title='Album [FLAC]', size=400_000_000, seeders=50) + mp3 = _Release(title='Album [MP3]', size=130_000_000, seeders=50) + assert pick_best_album_release([flac, mp3], _flac_quality_guess) is flac + + +def test_picker_uses_grabs_when_seeders_is_none() -> None: + """Usenet results have ``seeders=None`` — the picker should fall + back to ``grabs`` so popularity still drives the ranking.""" + cold = _Release(title='Album A [MP3]', size=200_000_000, seeders=None, grabs=1) + popular = _Release(title='Album B [MP3]', size=200_000_000, seeders=None, grabs=999) + assert pick_best_album_release([cold, popular], _flac_quality_guess) is popular + + +def test_picker_falls_back_when_all_below_floor() -> None: + """When every candidate is below the 40 MB album-size floor, + return the most-seeded one rather than None — the user still + wants a download attempt.""" + small_low = _Release(title='X', size=5_000_000, seeders=10) + small_high = _Release(title='Y', size=8_000_000, seeders=200) + assert pick_best_album_release([small_low, small_high], _flac_quality_guess) is small_high + + +def test_picker_size_floor_matches_constant() -> None: + """If someone moves the constant the floor moves with it — pin + the relationship to catch accidental literals creeping back in.""" + just_below = _Release(title='Below', size=ALBUM_PICK_MIN_BYTES - 1, seeders=999) + just_above = _Release(title='Above', size=ALBUM_PICK_MIN_BYTES + 1, seeders=1) + assert pick_best_album_release([just_below, just_above], _flac_quality_guess) is just_above + + +def test_picker_rejects_oversized_box_sets() -> None: + """Anything past 3 GB drops out of the preferred pool — most likely + a multi-disc box set with scans + bonus material, not what the + user asked for.""" + sane = _Release(title='Album [FLAC]', size=400_000_000, seeders=10) + box = _Release(title='Album Box [FLAC]', size=ALBUM_PICK_MAX_BYTES + 1_000_000, seeders=999) + # Sane wins even with 100x fewer seeders, because box is outside + # the preferred range. + assert pick_best_album_release([sane, box], _flac_quality_guess) is sane + + +# --------------------------------------------------------------------------- +# quality_score +# --------------------------------------------------------------------------- + + +def test_quality_score_orders_formats() -> None: + assert quality_score('Album [FLAC]', _flac_quality_guess) > quality_score('Album [MP3]', _flac_quality_guess) + assert quality_score('Album [AAC]', _flac_quality_guess) > quality_score('Album [MP3]', _flac_quality_guess) + assert quality_score('Bare title', _flac_quality_guess) == quality_score('Album [MP3]', _flac_quality_guess) + + +# --------------------------------------------------------------------------- +# unique_staging_path +# --------------------------------------------------------------------------- + + +def test_unique_staging_path_returns_natural_when_clear(tmp_path: Path) -> None: + src = tmp_path / 'src.flac' + src.write_bytes(b'fLaC') + staging = tmp_path / 'staging' + staging.mkdir() + assert unique_staging_path(staging, src) == staging / 'src.flac' + + +def test_unique_staging_path_suffixes_on_collision(tmp_path: Path) -> None: + src = tmp_path / 'src.flac' + src.write_bytes(b'fLaC') + staging = tmp_path / 'staging' + staging.mkdir() + (staging / 'src.flac').write_bytes(b'existing') + assert unique_staging_path(staging, src) == staging / 'src_1.flac' + + +def test_unique_staging_path_increments_suffix(tmp_path: Path) -> None: + src = tmp_path / 'src.flac' + src.write_bytes(b'fLaC') + staging = tmp_path / 'staging' + staging.mkdir() + (staging / 'src.flac').write_bytes(b'1') + (staging / 'src_1.flac').write_bytes(b'2') + (staging / 'src_2.flac').write_bytes(b'3') + assert unique_staging_path(staging, src) == staging / 'src_3.flac' + + +# --------------------------------------------------------------------------- +# atomic_copy_to_staging +# --------------------------------------------------------------------------- + + +def test_atomic_copy_lands_at_final_path(tmp_path: Path) -> None: + src = tmp_path / 'src.flac' + src.write_bytes(b'fLaC payload') + dest = tmp_path / 'staging' / 'track.flac' + dest.parent.mkdir() + assert atomic_copy_to_staging(src, dest) is True + assert dest.read_bytes() == b'fLaC payload' + + +def test_atomic_copy_leaves_no_tmp_files_after_success(tmp_path: Path) -> None: + """The .tmp. sidecar must be cleaned up by the rename — + no orphan files left behind on a successful copy.""" + src = tmp_path / 'src.flac' + src.write_bytes(b'data') + dest = tmp_path / 'staging' / 'track.flac' + dest.parent.mkdir() + atomic_copy_to_staging(src, dest) + tmp_files = list(dest.parent.glob('*.tmp.*')) + assert tmp_files == [] + + +def test_atomic_copy_never_exposes_partial_to_extension_scanner(tmp_path: Path) -> None: + """Auto-Import filters by audio extension — the in-flight file + must NEVER be visible at its final extension until the copy is + complete. We probe this by scanning the staging dir in parallel + with the copy and assert the audio file is either absent OR + fully written. + """ + src = tmp_path / 'src.flac' + src.write_bytes(b'x' * (2 * 1024 * 1024)) + dest = tmp_path / 'staging' / 'track.flac' + dest.parent.mkdir() + + stop = threading.Event() + saw_partial = threading.Event() + expected_size = src.stat().st_size + + def _scan_loop(): + while not stop.is_set(): + try: + files = [p for p in dest.parent.iterdir() if p.suffix == '.flac'] + except FileNotFoundError: + continue + for fp in files: + size = fp.stat().st_size + if 0 < size < expected_size: + saw_partial.set() + return + + scanner = threading.Thread(target=_scan_loop, daemon=True) + scanner.start() + try: + for i in range(5): + target = dest.with_name(f'track_{i}.flac') + atomic_copy_to_staging(src, target) + # Give the scanner a moment to drain any final scan iteration. + time.sleep(0.05) + finally: + stop.set() + scanner.join(timeout=1.0) + + assert not saw_partial.is_set(), \ + "Scanner observed a partial audio file — atomic copy contract broken" + + +def test_copy_audio_files_atomically_skips_failures(tmp_path: Path) -> None: + """One file failing to copy shouldn't stop the rest from being + staged — partial results are better than a complete bailout.""" + src_a = tmp_path / 'a.flac' + src_a.write_bytes(b'a') + src_missing = tmp_path / 'does-not-exist.flac' # never created + src_c = tmp_path / 'c.flac' + src_c.write_bytes(b'c') + staging = tmp_path / 'staging' + out = copy_audio_files_atomically([src_a, src_missing, src_c], staging) + assert len(out) == 2 + landed = sorted(Path(p).name for p in out) + assert landed == ['a.flac', 'c.flac'] + + +def test_copy_audio_files_atomically_creates_staging_dir(tmp_path: Path) -> None: + src = tmp_path / 'a.flac' + src.write_bytes(b'a') + staging = tmp_path / 'nested' / 'staging' / 'dir' + out = copy_audio_files_atomically([src], staging) + assert len(out) == 1 + assert staging.exists() + + +# --------------------------------------------------------------------------- +# Config-driven poll cadence +# --------------------------------------------------------------------------- + + +def test_get_poll_interval_uses_default_when_unset() -> None: + with patch('core.download_plugins.album_bundle.config_manager') as cm: + cm.get.return_value = DEFAULT_POLL_INTERVAL_SECONDS + assert get_poll_interval() == DEFAULT_POLL_INTERVAL_SECONDS + + +def test_get_poll_interval_honours_override() -> None: + with patch('core.download_plugins.album_bundle.config_manager') as cm: + cm.get.return_value = 5 + assert get_poll_interval() == 5.0 + + +def test_get_poll_interval_falls_back_on_garbage() -> None: + """Non-numeric / non-positive values fall back to the default + rather than crashing the poll loop.""" + with patch('core.download_plugins.album_bundle.config_manager') as cm: + cm.get.return_value = 'not-a-number' + assert get_poll_interval() == DEFAULT_POLL_INTERVAL_SECONDS + cm.get.return_value = -1 + assert get_poll_interval() == DEFAULT_POLL_INTERVAL_SECONDS + + +def test_get_poll_timeout_uses_default_when_unset() -> None: + with patch('core.download_plugins.album_bundle.config_manager') as cm: + cm.get.return_value = DEFAULT_POLL_TIMEOUT_SECONDS + assert get_poll_timeout() == DEFAULT_POLL_TIMEOUT_SECONDS + + +def test_get_poll_timeout_honours_override() -> None: + """Users with slow trackers / large box sets can extend the + deadline without touching code.""" + with patch('core.download_plugins.album_bundle.config_manager') as cm: + cm.get.return_value = 86_400 # 24h + assert get_poll_timeout() == 86_400.0 + + +def test_get_poll_timeout_falls_back_on_garbage() -> None: + with patch('core.download_plugins.album_bundle.config_manager') as cm: + cm.get.return_value = '' + assert get_poll_timeout() == DEFAULT_POLL_TIMEOUT_SECONDS + cm.get.return_value = 0 + assert get_poll_timeout() == DEFAULT_POLL_TIMEOUT_SECONDS diff --git a/tests/test_torrent_usenet_plugins.py b/tests/test_torrent_usenet_plugins.py index f830a7ed..40779240 100644 --- a/tests/test_torrent_usenet_plugins.py +++ b/tests/test_torrent_usenet_plugins.py @@ -395,20 +395,22 @@ def test_plugins_conform_to_protocol() -> None: def test_torrent_album_pick_prefers_seeded_flac(tmp_path: Path) -> None: """Album bundle picker prefers high-seeded FLAC over low-seeded MP3 of comparable size — protects against picking a dead torrent.""" - from core.download_plugins.torrent import _pick_best_album_release + from core.download_plugins.album_bundle import pick_best_album_release + from core.download_plugins.torrent import _guess_quality_from_title flac = _make_torrent_result(title='Kendrick Lamar - GNX [FLAC]', size=400_000_000, seeders=120) mp3 = _make_torrent_result(title='Kendrick Lamar - GNX [MP3 320]', size=120_000_000, seeders=5, guid='guid-2') - picked = _pick_best_album_release([flac, mp3]) + picked = pick_best_album_release([flac, mp3], _guess_quality_from_title) assert picked is flac def test_torrent_album_pick_drops_too_small() -> None: """Single-track torrents (~10 MB) shouldn't be picked when the user is downloading a whole album — the size floor (40 MB) catches them.""" - from core.download_plugins.torrent import _pick_best_album_release + from core.download_plugins.album_bundle import pick_best_album_release + from core.download_plugins.torrent import _guess_quality_from_title single = _make_torrent_result(title='Kendrick Lamar - HUMBLE', size=10_000_000, seeders=500) album = _make_torrent_result(title='Kendrick Lamar - DAMN [MP3]', size=120_000_000, seeders=50, guid='guid-2') - picked = _pick_best_album_release([single, album]) + picked = pick_best_album_release([single, album], _guess_quality_from_title) assert picked is album @@ -416,26 +418,27 @@ def test_torrent_album_pick_falls_back_when_all_outside_size_range() -> None: """If every candidate is below the floor (e.g. all results are singles), pick the most-seeded one rather than returning None — user still wants a download even if it's a track torrent.""" - from core.download_plugins.torrent import _pick_best_album_release + from core.download_plugins.album_bundle import pick_best_album_release + from core.download_plugins.torrent import _guess_quality_from_title small_a = _make_torrent_result(title='X [MP3]', size=8_000_000, seeders=5) small_b = _make_torrent_result(title='Y [MP3]', size=9_000_000, seeders=80, guid='guid-2') - picked = _pick_best_album_release([small_a, small_b]) + picked = pick_best_album_release([small_a, small_b], _guess_quality_from_title) assert picked is small_b def test_unique_staging_path_handles_collision(tmp_path: Path) -> None: - from core.download_plugins.torrent import _unique_staging_path + from core.download_plugins.album_bundle import unique_staging_path src = tmp_path / 'src' / 'track.flac' src.parent.mkdir() src.write_bytes(b'fLaC') dest_dir = tmp_path / 'staging' dest_dir.mkdir() # First call returns the natural name. - first = _unique_staging_path(dest_dir, src) + first = unique_staging_path(dest_dir, src) assert first == dest_dir / 'track.flac' first.write_bytes(b'fLaC') # Second call picks a non-colliding suffix. - second = _unique_staging_path(dest_dir, src) + second = unique_staging_path(dest_dir, src) assert second == dest_dir / 'track_1.flac' From ad59bf05a1324b85a33433bd010a62d1d0d6c8ab Mon Sep 17 00:00:00 2001 From: Broque Thomas <26755000+Nezreka@users.noreply.github.com> Date: Wed, 20 May 2026 20:29:50 -0700 Subject: [PATCH 12/25] refactor(downloads): lift album-bundle gate into its own module MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Per code review: ~90 lines of inline gate logic in ``run_full_missing_tracks_process`` was inflating an already-580- line worker function and was non-testable in isolation. Lifted to ``core/downloads/album_bundle_dispatch.py`` with two entry points: - ``is_eligible(mode, is_album, album_name, artist_name)`` — pure predicate, no side effects, easy to assert against. Splits the gate decision from the resolution + run step so tests can pin the gate semantics without standing up a plugin. - ``try_dispatch(...)`` — full flow. Returns True iff the master worker should stop (gate fired + failed); False = engaged-and- succeeded OR didn't engage, both fall through to per-track. State access is now decoupled from ``runtime_state``: - New ``BatchStateAccess`` Protocol with two methods (``update_fields``, ``mark_failed``). - Concrete impl ``_BatchStateAccessImpl`` lives in master.py and wraps the tasks_lock + dict ops the original inline code did. - Injected via parameter so the dispatch module never imports ``download_batches`` / ``tasks_lock`` directly. Same goes for the plugin resolver and config getter — both injected, so the dispatcher works against any orchestrator / config implementation (including the in-test fakes). Behavior unchanged. The master worker call site is now 11 lines of boilerplate instead of 90 lines of inline conditional. Plugin contract (``download_album_to_staging`` return dict shape) unchanged. - core/downloads/album_bundle_dispatch.py: new module owning the gate + execution. ~150 lines including docstrings and the Protocol definition. - core/downloads/master.py: gate call site shrunk to a single ``if _album_bundle_dispatch.try_dispatch(...): return``. New ``_BatchStateAccessImpl`` class implements the Protocol against the existing ``download_batches`` dict + ``tasks_lock`` so the dispatcher gets injected access instead of importing them. - tests/test_album_bundle_dispatch.py: 16 new tests covering the pure predicate (album-required, mode allowlist, name validation, case insensitivity), the resolver-failure fall-through (plugin missing, plugin lacks method, resolver raises), the success path returning False so per-track flows, the failure path returning True with state.mark_failed called, plugin-raise treated as a normal failure, whitespace stripping on names, and progress-callback mirroring lifecycle events into batch state. --- core/downloads/album_bundle_dispatch.py | 179 ++++++++++++++ core/downloads/master.py | 130 +++------- tests/test_album_bundle_dispatch.py | 300 ++++++++++++++++++++++++ 3 files changed, 516 insertions(+), 93 deletions(-) create mode 100644 core/downloads/album_bundle_dispatch.py create mode 100644 tests/test_album_bundle_dispatch.py diff --git a/core/downloads/album_bundle_dispatch.py b/core/downloads/album_bundle_dispatch.py new file mode 100644 index 00000000..71692f6c --- /dev/null +++ b/core/downloads/album_bundle_dispatch.py @@ -0,0 +1,179 @@ +"""Album-bundle dispatch for torrent / usenet single-source downloads. + +Lifted from ``run_full_missing_tracks_process`` so the master +worker doesn't carry a 90-line inline branch and so the gate logic +can be unit-tested in isolation. + +The gate fires only when ALL conditions hold: + +- Batch is an album-context download (``is_album_download`` flag). +- Active download source is ``torrent`` or ``usenet`` (single-source + mode — hybrid stays per-track to preserve fallback). +- Both album-name and artist-name are populated in batch context. +- The resolved plugin exposes ``download_album_to_staging``. + +When the gate engages it runs the plugin synchronously (the master +worker is already on a thread-pool executor) and mirrors the +plugin's lifecycle payloads into the batch state so the Downloads +page can render meaningful progress before per-track tasks exist. + +Return semantics: ``True`` means the gate handled the batch — the +master worker should stop and not run per-track analysis. ``False`` +means the gate didn't engage (or engaged-and-fell-back) — caller +continues the normal per-track flow. + +The ``BatchStateAccess`` Protocol exists so this module doesn't +import ``download_batches`` from runtime_state directly. The +caller (master worker) injects accessors so this module stays +testable without touching live runtime state. +""" + +from __future__ import annotations + +import logging +from typing import Any, Callable, Optional, Protocol + +logger = logging.getLogger(__name__) + + +class BatchStateAccess(Protocol): + """Narrow shim around the batch-state dict ops the dispatch needs. + + Two methods to keep the surface small: + - ``update_fields(batch_id, fields)`` — atomic merge into the + batch dict under tasks_lock. + - ``mark_failed(batch_id, error)`` — convenience for the failure + path (sets phase + error + album_bundle_state in one shot). + """ + + def update_fields(self, batch_id: str, fields: dict) -> None: ... + + def mark_failed(self, batch_id: str, error: str) -> None: ... + + +# Fields the album-bundle progress callback may carry. Anything in +# this set gets mirrored onto the batch row as ``album_bundle_`` +# so the Downloads page can render it without coupling to the +# specific payload shape. +_MIRRORED_KEYS = ('progress', 'release', 'speed', 'downloaded', + 'size', 'seeders', 'grabs', 'count') + + +def is_eligible( + *, + mode: str, + is_album: bool, + album_name: str, + artist_name: str, +) -> bool: + """Pure predicate: does this batch even qualify for the album + flow? Separate from the resolution+run step so tests can pin + the gate logic without standing up a plugin.""" + if not is_album: + return False + if (mode or '').lower() not in ('torrent', 'usenet'): + return False + if not (album_name or '').strip(): + return False + if not (artist_name or '').strip(): + return False + return True + + +def try_dispatch( + *, + batch_id: str, + is_album: bool, + album_context: Optional[dict], + artist_context: Optional[dict], + config_get: Callable[..., Any], + plugin_resolver: Callable[[str], Optional[Any]], + state: BatchStateAccess, +) -> bool: + """Attempt the album-bundle flow. Returns ``True`` iff the + master worker should return early (gate engaged and completed + — success OR failure). ``False`` means fall through to the + normal per-track flow. + + ``config_get`` is a callable shaped like ``config_manager.get``; + ``plugin_resolver`` resolves a source-name string to an + initialised plugin instance (or None); ``state`` is the + BatchStateAccess shim. Injecting these keeps the module + dependency-light + unit-testable. + """ + mode = (config_get('download_source.mode', 'soulseek') or 'soulseek').lower() + album_name = (album_context or {}).get('name') or '' + artist_name = (artist_context or {}).get('name') or '' + + if not is_eligible(mode=mode, is_album=is_album, + album_name=album_name, artist_name=artist_name): + return False + + album_name = album_name.strip() + artist_name = artist_name.strip() + + plugin = None + try: + plugin = plugin_resolver(mode) + except Exception as exc: + logger.warning("[Album Bundle] Could not resolve %s plugin: %s", mode, exc) + + if plugin is None or not hasattr(plugin, 'download_album_to_staging'): + logger.warning( + "[Album Bundle] Gate matched but plugin / context unavailable " + "(mode=%s album=%r artist=%r plugin=%s) — falling back to per-track flow", + mode, album_name, artist_name, + type(plugin).__name__ if plugin else None, + ) + return False + + staging_dir = config_get('import.staging_path', './Staging') or './Staging' + logger.info( + "[Album Bundle] Engaging %s album flow for '%s' by '%s' -> %s", + mode, album_name, artist_name, staging_dir, + ) + state.update_fields(batch_id, { + 'phase': 'album_downloading', + 'album_bundle_state': 'searching', + 'album_bundle_source': mode, + }) + + def _emit(payload): + """Mirror plugin lifecycle into batch state for UI rendering.""" + try: + fields = {'album_bundle_state': payload.get('state', '')} + for key in _MIRRORED_KEYS: + if key in payload: + fields[f'album_bundle_{key}'] = payload[key] + state.update_fields(batch_id, fields) + except Exception as exc: + logger.debug("[Album Bundle] emit failed: %s", exc) + + try: + outcome = plugin.download_album_to_staging( + album_name, artist_name, staging_dir, _emit, + ) + except Exception as exc: + logger.exception("[Album Bundle] %s plugin raised: %s", mode, exc) + outcome = {'success': False, 'error': f'Plugin error: {exc}'} + + if not outcome.get('success'): + err = outcome.get('error', 'Album bundle download failed') + logger.error("[Album Bundle] %s flow failed for '%s': %s", + mode, album_name, err) + state.mark_failed(batch_id, err) + return True + + logger.info( + "[Album Bundle] %s staged %d files for '%s' — handing off to per-track staging matcher", + mode, len(outcome.get('files', [])), album_name, + ) + state.update_fields(batch_id, { + 'phase': 'analysis', + 'album_bundle_state': 'staged', + }) + # Engaged-and-succeeded: we DON'T early-return because the + # per-track flow needs to run to create + complete the per-track + # task rows. Those tasks will hit try_staging_match and pull the + # files we just staged. + return False diff --git a/core/downloads/master.py b/core/downloads/master.py index 518797b9..90adbb4e 100644 --- a/core/downloads/master.py +++ b/core/downloads/master.py @@ -35,6 +35,7 @@ from difflib import SequenceMatcher from pathlib import Path from typing import Any, Callable +from core.downloads import album_bundle_dispatch as _album_bundle_dispatch from core.runtime_state import download_batches, download_tasks, tasks_lock logger = logging.getLogger(__name__) @@ -270,6 +271,26 @@ class MasterDeps: reset_wishlist_auto_processing: Callable[[], None] +class _BatchStateAccessImpl: + """Concrete ``BatchStateAccess`` for the runtime ``download_batches`` + dict — wraps the lock + the existing-batch check so the album- + bundle dispatcher stays decoupled from runtime_state.""" + + def update_fields(self, batch_id: str, fields: dict) -> None: + with tasks_lock: + row = download_batches.get(batch_id) + if row is not None: + row.update(fields) + + def mark_failed(self, batch_id: str, error: str) -> None: + with tasks_lock: + row = download_batches.get(batch_id) + if row is not None: + row['phase'] = 'failed' + row['error'] = error + row['album_bundle_state'] = 'failed' + + def run_full_missing_tracks_process(batch_id, playlist_id, tracks_json, deps: MasterDeps): """ A master worker that handles the entire missing tracks process: @@ -311,99 +332,22 @@ def run_full_missing_tracks_process(batch_id, playlist_id, tracks_json, deps: Ma if force_download_all: logger.warning(f"[Force Download] Force download mode enabled for batch {batch_id} - treating all tracks as missing") - # ════════════════════════════════════════════════════════════════ - # ALBUM-BUNDLE GATE for torrent / usenet single-source mode. - # - # Indexer-based sources (torrent / usenet) are release-level — a - # Prowlarr search for "Luther (with SZA)" returns the GNX album - # torrent at near-zero confidence against the track title. The - # per-track search loop fails for every track on the album. - # - # Workaround: when the user is downloading an album AND has - # torrent / usenet selected as the SINGLE active source (not - # hybrid — hybrid stays per-track to preserve fallback to - # Soulseek / streaming sources), do ONE Prowlarr search for the - # whole album, hand the picked release to the torrent / usenet - # client, walk the resulting audio files, and drop them into the - # staging folder. Each per-track task then hits the existing - # ``try_staging_match`` early-return in the per-track worker - # before any Prowlarr search fires, and the normal post- - # processing pipeline imports each matched file. - # - # Gate intentionally narrow: hybrid mode, non-album batches, and - # plugins without ``download_album_to_staging`` (i.e. every - # source other than torrent / usenet) all bypass this branch - # untouched. - # ════════════════════════════════════════════════════════════════ - _album_bundle_mode = (deps.config_manager.get('download_source.mode', 'soulseek') or 'soulseek').lower() - _is_torrent_or_usenet = _album_bundle_mode in ('torrent', 'usenet') - if batch_is_album and _is_torrent_or_usenet and batch_album_context and batch_artist_context: - _bundle_album = (batch_album_context.get('name') or '').strip() - _bundle_artist = (batch_artist_context.get('name') or '').strip() - _bundle_plugin = None - try: - _bundle_plugin = deps.download_orchestrator.client(_album_bundle_mode) - except Exception as _exc: - logger.warning("[Album Bundle] Could not resolve %s plugin: %s", _album_bundle_mode, _exc) - if _bundle_album and _bundle_artist and _bundle_plugin and hasattr(_bundle_plugin, 'download_album_to_staging'): - _staging_dir = deps.config_manager.get('import.staging_path', './Staging') or './Staging' - logger.info( - "[Album Bundle] Engaging %s album flow for '%s' by '%s' -> %s", - _album_bundle_mode, _bundle_album, _bundle_artist, _staging_dir, - ) - with tasks_lock: - if batch_id in download_batches: - download_batches[batch_id]['phase'] = 'album_downloading' - download_batches[batch_id]['album_bundle_state'] = 'searching' - download_batches[batch_id]['album_bundle_source'] = _album_bundle_mode - - def _bundle_emit(payload): - """Mirror the album-download lifecycle into batch state so the - Downloads page can render meaningful status while the torrent / - usenet job runs (the per-track tasks don't exist yet).""" - try: - with tasks_lock: - if batch_id in download_batches: - _row = download_batches[batch_id] - _row['album_bundle_state'] = payload.get('state', '') - for _k in ('progress', 'release', 'speed', 'downloaded', 'size', 'seeders', 'grabs', 'count'): - if _k in payload: - _row[f'album_bundle_{_k}'] = payload[_k] - except Exception as _emit_exc: - logger.debug("[Album Bundle] emit failed: %s", _emit_exc) - - try: - _bundle_outcome = _bundle_plugin.download_album_to_staging( - _bundle_album, _bundle_artist, _staging_dir, _bundle_emit, - ) - except Exception as _bundle_exc: - logger.exception("[Album Bundle] %s plugin raised: %s", _album_bundle_mode, _bundle_exc) - _bundle_outcome = {'success': False, 'error': f'Plugin error: {_bundle_exc}'} - - if not _bundle_outcome.get('success'): - _err = _bundle_outcome.get('error', 'Album bundle download failed') - logger.error("[Album Bundle] %s flow failed for '%s': %s", _album_bundle_mode, _bundle_album, _err) - with tasks_lock: - if batch_id in download_batches: - download_batches[batch_id]['phase'] = 'failed' - download_batches[batch_id]['error'] = _err - download_batches[batch_id]['album_bundle_state'] = 'failed' - return - logger.info( - "[Album Bundle] %s staged %d files for '%s' — handing off to per-track staging matcher", - _album_bundle_mode, len(_bundle_outcome.get('files', [])), _bundle_album, - ) - with tasks_lock: - if batch_id in download_batches: - download_batches[batch_id]['phase'] = 'analysis' - download_batches[batch_id]['album_bundle_state'] = 'staged' - else: - logger.warning( - "[Album Bundle] Gate matched but plugin / context unavailable " - "(mode=%s album=%r artist=%r plugin=%s) — falling back to per-track flow", - _album_bundle_mode, _bundle_album, _bundle_artist, - type(_bundle_plugin).__name__ if _bundle_plugin else None, - ) + # Album-bundle gate for torrent / usenet single-source mode. + # See ``core/downloads/album_bundle_dispatch`` for the full + # narrow-gate rationale. Returns True iff the master worker + # should stop (gate fired and failed); False = engaged-and- + # succeeded OR didn't engage, both fall through to per-track. + _bundle_state = _BatchStateAccessImpl() + if _album_bundle_dispatch.try_dispatch( + batch_id=batch_id, + is_album=batch_is_album, + album_context=batch_album_context, + artist_context=batch_artist_context, + config_get=deps.config_manager.get, + plugin_resolver=deps.download_orchestrator.client, + state=_bundle_state, + ): + return # Allow duplicate tracks across albums — when enabled, only skip tracks already # owned in THIS album, not tracks owned in other albums diff --git a/tests/test_album_bundle_dispatch.py b/tests/test_album_bundle_dispatch.py new file mode 100644 index 00000000..d60d0e84 --- /dev/null +++ b/tests/test_album_bundle_dispatch.py @@ -0,0 +1,300 @@ +"""Tests for ``core/downloads/album_bundle_dispatch.py``. + +Pins the gate predicate, the resolution + run flow, and the +fail / fall-through return contract. Mocks the config, plugin +resolver, and state access so the dispatcher is testable without +standing up runtime_state or a real plugin. +""" + +from __future__ import annotations + +from unittest.mock import MagicMock + +import pytest + +from core.downloads.album_bundle_dispatch import ( + BatchStateAccess, + is_eligible, + try_dispatch, +) + + +class _FakeState: + """In-memory ``BatchStateAccess`` for tests — records every + update so assertions can check the sequence of fields set.""" + + def __init__(self) -> None: + self.fields: dict = {} + self.update_calls: list = [] + self.failed_with: str = '' + + def update_fields(self, batch_id: str, fields: dict) -> None: + self.update_calls.append((batch_id, dict(fields))) + self.fields.update(fields) + + def mark_failed(self, batch_id: str, error: str) -> None: + self.failed_with = error + self.fields['phase'] = 'failed' + self.fields['error'] = error + self.fields['album_bundle_state'] = 'failed' + + +def _config(values: dict): + """Build a config_get callable from a flat dict.""" + def _get(key, default=None): + return values.get(key, default) + return _get + + +# --------------------------------------------------------------------------- +# is_eligible pure predicate +# --------------------------------------------------------------------------- + + +def test_is_eligible_requires_album_flag() -> None: + assert is_eligible(mode='torrent', is_album=False, + album_name='X', artist_name='Y') is False + + +def test_is_eligible_requires_torrent_or_usenet_mode() -> None: + for mode in ('soulseek', 'youtube', 'tidal', 'qobuz', 'hifi', + 'deezer_dl', 'amazon', 'lidarr', 'soundcloud', 'hybrid'): + assert is_eligible(mode=mode, is_album=True, + album_name='X', artist_name='Y') is False + + +def test_is_eligible_accepts_torrent_and_usenet() -> None: + assert is_eligible(mode='torrent', is_album=True, + album_name='X', artist_name='Y') is True + assert is_eligible(mode='usenet', is_album=True, + album_name='X', artist_name='Y') is True + + +def test_is_eligible_requires_non_empty_names() -> None: + assert is_eligible(mode='torrent', is_album=True, + album_name='', artist_name='Y') is False + assert is_eligible(mode='torrent', is_album=True, + album_name='X', artist_name='') is False + assert is_eligible(mode='torrent', is_album=True, + album_name=' ', artist_name='Y') is False + + +def test_is_eligible_case_insensitive_mode() -> None: + assert is_eligible(mode='TORRENT', is_album=True, + album_name='X', artist_name='Y') is True + + +# --------------------------------------------------------------------------- +# try_dispatch — gate evaluation +# --------------------------------------------------------------------------- + + +def test_dispatch_returns_false_when_not_album() -> None: + state = _FakeState() + plugin = MagicMock() + result = try_dispatch( + batch_id='b1', is_album=False, + album_context={'name': 'X'}, artist_context={'name': 'Y'}, + config_get=_config({'download_source.mode': 'torrent'}), + plugin_resolver=lambda _name: plugin, state=state, + ) + assert result is False + assert state.update_calls == [] + plugin.download_album_to_staging.assert_not_called() + + +def test_dispatch_returns_false_for_non_torrent_modes() -> None: + state = _FakeState() + plugin = MagicMock() + result = try_dispatch( + batch_id='b1', is_album=True, + album_context={'name': 'X'}, artist_context={'name': 'Y'}, + config_get=_config({'download_source.mode': 'soulseek'}), + plugin_resolver=lambda _name: plugin, state=state, + ) + assert result is False + assert state.update_calls == [] + + +def test_dispatch_returns_false_when_plugin_missing() -> None: + """No plugin available → fall through to per-track flow with a + warning. The state SHOULD NOT have been touched.""" + state = _FakeState() + result = try_dispatch( + batch_id='b1', is_album=True, + album_context={'name': 'X'}, artist_context={'name': 'Y'}, + config_get=_config({'download_source.mode': 'torrent'}), + plugin_resolver=lambda _name: None, state=state, + ) + assert result is False + assert state.update_calls == [] + + +def test_dispatch_returns_false_when_plugin_lacks_method() -> None: + state = _FakeState() + # Plugin that doesn't implement download_album_to_staging. + class _LegacyPlugin: + pass + result = try_dispatch( + batch_id='b1', is_album=True, + album_context={'name': 'X'}, artist_context={'name': 'Y'}, + config_get=_config({'download_source.mode': 'torrent'}), + plugin_resolver=lambda _name: _LegacyPlugin(), state=state, + ) + assert result is False + assert state.update_calls == [] + + +def test_dispatch_returns_false_when_resolver_raises() -> None: + """Plugin resolution can fail (registry not initialised); we log + and fall through rather than crashing the master worker.""" + state = _FakeState() + def _boom(_name): + raise RuntimeError("registry not initialised") + result = try_dispatch( + batch_id='b1', is_album=True, + album_context={'name': 'X'}, artist_context={'name': 'Y'}, + config_get=_config({'download_source.mode': 'torrent'}), + plugin_resolver=_boom, state=state, + ) + assert result is False + + +# --------------------------------------------------------------------------- +# try_dispatch — success / failure paths +# --------------------------------------------------------------------------- + + +def test_dispatch_success_returns_false_so_per_track_can_run() -> None: + """Success → master worker should CONTINUE to per-track flow so + each task can hit try_staging_match and find its file.""" + state = _FakeState() + plugin = MagicMock() + plugin.download_album_to_staging.return_value = { + 'success': True, 'files': ['/tmp/a.flac', '/tmp/b.flac'], 'error': None, + } + result = try_dispatch( + batch_id='b1', is_album=True, + album_context={'name': 'GNX'}, artist_context={'name': 'Kendrick Lamar'}, + config_get=_config({ + 'download_source.mode': 'torrent', + 'import.staging_path': '/staging/path', + }), + plugin_resolver=lambda _name: plugin, state=state, + ) + assert result is False + # Plugin was called with the right args. + args = plugin.download_album_to_staging.call_args + assert args.args[0] == 'GNX' + assert args.args[1] == 'Kendrick Lamar' + assert args.args[2] == '/staging/path' + # Phase transitioned through searching → analysis. + assert state.fields['phase'] == 'analysis' + assert state.fields['album_bundle_state'] == 'staged' + assert state.fields['album_bundle_source'] == 'torrent' + assert state.failed_with == '' + + +def test_dispatch_failure_returns_true_so_master_stops() -> None: + state = _FakeState() + plugin = MagicMock() + plugin.download_album_to_staging.return_value = { + 'success': False, 'files': [], 'error': 'No torrent results found', + } + result = try_dispatch( + batch_id='b1', is_album=True, + album_context={'name': 'GNX'}, artist_context={'name': 'Kendrick Lamar'}, + config_get=_config({'download_source.mode': 'torrent'}), + plugin_resolver=lambda _name: plugin, state=state, + ) + assert result is True + assert state.failed_with == 'No torrent results found' + assert state.fields['phase'] == 'failed' + + +def test_dispatch_plugin_exception_treated_as_failure() -> None: + """A bug / network error in the plugin must not propagate into + the master worker — caught + treated as a normal failure so + the batch reports the error cleanly.""" + state = _FakeState() + plugin = MagicMock() + plugin.download_album_to_staging.side_effect = RuntimeError("network down") + result = try_dispatch( + batch_id='b1', is_album=True, + album_context={'name': 'GNX'}, artist_context={'name': 'Kendrick Lamar'}, + config_get=_config({'download_source.mode': 'torrent'}), + plugin_resolver=lambda _name: plugin, state=state, + ) + assert result is True + assert 'network down' in state.failed_with + + +def test_dispatch_strips_whitespace_from_names() -> None: + """Trailing whitespace in batch context shouldn't fail the + eligibility predicate AND should be cleaned before passing to + the plugin.""" + state = _FakeState() + plugin = MagicMock() + plugin.download_album_to_staging.return_value = {'success': True, 'files': ['/x']} + try_dispatch( + batch_id='b1', is_album=True, + album_context={'name': ' GNX '}, artist_context={'name': ' Kendrick '}, + config_get=_config({'download_source.mode': 'torrent'}), + plugin_resolver=lambda _name: plugin, state=state, + ) + args = plugin.download_album_to_staging.call_args + assert args.args[0] == 'GNX' + assert args.args[1] == 'Kendrick' + + +def test_dispatch_progress_callback_mirrors_payload_to_state() -> None: + """The progress callback the plugin gets must mirror its + payload onto the batch state under ``album_bundle_*`` keys so + the Downloads page can render progress while the torrent + download runs.""" + state = _FakeState() + captured_emit = {} + + def _capture(album, artist, staging, emit): + captured_emit['fn'] = emit + emit({'state': 'searching', 'release': 'GNX [FLAC]'}) + emit({'state': 'downloading', 'progress': 0.42, 'speed': 1024 * 1024}) + emit({'state': 'staged', 'count': 12}) + return {'success': True, 'files': []} + + plugin = MagicMock() + plugin.download_album_to_staging.side_effect = _capture + try_dispatch( + batch_id='b1', is_album=True, + album_context={'name': 'GNX'}, artist_context={'name': 'Kendrick Lamar'}, + config_get=_config({'download_source.mode': 'torrent'}), + plugin_resolver=lambda _name: plugin, state=state, + ) + # State should have seen each of the three lifecycle emissions. + states_seen = [fields.get('album_bundle_state') + for _, fields in state.update_calls + if 'album_bundle_state' in fields] + assert 'searching' in states_seen + assert 'downloading' in states_seen + assert 'staged' in states_seen + # Numeric progress + release name made it through. + assert state.fields['album_bundle_release'] == 'GNX [FLAC]' + assert state.fields['album_bundle_progress'] == 0.42 + assert state.fields['album_bundle_count'] == 12 + + +# --------------------------------------------------------------------------- +# Protocol conformance — runtime impl must satisfy the contract +# --------------------------------------------------------------------------- + + +def test_runtime_state_impl_matches_protocol() -> None: + """Sanity check that the concrete BatchStateAccess impl in + master.py implements both methods. We don't import master.py + here (would pull in heavy deps); duck-check on the _FakeState + instead since it's a sibling impl of the same Protocol.""" + state: BatchStateAccess = _FakeState() + state.update_fields('b1', {'x': 1}) + state.mark_failed('b1', 'oops') + assert state.fields['x'] == 1 + assert state.fields['error'] == 'oops' From 440c3624f36dffc24c33df0d2b6d45d6f498e2aa Mon Sep 17 00:00:00 2001 From: Broque Thomas <26755000+Nezreka@users.noreply.github.com> Date: Wed, 20 May 2026 20:43:35 -0700 Subject: [PATCH 13/25] refactor(staging): inject batch-field accessor instead of importing runtime_state MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Per code review: the album-bundle provenance override added in an earlier commit reached into ``core.runtime_state.download_batches`` directly from inside the staging matcher. Sibling modules shouldn't import each other's globals — the existing StagingDeps pattern is the canonical way to inject everything else this helper needs. - core/downloads/staging.py: new optional ``get_batch_field`` callable on ``StagingDeps`` (defaults to None for backward compat with any caller that doesn't know about it yet). The inline ``from core.runtime_state import download_batches`` is gone; the helper now calls ``deps.get_batch_field(batch_id, 'album_bundle_source')`` and falls back to 'staging' when None is returned. Accessor exceptions are swallowed with a debug log so a deleted batch mid-process can't break the staging match. - web_server.py: ``_build_staging_deps`` injects a small ``_staging_get_batch_field`` helper that wraps the tasks_lock + download_batches dict access. Centralises the lock semantics in one place — the staging module no longer needs to know about the lock or the dict. - tests/test_staging_album_provenance.py: 5 new tests covering the full matrix — torrent override applied, usenet override applied, no override falls back to 'staging', missing accessor (default None) falls back to 'staging', accessor raising falls back to 'staging'. Each test seeds + cleans a synthetic task in runtime_state so the test doesn't bleed state across the suite. --- core/downloads/staging.py | 22 ++- tests/test_staging_album_provenance.py | 198 +++++++++++++++++++++++++ web_server.py | 12 ++ 3 files changed, 224 insertions(+), 8 deletions(-) create mode 100644 tests/test_staging_album_provenance.py diff --git a/core/downloads/staging.py b/core/downloads/staging.py index a8dda6c3..dca686ef 100644 --- a/core/downloads/staging.py +++ b/core/downloads/staging.py @@ -58,6 +58,12 @@ class StagingDeps: get_staging_file_cache: Callable[[str], list] docker_resolve_path: Callable[[str], str] post_process_matched_download_with_verification: Callable + # Optional batch-field accessor. Returns ``download_batches[batch_id].get(field)`` + # when the runtime state is available, ``None`` otherwise. Injected so + # this module doesn't have to import from runtime_state directly — + # keeps the dep surface explicit and the function unit-testable + # without a live batch dict. + get_batch_field: Callable[[str, str], Any] = None # type: ignore[assignment] def try_staging_match(task_id, batch_id, track, deps: StagingDeps): @@ -144,15 +150,15 @@ def try_staging_match(task_id, batch_id, track, deps: StagingDeps): # Mark task as completed with staging context. # If the batch was populated by the torrent / usenet album-bundle # flow, prefer that provenance label over generic 'staging' so the - # download history reflects the real source. + # download history reflects the real source. The accessor is + # injected via StagingDeps so this module doesn't reach into + # runtime_state directly (see deps.get_batch_field docstring). _provenance_override = None - try: - from core.runtime_state import download_batches as _db - _batch = _db.get(batch_id) if batch_id else None - if isinstance(_batch, dict): - _provenance_override = _batch.get('album_bundle_source') - except Exception: - _provenance_override = None + if batch_id and deps.get_batch_field is not None: + try: + _provenance_override = deps.get_batch_field(batch_id, 'album_bundle_source') + except Exception as _exc: + logger.debug("get_batch_field failed: %s", _exc) _provenance_username = _provenance_override or 'staging' with tasks_lock: if task_id in download_tasks: diff --git a/tests/test_staging_album_provenance.py b/tests/test_staging_album_provenance.py new file mode 100644 index 00000000..fc62a411 --- /dev/null +++ b/tests/test_staging_album_provenance.py @@ -0,0 +1,198 @@ +"""Tests for the album-bundle provenance override in +``core/downloads/staging.py``. + +Verifies that when ``StagingDeps.get_batch_field`` returns a source +override (i.e. the batch was populated by the torrent / usenet +album-bundle flow), the staging matcher records that source on the +task instead of the generic 'staging' username. Provenance recording +downstream uses ``task['username']`` to set ``source_service`` on +the persisted download row — so this is the single point that +controls whether the history modal shows 'Torrent' / 'Usenet' vs +'Staging' / 'Soulseek'. + +Mocks the rest of StagingDeps so the test doesn't touch the +filesystem, AcoustID, or post-processing. +""" + +from __future__ import annotations + +from types import SimpleNamespace +from unittest.mock import MagicMock + +from core.downloads.staging import StagingDeps, try_staging_match +from core.runtime_state import download_tasks, tasks_lock + + +def _make_deps(staging_file, transfer_dir, batch_field_value=None): + """Build a StagingDeps with mocked collaborators. ``batch_field_value`` + is what get_batch_field returns for ``album_bundle_source`` — None + means no override (generic staging match), 'torrent' / 'usenet' + means the album-bundle flow seeded the staging folder.""" + me = MagicMock() + me.normalize_string.side_effect = lambda s: (s or '').lower().strip() + config = MagicMock() + config.get.return_value = transfer_dir + return StagingDeps( + config_manager=config, + matching_engine=me, + get_staging_file_cache=lambda _b: [staging_file], + docker_resolve_path=lambda p: p, + post_process_matched_download_with_verification=lambda *a, **kw: None, + get_batch_field=(lambda _b, _f: batch_field_value) if batch_field_value is not None else (lambda _b, _f: None), + ) + + +def _seed_task(task_id: str, track_name: str, track_artist: str) -> None: + """Register a task in the runtime_state dict so try_staging_match + has something to mark complete.""" + with tasks_lock: + download_tasks[task_id] = { + 'status': 'searching', + 'track_info': { + 'name': track_name, + 'artists': [{'name': track_artist}], + '_is_explicit_album_download': False, + }, + } + + +def _cleanup_task(task_id: str) -> None: + with tasks_lock: + download_tasks.pop(task_id, None) + + +def test_staging_match_uses_torrent_override_when_present(tmp_path) -> None: + src = tmp_path / 'staging' / 'gnx_track_01.flac' + src.parent.mkdir() + src.write_bytes(b'fLaC') + transfer = tmp_path / 'transfer' + deps = _make_deps( + staging_file={'full_path': str(src), 'title': 'Luther', 'artist': 'Kendrick Lamar'}, + transfer_dir=str(transfer), + batch_field_value='torrent', + ) + track = SimpleNamespace(name='Luther', artists=['Kendrick Lamar']) + task_id = 'test_task_torrent_override' + _seed_task(task_id, 'Luther', 'Kendrick Lamar') + try: + ok = try_staging_match(task_id, 'batch_x', track, deps) + assert ok is True + with tasks_lock: + row = download_tasks[task_id] + assert row['username'] == 'torrent', \ + f"Expected provenance override 'torrent', got {row['username']!r}" + assert row['staging_match'] is True + finally: + _cleanup_task(task_id) + + +def test_staging_match_uses_usenet_override_when_present(tmp_path) -> None: + src = tmp_path / 'staging' / 'gnx_track_01.flac' + src.parent.mkdir() + src.write_bytes(b'fLaC') + transfer = tmp_path / 'transfer' + deps = _make_deps( + staging_file={'full_path': str(src), 'title': 'Luther', 'artist': 'Kendrick Lamar'}, + transfer_dir=str(transfer), + batch_field_value='usenet', + ) + track = SimpleNamespace(name='Luther', artists=['Kendrick Lamar']) + task_id = 'test_task_usenet_override' + _seed_task(task_id, 'Luther', 'Kendrick Lamar') + try: + try_staging_match(task_id, 'batch_x', track, deps) + with tasks_lock: + assert download_tasks[task_id]['username'] == 'usenet' + finally: + _cleanup_task(task_id) + + +def test_staging_match_falls_back_to_staging_without_override(tmp_path) -> None: + """When no batch override is present (manual file drop, or + batch has no album_bundle_source field), the staging matcher + uses the historical 'staging' username.""" + src = tmp_path / 'staging' / 'gnx_track_01.flac' + src.parent.mkdir() + src.write_bytes(b'fLaC') + transfer = tmp_path / 'transfer' + deps = _make_deps( + staging_file={'full_path': str(src), 'title': 'Luther', 'artist': 'Kendrick Lamar'}, + transfer_dir=str(transfer), + batch_field_value=None, + ) + track = SimpleNamespace(name='Luther', artists=['Kendrick Lamar']) + task_id = 'test_task_no_override' + _seed_task(task_id, 'Luther', 'Kendrick Lamar') + try: + try_staging_match(task_id, 'batch_x', track, deps) + with tasks_lock: + assert download_tasks[task_id]['username'] == 'staging' + finally: + _cleanup_task(task_id) + + +def test_staging_match_handles_missing_batch_field_callable(tmp_path) -> None: + """Backward compat: callers that build StagingDeps without + supplying ``get_batch_field`` (it defaults to None) still + work — staging matcher falls back to the 'staging' username.""" + src = tmp_path / 'staging' / 'gnx_track_01.flac' + src.parent.mkdir() + src.write_bytes(b'fLaC') + transfer = tmp_path / 'transfer' + + me = MagicMock() + me.normalize_string.side_effect = lambda s: (s or '').lower().strip() + config = MagicMock() + config.get.return_value = str(transfer) + deps = StagingDeps( + config_manager=config, + matching_engine=me, + get_staging_file_cache=lambda _b: [{'full_path': str(src), 'title': 'Luther', 'artist': 'Kendrick Lamar'}], + docker_resolve_path=lambda p: p, + post_process_matched_download_with_verification=lambda *a, **kw: None, + # get_batch_field omitted — defaults to None + ) + track = SimpleNamespace(name='Luther', artists=['Kendrick Lamar']) + task_id = 'test_task_no_accessor' + _seed_task(task_id, 'Luther', 'Kendrick Lamar') + try: + try_staging_match(task_id, 'batch_x', track, deps) + with tasks_lock: + assert download_tasks[task_id]['username'] == 'staging' + finally: + _cleanup_task(task_id) + + +def test_staging_match_swallows_accessor_exception(tmp_path) -> None: + """If the injected accessor raises (e.g. the batch was deleted + mid-process), the staging matcher should fall back to 'staging' + rather than failing the whole match.""" + src = tmp_path / 'staging' / 'gnx_track_01.flac' + src.parent.mkdir() + src.write_bytes(b'fLaC') + transfer = tmp_path / 'transfer' + + def _boom(_b, _f): + raise RuntimeError("batch went away") + + me = MagicMock() + me.normalize_string.side_effect = lambda s: (s or '').lower().strip() + config = MagicMock() + config.get.return_value = str(transfer) + deps = StagingDeps( + config_manager=config, + matching_engine=me, + get_staging_file_cache=lambda _b: [{'full_path': str(src), 'title': 'Luther', 'artist': 'Kendrick Lamar'}], + docker_resolve_path=lambda p: p, + post_process_matched_download_with_verification=lambda *a, **kw: None, + get_batch_field=_boom, + ) + track = SimpleNamespace(name='Luther', artists=['Kendrick Lamar']) + task_id = 'test_task_accessor_raises' + _seed_task(task_id, 'Luther', 'Kendrick Lamar') + try: + try_staging_match(task_id, 'batch_x', track, deps) + with tasks_lock: + assert download_tasks[task_id]['username'] == 'staging' + finally: + _cleanup_task(task_id) diff --git a/web_server.py b/web_server.py index ddf3ebe4..2b151d1f 100644 --- a/web_server.py +++ b/web_server.py @@ -16500,6 +16500,17 @@ def _get_staging_file_cache(batch_id): from core.downloads import staging as _downloads_staging +def _staging_get_batch_field(batch_id, field): + """Accessor injected into StagingDeps so the staging-match helper + can read an album-bundle provenance override off the batch state + without importing ``download_batches`` directly.""" + with tasks_lock: + row = download_batches.get(batch_id) + if isinstance(row, dict): + return row.get(field) + return None + + def _build_staging_deps(): """Build the StagingDeps bundle from web_server.py globals on each call.""" return _downloads_staging.StagingDeps( @@ -16508,6 +16519,7 @@ def _build_staging_deps(): get_staging_file_cache=_get_staging_file_cache, docker_resolve_path=docker_resolve_path, post_process_matched_download_with_verification=_post_process_matched_download_with_verification, + get_batch_field=_staging_get_batch_field, ) From 1c120a7fb70736ebaa0fdc1e3ba0b45c702165c5 Mon Sep 17 00:00:00 2001 From: Broque Thomas <26755000+Nezreka@users.noreply.github.com> Date: Wed, 20 May 2026 20:48:48 -0700 Subject: [PATCH 14/25] chore(downloads): add config defaults + clarify validation fallback scope MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Wraps up the code-review refactor pass. - config/settings.py: ``download_source`` defaults gain ``album_bundle_poll_interval_seconds`` (default 2s) and ``album_bundle_timeout_seconds`` (default 6h, was a hard-coded ``6 * 60 * 60`` magic constant in torrent.py). The plugin reads these via ``album_bundle.get_poll_interval`` / ``get_poll_timeout`` with safe fallback to the defaults when the config value is missing / non-numeric. ``mode`` doc-comment extended to list ``torrent`` and ``usenet``. - core/downloads/validation.py: comment block above the album-name fallback rewritten to document when the fallback actually runs now — single-track hybrid downloads only, because the album- bundle gate handles single-source mode and the hybrid chain filter strips torrent / usenet from album batches. Code path unchanged; just clarifies the contract for the next reader. - webui/static/helper.js: WHATS_NEW entry summarising the refactor pass (helper extraction, dispatch lift, staging deps injection, atomic copy, configurable timeout, test additions). The /loop of: extract → inject → test was sweep enough to drop the gate code's coupling to 2-3 modules and put 49 unit tests behind the new boundaries. Code-review feedback addressed: 1. album_bundle.py extracted ✓ 2. Dispatch lifted out of master.py ✓ 3. staging.py decoupled from runtime_state ✓ 4. Validation fallback scope documented ✓ 5. Poll timeout config-driven ✓ 6. ``amazon`` provenance owned in a prior commit ✓ 7. End-to-end-shaped tests added (test_album_bundle_dispatch.py) 8. Auto-Import race closed via atomic copy ✓ --- config/settings.py | 10 +++++++++- core/downloads/validation.py | 28 ++++++++++++++++++++-------- webui/static/helper.js | 1 + 3 files changed, 30 insertions(+), 9 deletions(-) diff --git a/config/settings.py b/config/settings.py index aa989f41..78b019fc 100644 --- a/config/settings.py +++ b/config/settings.py @@ -480,11 +480,19 @@ class ConfigManager: "search_min_delay_seconds": 0, }, "download_source": { - "mode": "soulseek", # Options: "soulseek", "youtube", "tidal", "qobuz", "hifi", "hybrid" + "mode": "soulseek", # Options: "soulseek", "youtube", "tidal", "qobuz", "hifi", "hybrid", "torrent", "usenet" "hybrid_primary": "soulseek", # Legacy: primary source for hybrid mode "hybrid_secondary": "youtube", # Legacy: fallback source for hybrid mode "hybrid_order": [], # Ordered list of sources for hybrid mode (overrides primary/secondary) "stream_source": "youtube", # Options: "youtube" (instant, default), "active" (use download source; falls back to youtube if soulseek) + # Album-bundle (torrent / usenet single-source) poll tuning. + # Downloader is polled every N seconds until the release + # lands; whole job aborts at the timeout. Defaults match + # the previous hard-coded constants. Users on slow private + # trackers / large box sets can extend the timeout without + # editing source. + "album_bundle_poll_interval_seconds": 2.0, + "album_bundle_timeout_seconds": 6 * 60 * 60, # 6 hours }, "tidal_download": { "quality": "lossless", # Options: "low", "high", "lossless", "hires" diff --git a/core/downloads/validation.py b/core/downloads/validation.py index 337cde73..714ffb6c 100644 --- a/core/downloads/validation.py +++ b/core/downloads/validation.py @@ -146,14 +146,26 @@ def get_valid_candidates(results, spotify_track, query): candidate_duration_ms=r.duration or 0, ) - # Torrent / usenet results are typically release-level (album torrents). - # Looking for "Luther (with SZA)" against a candidate titled - # "GNX (2024) [FLAC]" scores ~0 on track-title alone, even though - # the album torrent does in fact contain the wanted track. Score - # the candidate title against the wanted track's ALBUM name too - # and take the max, so album-level releases match every track on - # them. The album_track_count bonus only kicks in when we have - # a non-empty album string to compare against. + # Album-name fallback for torrent / usenet per-track results. + # + # When this fallback runs: hybrid mode + non-album batch (single + # track wishlist / playlist of singles). Album-context batches + # never reach here — the album-bundle gate in + # core/downloads/album_bundle_dispatch.py engages the bulk- + # download flow in single-source mode, and the hybrid chain + # filter in core/downloads/task_worker.py strips torrent / + # usenet from album batches in hybrid mode. What's left is the + # single-track-in-hybrid case where a user is searching for one + # track and the only torrent / usenet result is the album that + # contains it. + # + # Without this fallback, "Luther (with SZA)" against a + # candidate titled "GNX (2024) [FLAC]" scores ~0 on track-title + # alone — even though the album torrent does in fact contain + # the wanted track. Scoring the candidate title against the + # wanted track's ALBUM name and taking the max gives album- + # level releases a fair shot. The Auto-Import sweep then picks + # the right file out of the downloaded album folder. if r.username in ('torrent', 'usenet') and spotify_track and spotify_track.album: album_conf, _ = matching_engine.score_track_match( source_title=spotify_track.album, diff --git a/webui/static/helper.js b/webui/static/helper.js index 9b184462..a48529f1 100644 --- a/webui/static/helper.js +++ b/webui/static/helper.js @@ -3415,6 +3415,7 @@ function closeHelperSearch() { const WHATS_NEW = { '2.6.0': [ { unreleased: true }, + { title: 'Code review refactor pass on the torrent / usenet flow', desc: 'cleanup commits before review. Lifted the shared album-pick + staging-collision helpers out of torrent.py into a new core/download_plugins/album_bundle.py module so usenet.py no longer reaches into a sibling plugin\'s private surface. Lifted the ~90-line inline album-bundle gate out of run_full_missing_tracks_process into core/downloads/album_bundle_dispatch.py with a clean pure-predicate / inject-deps design so the gate is unit-testable in isolation. Replaced the staging matcher\'s direct download_batches import with an injected get_batch_field accessor on StagingDeps. Made the 6h poll timeout configurable via download_source.album_bundle_timeout_seconds. Added atomic .tmp + rename copy so the Auto-Import worker can never observe a partial audio file during the album-bundle copy loop. 49 new tests across album_bundle, album_bundle_dispatch, and staging-provenance modules pin the contracts.' }, { title: 'Hybrid mode skips torrent / usenet on album batches', desc: 'follow-up to the album-bundle flow. When download mode is set to Hybrid AND the batch is an album, the per-track search loop now silently strips torrent and usenet from the source chain — they\'re release-level sources that can\'t match per-track meaningfully, and the album-bundle handling only fires in single-source mode. Without the skip, hybrid + torrent-first would fire N redundant Prowlarr searches per album and rely on Auto-Import sweeping Staging to recover. Cleaner now: hybrid falls straight through to per-track-compatible sources (Soulseek / streaming) for albums, and torrent / usenet still get a shot for single-track / wishlist / basic-search use cases where per-track makes sense.' }, { title: 'Album-bundle flow for torrent / usenet downloads', desc: 'fixes the core architectural problem with indexer-based sources. Prowlarr returns release-level torrents — searching per-track for "Luther (with SZA)" against the GNX album torrent scores near-zero and the orchestrator rejects every candidate. New gated flow: when downloading an album AND torrent or usenet is the single active source (not hybrid), SoulSync now does ONE Prowlarr search for the whole release, picks the best torrent (prefers FLAC, high seeders, reasonable size — drops single-track torrents that snuck in), hands it to your torrent / usenet client, walks the resulting audio files (extracting .zip/.rar/.tar if needed), and drops them all into the staging folder. The existing per-track staging matcher then imports each one to the library by fuzzy title match — same path as the Auto-Import flow. Gate is strictly opt-in: per-track flow is completely untouched for hybrid mode, non-album downloads, and every other source. 5 new tests cover the album picker (seeded-FLAC preference, size floor for single-track torrents, fallback when all candidates are small) and the staging path collision handler.' }, { title: 'Filesystem-access heads-up for torrent / usenet sources', desc: 'new advisory card on the Indexers & Downloaders tab explaining the cleanest setup: point ALL your downloaders (Soulseek, qBittorrent, SABnzbd / NZBGet) at the same download folder. One folder, one mount, everything just works. Bare-metal needs no change; Docker users can reuse the existing ./downloads mount and just configure each client to write there. docker-compose.yml updated to call this out as the easiest path, with optional commented placeholders for users who prefer separate folders per protocol.' }, From 8b0de9eb76c596f81c90c6c20526b2388075166a Mon Sep 17 00:00:00 2001 From: Broque Thomas <26755000+Nezreka@users.noreply.github.com> Date: Wed, 20 May 2026 21:39:06 -0700 Subject: [PATCH 15/25] fix(downloads): harden album bundle staging Route torrent and Usenet album bundles through private per-batch staging so Auto-Import cannot race public staging or duplicate imports. Expose album-bundle progress in batch status and render it on the Downloads page while the external client is still downloading. Tighten release handoff safety by rejecting archive path traversal, ignoring torrent candidates without a usable URL, and skipping Soulseek source reuse for torrent/Usenet batches. Tests: .venv/bin/python -m pytest tests/downloads/test_downloads_status.py tests/test_album_bundle_dispatch.py tests/downloads/test_downloads_staging.py tests/test_torrent_usenet_plugins.py --- core/archive_pipeline.py | 6 +++ core/download_plugins/torrent.py | 3 +- core/downloads/album_bundle_dispatch.py | 14 +++++- core/downloads/staging.py | 17 ++++++++ core/downloads/status.py | 46 +++++++++++++++++++- tests/downloads/test_downloads_staging.py | 52 +++++++++++++++++++++++ tests/downloads/test_downloads_status.py | 52 +++++++++++++++++++++++ tests/test_album_bundle_dispatch.py | 24 ++++++++++- tests/test_torrent_usenet_plugins.py | 18 +++++++- web_server.py | 18 ++++++-- webui/static/pages-extra.js | 44 ++++++++++++++++++- webui/static/style.css | 3 ++ 12 files changed, 286 insertions(+), 11 deletions(-) diff --git a/core/archive_pipeline.py b/core/archive_pipeline.py index 12248e93..138e520c 100644 --- a/core/archive_pipeline.py +++ b/core/archive_pipeline.py @@ -225,6 +225,12 @@ def _extract_7z(archive_path: Path, dest: Path) -> Optional[Path]: return None try: with py7zr.SevenZipFile(archive_path, 'r') as sz: + dest_resolved = dest.resolve() + for name in sz.getnames(): + target = (dest_resolved / name).resolve() + if dest_resolved not in target.parents and target != dest_resolved: + logger.error("archive_pipeline: refusing path-traversal 7z member %r", name) + return None sz.extractall(path=dest) return dest except Exception as e: diff --git a/core/download_plugins/torrent.py b/core/download_plugins/torrent.py index b2e1187e..a04c8247 100644 --- a/core/download_plugins/torrent.py +++ b/core/download_plugins/torrent.py @@ -465,7 +465,8 @@ class TorrentDownloadPlugin(DownloadSourcePlugin): result['error'] = f'Prowlarr search failed: {e}' return result - candidates = [r for r in search_results if r.protocol == 'torrent'] + candidates = [r for r in search_results + if r.protocol == 'torrent' and (r.magnet_uri or r.download_url)] if not candidates: result['error'] = f'No torrent results found for "{query}"' return result diff --git a/core/downloads/album_bundle_dispatch.py b/core/downloads/album_bundle_dispatch.py index 71692f6c..6a74914f 100644 --- a/core/downloads/album_bundle_dispatch.py +++ b/core/downloads/album_bundle_dispatch.py @@ -31,6 +31,7 @@ testable without touching live runtime state. from __future__ import annotations import logging +from pathlib import Path from typing import Any, Callable, Optional, Protocol logger = logging.getLogger(__name__) @@ -127,7 +128,11 @@ def try_dispatch( ) return False - staging_dir = config_get('import.staging_path', './Staging') or './Staging' + staging_root = config_get( + 'download_source.album_bundle_staging_path', + 'storage/album_bundle_staging', + ) or 'storage/album_bundle_staging' + staging_dir = str(Path(staging_root) / _safe_batch_dirname(batch_id)) logger.info( "[Album Bundle] Engaging %s album flow for '%s' by '%s' -> %s", mode, album_name, artist_name, staging_dir, @@ -136,6 +141,8 @@ def try_dispatch( 'phase': 'album_downloading', 'album_bundle_state': 'searching', 'album_bundle_source': mode, + 'album_bundle_staging_path': staging_dir, + 'album_bundle_private_staging': True, }) def _emit(payload): @@ -177,3 +184,8 @@ def try_dispatch( # task rows. Those tasks will hit try_staging_match and pull the # files we just staged. return False + + +def _safe_batch_dirname(batch_id: str) -> str: + safe = ''.join(ch if ch.isalnum() or ch in ('-', '_') else '_' for ch in str(batch_id or 'batch')) + return safe or 'batch' diff --git a/core/downloads/staging.py b/core/downloads/staging.py index dca686ef..ecd38e40 100644 --- a/core/downloads/staging.py +++ b/core/downloads/staging.py @@ -160,6 +160,14 @@ def try_staging_match(task_id, batch_id, track, deps: StagingDeps): except Exception as _exc: logger.debug("get_batch_field failed: %s", _exc) _provenance_username = _provenance_override or 'staging' + _private_album_bundle_staging = False + if batch_id and deps.get_batch_field is not None: + try: + _private_album_bundle_staging = bool( + deps.get_batch_field(batch_id, 'album_bundle_private_staging') + ) + except Exception as _exc: + logger.debug("get_batch_field failed: %s", _exc) with tasks_lock: if task_id in download_tasks: download_tasks[task_id]['status'] = 'post_processing' @@ -167,6 +175,15 @@ def try_staging_match(task_id, batch_id, track, deps: StagingDeps): download_tasks[task_id]['username'] = _provenance_username download_tasks[task_id]['staging_match'] = True + if _private_album_bundle_staging: + try: + os.remove(best_match['full_path']) + logger.debug("[Staging] Removed private album-bundle staging file: %s", best_match['full_path']) + except FileNotFoundError: + pass + except Exception as _exc: + logger.debug("[Staging] Could not remove private album-bundle staging file: %s", _exc) + # Run post-processing (tagging, AcoustID verification, path building) context_key = f"staging_{task_id}" with tasks_lock: diff --git a/core/downloads/status.py b/core/downloads/status.py index 7a0ec08c..6c9fac4f 100644 --- a/core/downloads/status.py +++ b/core/downloads/status.py @@ -247,6 +247,9 @@ def build_batch_status_data(batch_id: str, batch: dict, live_transfers_lookup: d "playlist_id": batch.get('playlist_id'), # Include playlist_id for rehydration "playlist_name": batch.get('playlist_name'), # Include playlist_name for reference } + album_bundle = _build_album_bundle_status(batch) + if album_bundle: + response_data['album_bundle'] = album_bundle if response_data["phase"] == 'analysis': response_data['analysis_progress'] = { @@ -610,7 +613,7 @@ def build_unified_downloads_response(limit: int, deps: StatusDeps) -> dict: for bid, batch in download_batches.items(): queue = batch.get('queue', []) statuses = [download_tasks[tid]['status'] for tid in queue if tid in download_tasks] - batch_summaries.append({ + summary = { 'batch_id': bid, 'playlist_id': batch.get('playlist_id', ''), 'batch_name': batch.get('playlist_name') or batch.get('album_name') or '', @@ -621,7 +624,11 @@ def build_unified_downloads_response(limit: int, deps: StatusDeps) -> dict: 'failed': sum(1 for s in statuses if s in ('failed', 'not_found', 'cancelled')), 'active': sum(1 for s in statuses if s in ('downloading', 'searching', 'post_processing')), 'queued': sum(1 for s in statuses if s in ('queued', 'pending')), - }) + } + album_bundle = _build_album_bundle_status(batch) + if album_bundle: + summary['album_bundle'] = album_bundle + batch_summaries.append(summary) return { 'success': True, @@ -630,3 +637,38 @@ def build_unified_downloads_response(limit: int, deps: StatusDeps) -> dict: 'batches': batch_summaries, 'timestamp': time.time(), } + + +def _build_album_bundle_status(batch: dict) -> dict: + """Return public batch-level status for torrent/Usenet album-bundle work.""" + state = batch.get('album_bundle_state') + if not state: + return {} + + status = { + 'state': state, + 'source': batch.get('album_bundle_source'), + 'release': batch.get('album_bundle_release'), + 'progress': batch.get('album_bundle_progress'), + 'progress_percent': _album_bundle_progress_percent( + batch.get('album_bundle_progress') + ), + 'speed': batch.get('album_bundle_speed'), + 'downloaded': batch.get('album_bundle_downloaded'), + 'size': batch.get('album_bundle_size'), + 'seeders': batch.get('album_bundle_seeders'), + 'grabs': batch.get('album_bundle_grabs'), + 'count': batch.get('album_bundle_count'), + } + return {key: value for key, value in status.items() if value is not None} + + +def _album_bundle_progress_percent(value: Any) -> int: + try: + progress = float(value) + except (TypeError, ValueError): + return 0 + + if progress <= 1: + progress *= 100 + return max(0, min(100, int(round(progress)))) diff --git a/tests/downloads/test_downloads_staging.py b/tests/downloads/test_downloads_staging.py index d11153c4..e75e633c 100644 --- a/tests/downloads/test_downloads_staging.py +++ b/tests/downloads/test_downloads_staging.py @@ -60,6 +60,7 @@ def _build_deps( transfer_path, staging_files=None, post_process_calls=None, + get_batch_field=None, ): post_process_calls = post_process_calls if post_process_calls is not None else [] deps = ds.StagingDeps( @@ -68,6 +69,7 @@ def _build_deps( get_staging_file_cache=lambda batch_id: staging_files or [], docker_resolve_path=lambda p: p, # passthrough post_process_matched_download_with_verification=lambda *a, **kw: post_process_calls.append((a, kw)), + get_batch_field=get_batch_field, ) deps._post_process_calls = post_process_calls return deps @@ -157,6 +159,56 @@ def test_exact_match_copies_to_transfer_and_marks_post_processing(tmp_path): assert context_key == 'staging_t4' +def test_private_album_bundle_staging_source_is_removed_after_claim(tmp_path): + src_file = tmp_path / 'private' / 'Hello.flac' + src_file.parent.mkdir() + src_file.write_bytes(b'fake audio') + + transfer_dir = tmp_path / 'transfer' + + def get_batch_field(_batch_id, field): + if field == 'album_bundle_source': + return 'torrent' + if field == 'album_bundle_private_staging': + return True + return None + + deps = _build_deps( + transfer_path=str(transfer_dir), + staging_files=[ + {'full_path': str(src_file), 'title': 'Hello', 'artist': 'Artist One'}, + ], + get_batch_field=get_batch_field, + ) + _seed_task('t_private') + + result = ds.try_staging_match('t_private', 'b_private', _Track(), deps) + + assert result is True + assert (transfer_dir / 'Hello.flac').exists() + assert not src_file.exists() + assert download_tasks['t_private']['username'] == 'torrent' + + +def test_public_staging_source_is_kept_after_match(tmp_path): + src_file = tmp_path / 'staging' / 'Hello.flac' + src_file.parent.mkdir() + src_file.write_bytes(b'fake audio') + + deps = _build_deps( + transfer_path=str(tmp_path / 'transfer'), + staging_files=[ + {'full_path': str(src_file), 'title': 'Hello', 'artist': 'Artist One'}, + ], + ) + _seed_task('t_public') + + result = ds.try_staging_match('t_public', 'b_public', _Track(), deps) + + assert result is True + assert src_file.exists() + + def test_existing_file_in_transfer_gets_staging_suffix(tmp_path): """If destination already exists, suffix '_staging' added to avoid overwrite.""" src_file = tmp_path / 'staging' / 'Hello.flac' diff --git a/tests/downloads/test_downloads_status.py b/tests/downloads/test_downloads_status.py index aa690866..cc9a5d15 100644 --- a/tests/downloads/test_downloads_status.py +++ b/tests/downloads/test_downloads_status.py @@ -85,6 +85,38 @@ def test_analysis_phase_includes_analysis_progress_and_results(): assert out['analysis_results'] == [{'track_index': 0, 'found': True}] +def test_album_downloading_phase_exposes_bundle_progress_without_task_safety_valve(): + deps, _ = _build_deps(config=_FakeConfig({'soulseek.download_timeout': 1})) + download_tasks['t1'] = { + 'track_index': 0, + 'status': 'queued', + 'track_info': {}, + 'status_change_time': 0, + } + batch = { + 'phase': 'album_downloading', + 'queue': ['t1'], + 'album_bundle_state': 'downloading', + 'album_bundle_source': 'torrent', + 'album_bundle_release': 'GNX [FLAC]', + 'album_bundle_progress': 0.42, + 'album_bundle_speed': 2048, + 'album_bundle_size': 4096, + } + out = st.build_batch_status_data('b1', batch, {}, deps) + assert out['album_bundle'] == { + 'state': 'downloading', + 'source': 'torrent', + 'release': 'GNX [FLAC]', + 'progress': 0.42, + 'progress_percent': 42, + 'speed': 2048, + 'size': 4096, + } + assert 'tasks' not in out + assert download_tasks['t1']['status'] == 'queued' + + def test_complete_phase_includes_wishlist_summary_when_present(): deps, _ = _build_deps() batch = { @@ -340,6 +372,26 @@ def test_batched_status_no_filter_returns_all_batches(): assert set(out['batches'].keys()) == {'b1', 'b2'} +def test_unified_downloads_response_includes_album_bundle_summary(): + deps, _ = _build_deps() + download_batches['b1'] = { + 'phase': 'album_downloading', + 'playlist_id': 'pl1', + 'playlist_name': 'GNX', + 'queue': [], + 'album_bundle_state': 'downloading', + 'album_bundle_source': 'torrent', + 'album_bundle_progress': 75, + } + out = st.build_unified_downloads_response(300, deps) + assert out['batches'][0]['album_bundle'] == { + 'state': 'downloading', + 'source': 'torrent', + 'progress': 75, + 'progress_percent': 75, + } + + def test_batched_status_metadata_present(): deps, _ = _build_deps() download_batches['b1'] = {'phase': 'unknown'} diff --git a/tests/test_album_bundle_dispatch.py b/tests/test_album_bundle_dispatch.py index d60d0e84..68579f6d 100644 --- a/tests/test_album_bundle_dispatch.py +++ b/tests/test_album_bundle_dispatch.py @@ -187,14 +187,36 @@ def test_dispatch_success_returns_false_so_per_track_can_run() -> None: args = plugin.download_album_to_staging.call_args assert args.args[0] == 'GNX' assert args.args[1] == 'Kendrick Lamar' - assert args.args[2] == '/staging/path' + assert args.args[2].replace('\\', '/').endswith('storage/album_bundle_staging/b1') # Phase transitioned through searching → analysis. assert state.fields['phase'] == 'analysis' assert state.fields['album_bundle_state'] == 'staged' assert state.fields['album_bundle_source'] == 'torrent' + assert state.fields['album_bundle_private_staging'] is True + assert state.fields['album_bundle_staging_path'].replace('\\', '/').endswith('storage/album_bundle_staging/b1') assert state.failed_with == '' +def test_dispatch_uses_configured_private_album_bundle_staging_root() -> None: + state = _FakeState() + plugin = MagicMock() + plugin.download_album_to_staging.return_value = {'success': True, 'files': ['/tmp/a.flac']} + + try_dispatch( + batch_id='batch:with/slash', is_album=True, + album_context={'name': 'GNX'}, artist_context={'name': 'Kendrick Lamar'}, + config_get=_config({ + 'download_source.mode': 'torrent', + 'download_source.album_bundle_staging_path': '/private/staging', + }), + plugin_resolver=lambda _name: plugin, state=state, + ) + + staging_arg = plugin.download_album_to_staging.call_args.args[2].replace('\\', '/') + assert staging_arg == '/private/staging/batch_with_slash' + assert state.fields['album_bundle_staging_path'].replace('\\', '/') == staging_arg + + def test_dispatch_failure_returns_true_so_master_stops() -> None: state = _FakeState() plugin = MagicMock() diff --git a/tests/test_torrent_usenet_plugins.py b/tests/test_torrent_usenet_plugins.py index 40779240..57f11079 100644 --- a/tests/test_torrent_usenet_plugins.py +++ b/tests/test_torrent_usenet_plugins.py @@ -12,7 +12,7 @@ from __future__ import annotations import asyncio from pathlib import Path from typing import Optional -from unittest.mock import MagicMock, patch +from unittest.mock import AsyncMock, MagicMock, patch import pytest @@ -452,6 +452,22 @@ def test_torrent_album_to_staging_short_circuits_when_not_configured() -> None: assert 'not configured' in outcome['error'].lower() +def test_torrent_album_to_staging_ignores_candidates_without_download_url(tmp_path: Path) -> None: + plugin = TorrentDownloadPlugin() + fake_adapter = MagicMock() + fake_adapter.is_configured.return_value = True + with patch.object(plugin, 'is_configured', return_value=True), \ + patch.object(plugin._prowlarr, 'search', new=AsyncMock(return_value=[ + _make_torrent_result(download_url=None, magnet_uri=None), + ])), \ + patch('core.download_plugins.torrent.get_active_torrent_adapter', return_value=fake_adapter): + outcome = plugin.download_album_to_staging('GNX', 'Kendrick Lamar', str(tmp_path)) + + assert outcome['success'] is False + assert 'No torrent results' in outcome['error'] + fake_adapter.add_torrent.assert_not_called() + + def test_registry_includes_torrent_and_usenet() -> None: """The registry decides what shows up in the orchestrator's iteration helpers. If we forget to register a new plugin the diff --git a/web_server.py b/web_server.py index 2b151d1f..0e7e2d8e 100644 --- a/web_server.py +++ b/web_server.py @@ -16463,9 +16463,10 @@ def _get_staging_file_cache(batch_id): """Scan staging folder once per batch and cache the results.""" with _staging_cache_lock: if batch_id in _staging_cache: - return _staging_cache[batch_id] + cached = _staging_cache[batch_id] + return [f for f in cached if os.path.exists(f.get('full_path', ''))] - staging_path = get_staging_path() + staging_path = _get_album_bundle_staging_path(batch_id) or get_staging_path() if not os.path.isdir(staging_path): with _staging_cache_lock: _staging_cache[batch_id] = [] @@ -16496,6 +16497,17 @@ def _get_staging_file_cache(batch_id): return files +def _get_album_bundle_staging_path(batch_id): + """Return the private album-bundle staging path for this batch, if any.""" + if not batch_id: + return None + with tasks_lock: + row = download_batches.get(batch_id) + if isinstance(row, dict) and row.get('album_bundle_private_staging'): + return row.get('album_bundle_staging_path') + return None + + # Staging-folder match shortcut lives in core/downloads/staging.py. from core.downloads import staging as _downloads_staging @@ -16659,7 +16671,7 @@ def _store_batch_source(batch_id, username, filename): """Browse the successful download's folder and store results on the batch for reuse.""" _sr = source_reuse_logger _sr.info(f"_store_batch_source called: batch={batch_id}, user={username}, file={filename}") - if not batch_id or username in ('youtube', 'tidal', 'qobuz', 'hifi', 'deezer_dl', 'lidarr', 'soundcloud', 'amazon'): + if not batch_id or username in ('youtube', 'tidal', 'qobuz', 'hifi', 'deezer_dl', 'lidarr', 'soundcloud', 'amazon', 'torrent', 'usenet'): _sr.info(f"Skipped — no batch_id or streaming source ({username})") return diff --git a/webui/static/pages-extra.js b/webui/static/pages-extra.js index b794ddef..915040f1 100644 --- a/webui/static/pages-extra.js +++ b/webui/static/pages-extra.js @@ -2445,6 +2445,34 @@ function _adlEsc(str) { return String(str).replace(/&/g, '&').replace(//g, '>').replace(/"/g, '"'); } +function _adlBundleProgressPercent(bundle) { + if (!bundle) return 0; + const raw = bundle.progress_percent ?? bundle.progress ?? 0; + let progress = Number(raw); + if (!Number.isFinite(progress)) progress = 0; + if (progress <= 1) progress *= 100; + return Math.max(0, Math.min(100, Math.round(progress))); +} + +function _adlFormatBytes(bytes) { + const value = Number(bytes); + if (!Number.isFinite(value) || value <= 0) return ''; + const units = ['B', 'KB', 'MB', 'GB', 'TB']; + let size = value; + let unit = 0; + while (size >= 1024 && unit < units.length - 1) { + size /= 1024; + unit += 1; + } + const decimals = size >= 10 || unit === 0 ? 0 : 1; + return `${size.toFixed(decimals)} ${units[unit]}`; +} + +function _adlFormatSpeed(bytesPerSecond) { + const formatted = _adlFormatBytes(bytesPerSecond); + return formatted ? `${formatted}/s` : ''; +} + async function adlClearCompleted() { try { const resp = await fetch('/api/downloads/clear-completed', { method: 'POST' }); @@ -2504,12 +2532,16 @@ function _adlRenderBatchPanel() { const colorStyle = colorIdx >= 0 ? `border-left-color: rgba(var(--batch-color-${colorIdx}), 0.6)` : ''; const isExpanded = _adlExpandedBatches.has(batch.batch_id); const isFiltered = _adlFilterBatchId === batch.batch_id; + const albumBundle = batch.album_bundle || null; + const bundleProgress = _adlBundleProgressPercent(albumBundle); const total = batch.total || 1; const done = batch.completed + batch.failed; - const pct = Math.round((done / total) * 100); + const pct = batch.phase === 'album_downloading' + ? bundleProgress + : Math.round((done / total) * 100); const hasFailed = batch.failed > 0; const isTerminal = batch.phase === 'complete' || batch.phase === 'cancelled' || batch.phase === 'error'; - const isActive = batch.phase === 'downloading' && batch.active > 0; + const isActive = (batch.phase === 'downloading' && batch.active > 0) || batch.phase === 'album_downloading'; // Fade progress for completing batches let fadeStyle = ''; @@ -2532,6 +2564,14 @@ function _adlRenderBatchPanel() { if (batch.phase === 'analysis') { phaseText = 'Analyzing...'; phaseIcon = ''; + } else if (batch.phase === 'album_downloading') { + const source = albumBundle && albumBundle.source ? `${albumBundle.source} ` : ''; + const release = albumBundle && albumBundle.release ? ` - ${albumBundle.release}` : ''; + const speed = _adlFormatSpeed(albumBundle && albumBundle.speed); + const size = _adlFormatBytes(albumBundle && albumBundle.size); + const detail = speed || size ? ` (${[speed, size].filter(Boolean).join(' of ')})` : ''; + phaseText = `${source}album ${albumBundle && albumBundle.state ? albumBundle.state : 'downloading'} ${bundleProgress}%${release}${detail}`; + phaseIcon = ''; } else if (batch.phase === 'downloading') { phaseText = `${batch.completed}/${total} tracks`; if (batch.active > 0) phaseIcon = ''; diff --git a/webui/static/style.css b/webui/static/style.css index 2c24fcb2..cad3e987 100644 --- a/webui/static/style.css +++ b/webui/static/style.css @@ -58212,6 +58212,9 @@ body.reduce-effects *::after { font-size: 0.7rem; color: rgba(255, 255, 255, 0.4); margin-top: 2px; + white-space: nowrap; + overflow: hidden; + text-overflow: ellipsis; } .adl-batch-card-source { From 6c9b43225a273b3701757de4fd2a5a557c1ffc8f Mon Sep 17 00:00:00 2001 From: Broque Thomas <26755000+Nezreka@users.noreply.github.com> Date: Thu, 21 May 2026 14:22:21 -0700 Subject: [PATCH 16/25] Add torrent and usenet release staging support Adds torrent/usenet as release-oriented download sources with album-bundle staging, live progress reporting, and post-processing that selects the requested audio file from completed releases instead of blindly importing the first file. Keeps album-bundle behavior gated to single-source torrent/usenet album downloads, excludes release sources from hybrid album per-track searches, and allows hybrid non-album tracks to use release results safely. Improves staged-release matching for featured/bonus track filenames while preserving version mismatches, records torrent/usenet provenance in library history, and updates service/status UI labels. Covers the flow with focused lifecycle, status, staging, validation, task worker, post-processing, and import side-effect tests. --- core/connection_test.py | 12 ++ core/download_plugins/torrent.py | 8 +- core/download_plugins/types.py | 1 + core/download_plugins/usenet.py | 2 + core/downloads/lifecycle.py | 45 +++++ core/downloads/monitor.py | 123 ++++++++----- core/downloads/post_processing.py | 158 +++++++++++++++- core/downloads/staging.py | 97 +++++++++- core/downloads/status.py | 47 +++-- core/downloads/task_worker.py | 77 +++++++- core/downloads/validation.py | 49 ++++- core/imports/side_effects.py | 4 + tests/downloads/test_downloads_lifecycle.py | 42 +++++ .../test_downloads_post_processing.py | 155 ++++++++++++++++ tests/downloads/test_downloads_staging.py | 174 ++++++++++++++++++ tests/downloads/test_downloads_status.py | 126 +++++++++++++ tests/downloads/test_downloads_task_worker.py | 54 +++++- tests/downloads/test_downloads_validation.py | 44 +++++ tests/imports/test_import_side_effects.py | 38 ++++ tests/test_manual_pick_no_auto_retry.py | 129 +++++++++++++ web_server.py | 52 +++++- webui/static/downloads.js | 57 ++++++ webui/static/shared-helpers.js | 8 +- 23 files changed, 1399 insertions(+), 103 deletions(-) diff --git a/core/connection_test.py b/core/connection_test.py index c470cfec..745caa2a 100644 --- a/core/connection_test.py +++ b/core/connection_test.py @@ -191,6 +191,12 @@ def run_service_test(service, test_config): 'tidal': "Tidal download source ready.", 'qobuz': "Qobuz download source ready.", 'hifi': "HiFi download source ready.", + 'deezer_dl': "Deezer download source ready.", + 'amazon': "Amazon download source ready.", + 'lidarr': "Lidarr download source ready.", + 'soundcloud': "SoundCloud download source ready.", + 'torrent': "Torrent download source ready.", + 'usenet': "Usenet download source ready.", 'hybrid': "Download sources ready (Hybrid mode)." } message = mode_messages.get(download_mode, "Download source connected.") @@ -203,6 +209,12 @@ def run_service_test(service, test_config): 'tidal': "Tidal download source not available. Check authentication.", 'qobuz': "Qobuz download source not available. Check authentication.", 'hifi': "HiFi download source not available. Public API instances may be down.", + 'deezer_dl': "Deezer download source not available. Check authentication.", + 'amazon': "Amazon download source not available.", + 'lidarr': "Lidarr download source not available. Check Lidarr URL and API key.", + 'soundcloud': "SoundCloud download source not available.", + 'torrent': "Torrent download source not available. Check Prowlarr and torrent client settings.", + 'usenet': "Usenet download source not available. Check Prowlarr and usenet client settings.", 'hybrid': "Could not connect to download sources. Check configuration." } error = mode_errors.get(download_mode, "Download source connection failed.") diff --git a/core/download_plugins/torrent.py b/core/download_plugins/torrent.py index a04c8247..388cbe98 100644 --- a/core/download_plugins/torrent.py +++ b/core/download_plugins/torrent.py @@ -16,8 +16,9 @@ Two flows: polls the adapter for completion. 3. On completion the thread walks the adapter-reported save path via ``archive_pipeline.collect_audio_after_extraction`` and - marks the download succeeded with the first audio file as the - primary ``file_path``. + exposes the full audio-file list. Post-processing can then pick + the requested track from a completed release instead of importing + the first file blindly. **Album-bundle flow** (album-context batch downloads — wired in ``core/downloads/master.py``) — @@ -257,6 +258,7 @@ class TorrentDownloadPlugin(DownloadSourcePlugin): 'transferred': 0, 'speed': 0, 'file_path': None, + 'audio_files': [], 'torrent_hash': None, 'error': None, } @@ -353,6 +355,7 @@ class TorrentDownloadPlugin(DownloadSourcePlugin): row['state'] = 'Completed, Succeeded' row['progress'] = 100.0 row['file_path'] = str(primary) + row['audio_files'] = [str(path) for path in audio_files] logger.info("Torrent download complete: %s -> %s (%d audio files)", download_id[:8], primary.name, len(audio_files)) @@ -659,4 +662,5 @@ def _row_to_status(row: Dict[str, Any]) -> DownloadStatus: speed=int(row.get('speed', 0)), time_remaining=None, file_path=row.get('file_path'), + audio_files=row.get('audio_files') or None, ) diff --git a/core/download_plugins/types.py b/core/download_plugins/types.py index 46e250c3..59d52913 100644 --- a/core/download_plugins/types.py +++ b/core/download_plugins/types.py @@ -197,3 +197,4 @@ class DownloadStatus: speed: int time_remaining: Optional[int] = None file_path: Optional[str] = None + audio_files: Optional[List[str]] = None diff --git a/core/download_plugins/usenet.py b/core/download_plugins/usenet.py index ea0f4397..419bc380 100644 --- a/core/download_plugins/usenet.py +++ b/core/download_plugins/usenet.py @@ -190,6 +190,7 @@ class UsenetDownloadPlugin(DownloadSourcePlugin): 'transferred': 0, 'speed': 0, 'file_path': None, + 'audio_files': [], 'job_id': None, 'error': None, } @@ -281,6 +282,7 @@ class UsenetDownloadPlugin(DownloadSourcePlugin): row['state'] = 'Completed, Succeeded' row['progress'] = 100.0 row['file_path'] = str(primary) + row['audio_files'] = [str(path) for path in audio_files] logger.info("Usenet download complete: %s -> %s (%d audio files)", download_id[:8], primary.name, len(audio_files)) diff --git a/core/downloads/lifecycle.py b/core/downloads/lifecycle.py index 2a29feec..5891e8be 100644 --- a/core/downloads/lifecycle.py +++ b/core/downloads/lifecycle.py @@ -26,9 +26,11 @@ Lifted verbatim from web_server.py. Dependencies injected via from __future__ import annotations import logging +import shutil import time import traceback from dataclasses import dataclass +from pathlib import Path from typing import Any, Callable, Optional from core.downloads.history import record_sync_history_completion @@ -42,6 +44,47 @@ from core.runtime_state import ( logger = logging.getLogger(__name__) +def _safe_batch_dirname(batch_id: str) -> str: + safe = ''.join(ch if ch.isalnum() or ch in ('-', '_') else '_' for ch in str(batch_id or 'batch')) + return safe or 'batch' + + +def _cleanup_private_album_bundle_staging(batch_id: str, batch: dict) -> None: + """Best-effort cleanup for torrent/usenet private staging copies. + + The torrent/usenet clients keep their own completed download folders. + This only removes SoulSync's per-batch copy under album-bundle staging. + """ + if not batch.get('album_bundle_private_staging'): + return + if (batch.get('album_bundle_source') or '').lower() not in ('torrent', 'usenet'): + return + + staging_path = batch.get('album_bundle_staging_path') + if not staging_path: + return + + path = Path(staging_path) + expected_name = _safe_batch_dirname(batch_id) + if path.name != expected_name: + logger.warning( + "[Album Bundle] Refusing to clean private staging path with unexpected name: %s", + staging_path, + ) + return + if not path.exists(): + return + if not path.is_dir(): + logger.warning("[Album Bundle] Refusing to clean non-directory staging path: %s", staging_path) + return + + try: + shutil.rmtree(path) + logger.info("[Album Bundle] Cleaned private staging folder for batch %s: %s", batch_id, staging_path) + except Exception as exc: + logger.warning("[Album Bundle] Could not clean private staging folder %s: %s", staging_path, exc) + + @dataclass class LifecycleDeps: """Bundle of cross-cutting deps the batch lifecycle needs.""" @@ -398,6 +441,7 @@ def on_download_completed(batch_id: str, task_id: str, success: bool, deps: Life logger.info(f"[Batch Manager] Batch {batch_id} complete - stopping monitor") deps.download_monitor.stop_monitoring(batch_id) + _cleanup_private_album_bundle_staging(batch_id, batch) # M3U REGENERATION: Regenerate M3U with real library paths now that # all post-processing (tagging, moving, DB writes) is complete. @@ -606,6 +650,7 @@ def check_batch_completion_v2(batch_id: str, deps: LifecycleDeps) -> Optional[bo logger.info(f"[Completion Check V2] Batch {batch_id} complete - stopping monitor") deps.download_monitor.stop_monitoring(batch_id) + _cleanup_private_album_bundle_staging(batch_id, batch) # REPAIR: Scan all album folders from this batch for track number issues if deps.repair_worker: diff --git a/core/downloads/monitor.py b/core/downloads/monitor.py index 62d515e5..c4b645f5 100644 --- a/core/downloads/monitor.py +++ b/core/downloads/monitor.py @@ -34,6 +34,32 @@ _start_next_batch_of_downloads = None _orphaned_download_keys = None missing_download_executor = None download_orchestrator = None +_RELEASE_SOURCE_NAMES = frozenset(('torrent', 'usenet')) + + +def _download_id_key(download_id): + return f"download_id::{download_id}" if download_id else None + + +def _is_release_task(task): + ti = task.get('track_info') if isinstance(task.get('track_info'), dict) else {} + username = task.get('username') or ti.get('username') + return username in _RELEASE_SOURCE_NAMES + + +def _lookup_live_info(task, live_transfers_lookup): + ti = task.get('track_info') if isinstance(task.get('track_info'), dict) else {} + download_id = task.get('download_id') + if _is_release_task(task): + by_id = live_transfers_lookup.get(_download_id_key(download_id)) + if by_id: + return by_id + + task_filename = task.get('filename') or ti.get('filename') + task_username = task.get('username') or ti.get('username') + if not task_filename or not task_username: + return None + return live_transfers_lookup.get(_make_context_key(task_username, task_filename)) def init( @@ -139,55 +165,64 @@ class WebUIDownloadMonitor: for task_id in download_batches[batch_id].get('queue', []): task = download_tasks.get(task_id) - if not task or task['status'] not in ['downloading', 'queued']: + if not task: + continue + release_recoverable = ( + _is_release_task(task) + and task.get('download_id') + and task.get('status') in ['failed', 'not_found'] + ) + if task['status'] not in ['downloading', 'queued'] and not release_recoverable: continue # Check for timeouts and errors - retries handled directly in _should_retry_task # If _should_retry_task returns True, it means retries were exhausted - retry_exhausted = self._should_retry_task(task_id, task, live_transfers_lookup, current_time, deferred_ops) + retry_exhausted = False + if not release_recoverable: + retry_exhausted = self._should_retry_task(task_id, task, live_transfers_lookup, current_time, deferred_ops) # Collect exhausted tasks to handle outside lock (prevents deadlock) if retry_exhausted: exhausted_tasks.append((batch_id, task_id)) - # ENHANCED: Check for successful completions (especially YouTube) - task_filename = task.get('filename') or task.get('track_info', {}).get('filename') - task_username = task.get('username') or task.get('track_info', {}).get('username') + # ENHANCED: Check for successful completions (especially YouTube). + # Release-style sources can report a completed audio file + # name that differs from the original indexer URL/title + # stored on the task, so prefer the stable download_id. + live_info = _lookup_live_info(task, live_transfers_lookup) - if task_filename and task_username: - lookup_key = _make_context_key(task_username, task_filename) - live_info = live_transfers_lookup.get(lookup_key) - - if live_info: - state = live_info.get('state', '') - # Trigger post-processing if download is completed successfully - # slskd uses compound states like 'Completed, Succeeded' - use substring matching - # Must exclude error states first (matching _build_batch_status_data's prioritized checking) - has_error = ('Errored' in state or 'Failed' in state or 'Rejected' in state or 'TimedOut' in state) - has_completion = ('Completed' in state or 'Succeeded' in state) - # Verify bytes actually transferred before trusting state string. - # slskd can report "Completed" before the full file is flushed to disk, - # or on connection drops that leave a partial file. - if has_completion and not has_error: - expected_size = live_info.get('size', 0) - transferred = live_info.get('bytesTransferred', 0) - if expected_size > 0 and transferred < expected_size: - if not task.get('_incomplete_warned'): - logger.debug(f"Monitor: {task_id} state={state} but bytes incomplete ({transferred}/{expected_size}) — waiting") - task['_incomplete_warned'] = True - continue - if has_completion and not has_error and task['status'] == 'downloading': - task.pop('_incomplete_warned', None) - # CRITICAL FIX: Transition to 'post_processing' HERE so downloads - # don't depend on browser polling to trigger post-processing. - # Previously, post-processing was only submitted by _build_batch_status_data - # (called from browser-polled endpoints), meaning closing the browser - # left tasks stuck in 'downloading' forever. - task['status'] = 'post_processing' - task['status_change_time'] = current_time - logger.info(f"Monitor detected completed download for {task_id} ({state}) - submitting post-processing") - # Collect for handling outside the lock to prevent deadlock. - # _on_download_completed acquires tasks_lock which is non-reentrant. - completed_tasks.append((batch_id, task_id)) + if live_info: + state = live_info.get('state', '') + # Trigger post-processing if download is completed successfully + # slskd uses compound states like 'Completed, Succeeded' - use substring matching + # Must exclude error states first (matching _build_batch_status_data's prioritized checking) + has_error = ('Errored' in state or 'Failed' in state or 'Rejected' in state or 'TimedOut' in state) + has_completion = ('Completed' in state or 'Succeeded' in state) + # Verify bytes actually transferred before trusting state string. + # slskd can report "Completed" before the full file is flushed to disk, + # or on connection drops that leave a partial file. + if has_completion and not has_error: + expected_size = live_info.get('size', 0) + transferred = live_info.get('bytesTransferred', 0) + if expected_size > 0 and transferred < expected_size: + if not task.get('_incomplete_warned'): + logger.debug("Monitor: %s state=%s but bytes incomplete (%s/%s) - waiting", task_id, state, transferred, expected_size) + task['_incomplete_warned'] = True + continue + if has_completion and not has_error and ( + task['status'] == 'downloading' or release_recoverable + ): + task.pop('_incomplete_warned', None) + # CRITICAL FIX: Transition to 'post_processing' HERE so downloads + # don't depend on browser polling to trigger post-processing. + # Previously, post-processing was only submitted by _build_batch_status_data + # (called from browser-polled endpoints), meaning closing the browser + # left tasks stuck in 'downloading' forever. + task['status'] = 'post_processing' + task['status_change_time'] = current_time + logger.info(f"Monitor detected completed download for {task_id} ({state}) - submitting post-processing") + # Collect for handling outside the lock to prevent deadlock. + # _on_download_completed acquires tasks_lock which is non-reentrant. + completed_tasks.append((batch_id, task_id)) # ---- All work below runs WITHOUT tasks_lock held ---- if globals().get('IS_SHUTTING_DOWN', False) or not self.monitoring: @@ -308,7 +343,7 @@ class WebUIDownloadMonitor: for download in all_downloads: key = _make_context_key(download.username, download.filename) # Convert DownloadStatus to transfer dict format for monitor compatibility - live_transfers[key] = { + transfer_row = { 'id': download.id, 'filename': download.filename, 'username': download.username, @@ -318,6 +353,10 @@ class WebUIDownloadMonitor: 'bytesTransferred': download.transferred, 'averageSpeed': download.speed, } + live_transfers[key] = transfer_row + id_key = _download_id_key(download.id) + if id_key: + live_transfers[id_key] = transfer_row except Exception as yt_error: logger.error(f"Monitor: Could not fetch streaming source downloads: {yt_error}") @@ -352,7 +391,7 @@ class WebUIDownloadMonitor: return False lookup_key = _make_context_key(task_username, task_filename) - live_info = live_transfers_lookup.get(lookup_key) + live_info = _lookup_live_info(task, live_transfers_lookup) if not live_info: # User-initiated manual pick — skip auto-retry. The status diff --git a/core/downloads/post_processing.py b/core/downloads/post_processing.py index 1b2d04d2..2c17e90f 100644 --- a/core/downloads/post_processing.py +++ b/core/downloads/post_processing.py @@ -19,9 +19,11 @@ from __future__ import annotations import logging import os +import shutil import time import traceback from dataclasses import dataclass +from difflib import SequenceMatcher from pathlib import Path from typing import Any, Callable, Optional @@ -34,7 +36,7 @@ from core.imports.context import ( get_import_original_search, normalize_import_context, ) -from core.imports.filename import extract_track_number_from_filename +from core.imports.filename import extract_track_number_from_filename, parse_filename_metadata from core.metadata import enrichment as metadata_enrichment from core.runtime_state import ( download_tasks, @@ -46,6 +48,85 @@ from core.runtime_state import ( logger = logging.getLogger(__name__) +_AUDIO_EXTENSIONS = { + '.flac', '.ape', '.wav', '.alac', '.dsf', '.dff', '.aiff', '.aif', + '.opus', '.ogg', '.m4a', '.aac', '.mp3', '.wma', +} + + +def _is_audio_file(path: str) -> bool: + return Path(str(path or '')).suffix.lower() in _AUDIO_EXTENSIONS + + +def _reject_non_audio_found_file(found_file: Optional[str], file_location: Optional[str]) -> tuple[Optional[str], Optional[str]]: + if found_file and not _is_audio_file(found_file): + logger.warning( + "[Post-Processing] Ignoring non-audio candidate found during file search: %s", + found_file, + ) + return None, None + return found_file, file_location + + +def _normalize_match_text(value: str) -> str: + return ''.join(ch.lower() for ch in str(value or '') if ch.isalnum()) + + +def _release_audio_match_score(path: str, expected_title: str, expected_artist: str) -> float: + parsed = parse_filename_metadata(path) + parsed_title = parsed.get('title') or Path(path).stem + parsed_artist = parsed.get('artist') or '' + expected_title_norm = _normalize_match_text(expected_title) + parsed_title_norm = _normalize_match_text(parsed_title) + if expected_title_norm and ( + expected_title_norm in parsed_title_norm or parsed_title_norm in expected_title_norm + ): + title_score = 1.0 + else: + title_score = SequenceMatcher( + None, + expected_title_norm, + parsed_title_norm, + ).ratio() + if expected_artist and parsed_artist: + artist_score = SequenceMatcher( + None, + _normalize_match_text(expected_artist), + _normalize_match_text(parsed_artist), + ).ratio() + return (title_score * 0.75) + (artist_score * 0.25) + return title_score + + +def _track_title_from_task(track_info: Any, context: Optional[dict]) -> str: + if isinstance(track_info, dict): + title = track_info.get('name') or track_info.get('title') + if title: + return str(title) + return get_import_clean_title(context or {}, default='') + + +def _copy_release_audio_to_transfer(source_path: str, transfer_dir: str) -> Optional[str]: + try: + src = Path(source_path) + if not src.exists() or not src.is_file(): + return None + dest_dir = Path(transfer_dir) + dest_dir.mkdir(parents=True, exist_ok=True) + dest = dest_dir / src.name + if dest.exists(): + stem, suffix = src.stem, src.suffix + counter = 1 + while dest.exists(): + dest = dest_dir / f"{stem}_release_{counter}{suffix}" + counter += 1 + shutil.copy2(src, dest) + return str(dest) + except Exception as exc: + logger.warning("[Post-Processing] Could not copy release audio to transfer: %s", exc) + return None + + @dataclass class PostProcessDeps: """Bundle of dependencies the post-processing worker needs. @@ -190,14 +271,71 @@ def run_post_processing_worker(task_id: str, batch_id: str, deps: PostProcessDep # CRITICAL FIX: For YouTube downloads, the filename in task is 'id||title' (metadata), # but the actual file on disk is 'Title.mp3'. We must ask the client for the real path. + # Torrent/usenet also use opaque "url||display" filenames. Their completed + # job may contain a full release, so choose the best matching audio file + # and copy it to transfer before importing instead of moving the client's + # original completed download. if (task.get('username') == 'youtube' or '||' in str(task_filename)) and not found_file: - logger.info(f"[Post-Processing] Detected YouTube download task: {task_id}") + logger.info(f"[Post-Processing] Detected engine-backed download task: {task_id}") try: # Query the download orchestrator for the status which contains the real file path # CRITICAL FIX: Use the actual download_id designated by the client, not the internal task_id actual_download_id = task.get('download_id') or task_id status = deps.run_async(deps.download_orchestrator.get_download_status(actual_download_id)) - if status and status.file_path: + if status and task.get('username') in ('torrent', 'usenet'): + audio_files = list(getattr(status, 'audio_files', None) or []) + if not audio_files and getattr(status, 'file_path', None): + audio_files = [status.file_path] + expected_title = _track_title_from_task(track_info, context) + expected_artist = '' + if isinstance(track_info, dict): + artists = track_info.get('artists', []) + if isinstance(artists, list) and artists: + first_artist = artists[0] + expected_artist = first_artist.get('name', '') if isinstance(first_artist, dict) else str(first_artist) + elif isinstance(artists, str): + expected_artist = artists + if not expected_artist and context: + artist_ctx = get_import_context_artist(context) + expected_artist = artist_ctx.get('name', '') if isinstance(artist_ctx, dict) else '' + + scored_files = [ + (_release_audio_match_score(path, expected_title, expected_artist), path) + for path in audio_files + if _is_audio_file(path) + ] + scored_files.sort(reverse=True) + if scored_files: + best_score, best_path = scored_files[0] + logger.info( + "[Post-Processing] Best %s release file for '%s': %s (score %.2f)", + task.get('username'), expected_title, best_path, best_score, + ) + if best_score >= 0.80: + copied_path = _copy_release_audio_to_transfer(best_path, transfer_dir) + if copied_path: + found_file = copied_path + file_location = 'download' + logger.info( + "[Post-Processing] Copied matched %s release file to transfer: %s", + task.get('username'), copied_path, + ) + else: + logger.warning( + "[Post-Processing] No %s release file met match threshold for '%s' (best %.2f)", + task.get('username'), expected_title, best_score, + ) + if not found_file: + with tasks_lock: + if task_id in download_tasks: + download_tasks[task_id]['status'] = 'failed' + download_tasks[task_id]['error_message'] = ( + f"No matching audio file found in {task.get('username')} release" + ) + deps.on_download_completed(batch_id, task_id, False) + return + + if status and status.file_path and not found_file and task.get('username') not in ('torrent', 'usenet'): real_path = status.file_path if os.path.exists(real_path): # Determine if it's in download or transfer directory @@ -227,11 +365,12 @@ def run_post_processing_worker(task_id: str, batch_id: str, deps: PostProcessDep logger.info(f"[Post-Processing] Resolved actual YouTube filename: {found_file} (Location: {file_location})") else: - logger.warning(f"[Post-Processing] YouTube status reported path but file missing: {real_path}") + logger.warning(f"[Post-Processing] Engine status reported path but file missing: {real_path}") else: - logger.warning(f"[Post-Processing] YouTube status returned no file_path for task {task_id}") + if not found_file: + logger.warning(f"[Post-Processing] Engine status returned no file_path for task {task_id}") except Exception as e: - logger.error(f"[Post-Processing] Failed to retrieve YouTube task status: {e}") + logger.error(f"[Post-Processing] Failed to retrieve engine task status: {e}") _file_search_max_retries = 5 for retry_count in range(_file_search_max_retries): @@ -257,6 +396,7 @@ def run_post_processing_worker(task_id: str, batch_id: str, deps: PostProcessDep # Strategy 1: Try with original filename in both downloads and transfer logger.info("[Post-Processing] Strategy 1: Searching with original filename...") found_file, file_location = deps.find_completed_file(download_dir, task_filename, transfer_dir) + found_file, file_location = _reject_non_audio_found_file(found_file, file_location) if found_file: logger.info(f"[Post-Processing] Strategy 1 SUCCESS: Found file with original filename in {file_location}: {found_file}") @@ -269,7 +409,11 @@ def run_post_processing_worker(task_id: str, batch_id: str, deps: PostProcessDep found_result = deps.find_completed_file(transfer_dir, expected_final_filename) if found_result and found_result[0]: found_file, file_location = found_result[0], 'transfer' - logger.info(f"[Post-Processing] Strategy 2 SUCCESS: Found file with expected final filename: {found_file}") + found_file, file_location = _reject_non_audio_found_file(found_file, file_location) + if found_file: + logger.info(f"[Post-Processing] Strategy 2 SUCCESS: Found file with expected final filename: {found_file}") + else: + logger.error("[Post-Processing] Strategy 2 FAILED: Expected final filename resolved to a non-audio file") else: logger.error("[Post-Processing] Strategy 2 FAILED: Expected final filename not found in transfer folder") elif not expected_final_filename: diff --git a/core/downloads/staging.py b/core/downloads/staging.py index ecd38e40..f7ca8906 100644 --- a/core/downloads/staging.py +++ b/core/downloads/staging.py @@ -34,9 +34,12 @@ from __future__ import annotations import logging import os +import re from dataclasses import dataclass from typing import Any, Callable +from core.imports.filename import extract_track_number_from_filename + # `shutil` and `SequenceMatcher` are imported inline inside try_staging_match() # to keep the lift byte-identical with the original web_server.py function body. @@ -50,6 +53,56 @@ from core.runtime_state import ( logger = logging.getLogger(__name__) +def _coerce_positive_int(value: Any, default: int = 0) -> int: + try: + coerced = int(str(value).split('/')[0]) + except (TypeError, ValueError): + return default + return coerced if coerced > 0 else default + + +def _staging_title_variants(title: Any, normalize: Callable[[str], str]) -> list[str]: + """Return conservative title variants for release-file matching. + + Torrent / usenet release files often encode featured artists in the + filename/title while streaming metadata keeps them in the artist credit. + Strip only feature/bonus noise here; keep version words like remix, + extended, live, acoustic, etc. so distinct recordings do not collapse. + """ + raw = str(title or '').strip() + if not raw: + return [] + + compacted_separators = re.sub(r'[_]+', ' ', raw) + compacted_separators = re.sub(r'\s+', ' ', compacted_separators).strip() + + without_feat = re.sub( + r'\s*[\(\[]\s*(?:feat\.?|ft\.?|featuring)\s+[^)\]]*[\)\]]', + '', + compacted_separators, + flags=re.IGNORECASE, + ) + without_feat = re.sub( + r'\s+(?:feat\.?|ft\.?|featuring)\s+.*$', + '', + without_feat, + flags=re.IGNORECASE, + ) + without_bonus = re.sub( + r'\s*[\(\[]\s*bonus\s+track\s*[\)\]]', + '', + without_feat, + flags=re.IGNORECASE, + ) + + variants: list[str] = [] + for candidate in (raw, compacted_separators, without_feat, without_bonus): + normalized = normalize(candidate) + if normalized and normalized not in variants: + variants.append(normalized) + return variants + + @dataclass class StagingDeps: """Bundle of cross-cutting deps the staging-match helper needs.""" @@ -86,19 +139,24 @@ def try_staging_match(task_id, batch_id, track, deps: StagingDeps): normalize = deps.matching_engine.normalize_string norm_title = normalize(track_title) norm_artist = normalize(track_artist) + title_variants = _staging_title_variants(track_title, normalize) or [norm_title] best_match = None best_score = 0.0 for sf in staging_files: - sf_norm_title = normalize(sf['title']) + sf_title_variants = _staging_title_variants(sf['title'], normalize) sf_norm_artist = normalize(sf['artist']) - if not sf_norm_title: + if not sf_title_variants: continue # Title similarity (primary) - title_sim = SequenceMatcher(None, norm_title, sf_norm_title).ratio() + title_sim = max( + SequenceMatcher(None, expected, candidate).ratio() + for expected in title_variants + for candidate in sf_title_variants + ) if title_sim < 0.80: continue @@ -190,6 +248,8 @@ def try_staging_match(task_id, batch_id, track, deps: StagingDeps): track_info = download_tasks.get(task_id, {}).get('track_info', {}) if not isinstance(track_info, dict): track_info = {} + else: + track_info = dict(track_info) # Build spotify_artist / spotify_album context so post-processing can apply # the path template. Without these, _post_process_matched_download returns @@ -258,20 +318,37 @@ def try_staging_match(task_id, batch_id, track, deps: StagingDeps): ) has_clean_data = bool(track_title and track_artist and track_album_name) - track_number = ( - track_info.get('track_number', 0) or - getattr(track, 'track_number', 0) or 0 - ) - disc_number = ( - track_info.get('disc_number', 1) or - getattr(track, 'disc_number', 1) or 1 + file_track_number = ( + _coerce_positive_int(best_match.get('track_number'), 0) or + extract_track_number_from_filename(best_match.get('full_path', '')) ) + file_disc_number = _coerce_positive_int(best_match.get('disc_number'), 1) + if _private_album_bundle_staging: + track_number = file_track_number + disc_number = file_disc_number + else: + track_number = ( + _coerce_positive_int(track_info.get('track_number'), 0) or + _coerce_positive_int(track_info.get('trackNumber'), 0) or + _coerce_positive_int(getattr(track, 'track_number', 0), 0) or + file_track_number + ) + disc_number = ( + _coerce_positive_int(track_info.get('disc_number'), 0) or + _coerce_positive_int(track_info.get('discNumber'), 0) or + _coerce_positive_int(getattr(track, 'disc_number', 0), 0) or + file_disc_number + ) + track_info['track_number'] = track_number + track_info['disc_number'] = disc_number context = { 'track_info': track_info, 'spotify_artist': spotify_artist_ctx, 'spotify_album': spotify_album_ctx, 'original_search_result': { + 'username': _provenance_username, + 'filename': best_match.get('full_path', ''), 'title': track_title, 'artist': track_artist, 'spotify_clean_title': track_title, diff --git a/core/downloads/status.py b/core/downloads/status.py index 6c9fac4f..6e11a763 100644 --- a/core/downloads/status.py +++ b/core/downloads/status.py @@ -88,7 +88,9 @@ class StatusDeps: # slskd's live_transfers path and must NOT hit the engine fallback. _STREAMING_SOURCE_NAMES = frozenset(( 'youtube', 'tidal', 'qobuz', 'hifi', 'deezer_dl', 'lidarr', 'soundcloud', 'amazon', + 'torrent', 'usenet', )) +_RELEASE_SOURCE_NAMES = frozenset(('torrent', 'usenet')) # Keep these in sync with the engine plugins' state strings. _ENGINE_FAILURE_STATES = ('Errored', 'Failed', 'Rejected', 'TimedOut', 'Aborted') @@ -136,21 +138,12 @@ def _apply_engine_state_fallback( Mutates ``task`` in place (status / error_message) the same way the Soulseek branch does, so the next status poll sees the new state. - Submits post-processing on terminal success and fires - ``on_download_completed`` on terminal failure to free the worker - slot. + Submits post-processing on terminal success. Manual-pick failures + are completed here; automatic failures keep the existing retry path + in charge. """ if deps.download_orchestrator is None or deps.run_async is None: return - if task.get('status') in ('completed', 'failed', 'cancelled', 'not_found', 'post_processing'): - return - # Scope this fallback to user-initiated manual picks. Auto attempts - # already flow through the live_transfers_lookup IF branch (the engine - # pre-populates non-Soulseek records via get_all_downloads), and on - # failure the monitor's existing retry path picks the next candidate. - # Marking auto attempts failed here would short-circuit that fallback. - if not task.get('_user_manual_pick'): - return download_id = task.get('download_id') if not download_id: return @@ -158,6 +151,17 @@ def _apply_engine_state_fallback( username = task.get('username') or ti.get('username') if username not in _STREAMING_SOURCE_NAMES: return + manual_pick = bool(task.get('_user_manual_pick')) + if not manual_pick and username not in _RELEASE_SOURCE_NAMES: + return + release_source = username in _RELEASE_SOURCE_NAMES + terminal_status = task.get('status') in ('completed', 'failed', 'cancelled', 'not_found', 'post_processing') + if terminal_status and not ( + release_source + and task.get('status') in ('failed', 'not_found') + and download_id + ): + return try: record = deps.run_async( @@ -189,6 +193,14 @@ def _apply_engine_state_fallback( return if any(s in state_str for s in _ENGINE_FAILURE_STATES): + if not manual_pick: + task_status['status'] = task.get('status', 'downloading') + task_status['progress'] = _engine_progress_pct(record) + logger.info( + "[Engine Fallback] Task %s engine reports '%s' for auto %s attempt; leaving retry handling to monitor", + task_id, state_str, username, + ) + return if task['status'] != 'failed': task['status'] = 'failed' err = getattr(record, 'error_message', None) or getattr(record, 'error', None) or '' @@ -321,6 +333,17 @@ def build_batch_status_data(batch_id: str, batch: dict, live_transfers_lookup: d _ti = task.get('track_info') if isinstance(task.get('track_info'), dict) else {} task_filename = task.get('filename') or _ti.get('filename') task_username = task.get('username') or _ti.get('username') + if ( + task_username in _RELEASE_SOURCE_NAMES + and task['status'] not in ['completed', 'cancelled', 'post_processing'] + and task.get('download_id') + ): + _apply_engine_state_fallback( + task_id, task, task_status, batch_id, deps, + ) + batch_tasks.append(task_status) + continue + if task_filename and task_username: lookup_key = deps.make_context_key(task_username, task_filename) diff --git a/core/downloads/task_worker.py b/core/downloads/task_worker.py index 8fc42608..67b2a3fe 100644 --- a/core/downloads/task_worker.py +++ b/core/downloads/task_worker.py @@ -25,12 +25,41 @@ import traceback from dataclasses import dataclass from typing import Any, Callable, Optional -from core.runtime_state import download_tasks, tasks_lock +from core.runtime_state import download_batches, download_tasks, tasks_lock from core.spotify_client import Track as SpotifyTrack logger = logging.getLogger(__name__) +def _private_album_bundle_staging_miss_reason(batch_id: Optional[str], deps: Any) -> Optional[str]: + """Return a user-facing miss reason when per-track search should stop. + + Torrent / usenet album batches first download one private staged release, + then each track claims the matching staged file. If that claim fails after + the release is already staged, falling through to the normal per-track + search only retries release-level sources N times and can keep re-adding + the same torrent. Treat the staged release as authoritative for this pass. + """ + if not batch_id: + return None + + batch = download_batches.get(batch_id) + if not isinstance(batch, dict): + return None + + source = (batch.get('album_bundle_source') or '').lower() + mode = (getattr(deps.download_orchestrator, 'mode', '') or '').lower() + if ( + batch.get('album_bundle_private_staging') + and batch.get('album_bundle_state') == 'staged' + and source in ('torrent', 'usenet') + and mode == source + ): + return f'Track was not found in the staged {source} album release' + + return None + + @dataclass class TaskWorkerDeps: """Bundle of cross-cutting deps the per-task download worker needs.""" @@ -128,6 +157,21 @@ def download_track_worker(task_id: str, batch_id: Optional[str], deps: TaskWorke # === STAGING CHECK: Check staging folder for existing file before searching === if deps.try_staging_match(task_id, batch_id, track): return + staging_miss_reason = _private_album_bundle_staging_miss_reason(batch_id, deps) + if staging_miss_reason: + logger.warning( + "[Modal Worker] %s for '%s'; skipping redundant per-track %s search", + staging_miss_reason, + track.name, + getattr(deps.download_orchestrator, 'mode', 'release-source'), + ) + with tasks_lock: + if task_id in download_tasks: + download_tasks[task_id]['status'] = 'not_found' + download_tasks[task_id]['error_message'] = staging_miss_reason + if batch_id: + deps.on_download_completed(batch_id, task_id, False) + return # Initialize task state tracking (like GUI's parallel_search_tracking) with tasks_lock: @@ -147,6 +191,27 @@ def download_track_worker(task_id: str, batch_id: Optional[str], deps: TaskWorke artist_name = track.artists[0] if track.artists else None track_name = track.name + release_queries = [] + try: + _download_mode = (getattr(deps.download_orchestrator, 'mode', '') or '').lower() + _track_album = (getattr(track, 'album', '') or '').strip() + _track_title = (getattr(track, 'name', '') or '').strip() + _track_artists = list(getattr(track, 'artists', []) or []) + _first_artist = _track_artists[0] if _track_artists else '' + _primary_artist = ( + (_first_artist.get('name', '') if isinstance(_first_artist, dict) else str(_first_artist)) + or '' + ).strip() + if ( + _download_mode in ('torrent', 'usenet') + and _primary_artist + and _track_album + and _track_album.lower() not in ('unknown album', _track_title.lower()) + ): + release_queries.append(f"{_primary_artist} {_track_album}".strip()) + except Exception as _release_query_exc: + logger.debug("[Modal Worker] release query hint failed: %s", _release_query_exc) + # Start with matching engine queries search_queries = deps.matching_engine.generate_download_queries(track) @@ -175,8 +240,14 @@ def download_track_worker(task_id: str, batch_id: Optional[str], deps: TaskWorke if cleaned_name and cleaned_name.lower() != track_name.lower(): legacy_queries.append(cleaned_name.strip()) - # Combine enhanced queries with legacy fallbacks - all_queries = search_queries + legacy_queries + # Combine enhanced queries with legacy fallbacks. + # + # Torrent / usenet can use full album releases as a fallback for + # single-track requests, but trying the album release first makes + # playlist batches download whole albums before checking whether a + # track-shaped release exists. Keep release queries last so singles + # stay light when the indexer has a direct result. + all_queries = search_queries + legacy_queries + release_queries # Remove duplicates while preserving order unique_queries = [] diff --git a/core/downloads/validation.py b/core/downloads/validation.py index 714ffb6c..339d0e8e 100644 --- a/core/downloads/validation.py +++ b/core/downloads/validation.py @@ -25,6 +25,20 @@ def init(matching_engine_obj, download_orchestrator_obj): download_orchestrator = download_orchestrator_obj +def _torrent_usenet_artist_is_fallback(result): + """True when a release result has no parsed artist, only indexer filler.""" + if getattr(result, 'username', None) not in ('torrent', 'usenet'): + return False + artist = (getattr(result, 'artist', None) or '').strip() + if not artist: + return True + metadata = getattr(result, '_source_metadata', None) or {} + indexer = str(metadata.get('indexer') or '').strip() + if artist.lower() in ('torrent', 'usenet'): + return True + return bool(indexer and artist.lower() == indexer.lower()) + + def filter_soundcloud_previews(results, expected_track): """Drop SoundCloud preview snippets so they never reach the cache, the modal, or the auto-download attempt. @@ -136,13 +150,18 @@ def get_valid_candidates(results, spotify_track, query): ) continue - # Score using matching engine's generic scorer (same weights as Soulseek) + # Score using matching engine's generic scorer (same weights as Soulseek). + # Torrent/usenet release projections sometimes only have the indexer name + # in the artist field when a title did not parse as "Artist - Release". + # Treat that as unknown artist, not as a real mismatch. + has_only_fallback_artist = _torrent_usenet_artist_is_fallback(r) + candidate_artists = [] if has_only_fallback_artist else ([r.artist] if r.artist else []) confidence, match_type = matching_engine.score_track_match( source_title=expected_title, source_artists=expected_artists, source_duration_ms=expected_duration, candidate_title=r.title or '', - candidate_artists=[r.artist] if r.artist else [], + candidate_artists=candidate_artists, candidate_duration_ms=r.duration or 0, ) @@ -166,13 +185,14 @@ def get_valid_candidates(results, spotify_track, query): # wanted track's ALBUM name and taking the max gives album- # level releases a fair shot. The Auto-Import sweep then picks # the right file out of the downloaded album folder. - if r.username in ('torrent', 'usenet') and spotify_track and spotify_track.album: + expected_album = getattr(spotify_track, 'album', None) if spotify_track else None + if r.username in ('torrent', 'usenet') and expected_album: album_conf, _ = matching_engine.score_track_match( - source_title=spotify_track.album, + source_title=expected_album, source_artists=expected_artists, source_duration_ms=0, # albums don't have one duration candidate_title=r.title or '', - candidate_artists=[r.artist] if r.artist else [], + candidate_artists=candidate_artists, candidate_duration_ms=0, ) if album_conf > confidence: @@ -199,11 +219,10 @@ def get_valid_candidates(results, spotify_track, query): # Artist gate — streaming APIs (Tidal/Qobuz/HiFi/Deezer) have reliable metadata, # so "My Will" by "B. Starr" should never match expected "B小町". - # Skip for YouTube (video-title parsing is unreliable) and torrent/usenet - # (album-level releases legitimately don't expose per-track artist — - # the projection layer fills artist with the indexer name as a fallback, - # which would otherwise fail the gate against every Spotify artist). - if r.username not in ('youtube', 'torrent', 'usenet'): + # YouTube stays excluded because video-title parsing is unreliable. + # Torrent/usenet must also pass this gate so title-only matches + # from the wrong artist do not get downloaded. + if r.username != 'youtube' and not has_only_fallback_artist: from difflib import SequenceMatcher import re as _re _cand_artist_raw = r.artist or '' @@ -235,6 +254,16 @@ def get_valid_candidates(results, spotify_track, query): # so falling to SequenceMatcher means the strings are genuinely # different. 0.5 gives a safer buffer without blocking real # matches that would have scored above 0.85 anyway. + if r.username in ('torrent', 'usenet') and _best_artist < 0.5: + logger.info( + "[%s] Rejecting candidate due to artist mismatch: " + "expected=%s candidate=%r title=%r", + source_label, + list(expected_artists), + _cand_artist_raw, + r.title or '', + ) + continue if _best_artist < 0.5 and confidence < 0.85: continue diff --git a/core/imports/side_effects.py b/core/imports/side_effects.py index d1580564..6513aabf 100644 --- a/core/imports/side_effects.py +++ b/core/imports/side_effects.py @@ -198,6 +198,10 @@ def record_library_history_download(context: Dict[str, Any]) -> None: "deezer_dl": "Deezer", "lidarr": "Lidarr", "soundcloud": "SoundCloud", + "amazon": "Amazon", + "staging": "Staging", + "torrent": "Torrent", + "usenet": "Usenet", # Auto-import isn't a download source, but flows through the # same post-process pipeline (file lands → record provenance # + history → write to library DB). Tagging it as "Auto-Import" diff --git a/tests/downloads/test_downloads_lifecycle.py b/tests/downloads/test_downloads_lifecycle.py index fb67d157..8be74876 100644 --- a/tests/downloads/test_downloads_lifecycle.py +++ b/tests/downloads/test_downloads_lifecycle.py @@ -329,6 +329,48 @@ def test_batch_completion_emits_batch_complete_when_all_done(): assert 'b1' in monitor.stopped +def test_batch_completion_cleans_private_album_bundle_staging(tmp_path): + staging_dir = tmp_path / 'b1' + staging_dir.mkdir() + (staging_dir / 'leftover.flac').write_bytes(b'audio') + + download_tasks['t1'] = {'status': 'completed', 'track_info': {'name': 'X'}} + download_batches['b1'] = { + 'queue': ['t1'], 'queue_index': 1, 'active_count': 1, + 'max_concurrent': 1, 'permanently_failed_tracks': [], + 'cancelled_tracks': set(), + 'album_bundle_private_staging': True, + 'album_bundle_source': 'torrent', + 'album_bundle_staging_path': str(staging_dir), + } + deps, _ = _build_deps() + + lc.on_download_completed('b1', 't1', True, deps) + + assert not staging_dir.exists() + + +def test_batch_completion_keeps_unexpected_staging_path(tmp_path): + staging_dir = tmp_path / 'shared-staging' + staging_dir.mkdir() + (staging_dir / 'leftover.flac').write_bytes(b'audio') + + download_tasks['t1'] = {'status': 'completed', 'track_info': {'name': 'X'}} + download_batches['b1'] = { + 'queue': ['t1'], 'queue_index': 1, 'active_count': 1, + 'max_concurrent': 1, 'permanently_failed_tracks': [], + 'cancelled_tracks': set(), + 'album_bundle_private_staging': True, + 'album_bundle_source': 'torrent', + 'album_bundle_staging_path': str(staging_dir), + } + deps, _ = _build_deps() + + lc.on_download_completed('b1', 't1', True, deps) + + assert staging_dir.exists() + + def test_batch_completion_skips_emit_when_zero_successful(): """Don't emit batch_complete if nothing actually downloaded.""" download_tasks['t1'] = {'status': 'failed', 'track_info': {'name': 'X'}, 'track_index': 0} diff --git a/tests/downloads/test_downloads_post_processing.py b/tests/downloads/test_downloads_post_processing.py index a5f02d8c..455b2237 100644 --- a/tests/downloads/test_downloads_post_processing.py +++ b/tests/downloads/test_downloads_post_processing.py @@ -232,6 +232,28 @@ def test_file_found_in_downloads_with_context_runs_post_process_with_verificatio assert any(c[0] == 'post_process' for c in rec.calls) +def test_file_search_ignores_non_audio_candidates(monkeypatch): + download_tasks['t1'] = { + 'status': 'post_processing', + 'filename': 'Artist - Album.cue', + 'username': 'torrent', + 'track_info': {'name': 'Money'}, + } + matched_downloads_context['torrent::Artist - Album.cue'] = { + 'original_search_result': {'title': 'Money', 'track_number': 1}, + } + monkeypatch.setattr(pp.time, 'sleep', lambda s: None) + deps, rec = _build_deps( + find_completed_file=lambda *a, **kw: ('/downloads/Artist - Album.cue', 'download'), + ) + + pp.run_post_processing_worker('t1', 'b1', deps) + + assert download_tasks['t1']['status'] == 'failed' + assert not any(c[0] == 'post_process' for c in rec.calls) + assert ('on_complete', ('b1', 't1', False), {}) in rec.calls + + def test_file_found_in_downloads_no_context_marks_completed_directly(): """No matched context for the file → just mark completed since file exists.""" download_tasks['t1'] = { @@ -323,6 +345,139 @@ def test_youtube_task_uses_get_download_status_to_resolve_path(monkeypatch): assert any(c[0] == 'mark_completed' for c in rec.calls) +def test_torrent_release_copies_best_matching_audio_to_transfer(tmp_path): + release_dir = tmp_path / 'release' + release_dir.mkdir() + wrong = release_dir / '01 - Intro.flac' + right = release_dir / '02 - Money.flac' + wrong.write_bytes(b'wrong') + right.write_bytes(b'right') + transfer_dir = tmp_path / 'transfer' + + filename = 'magnet:?xt=abc||Artist - Album' + download_tasks['t1'] = { + 'status': 'post_processing', + 'filename': filename, + 'username': 'torrent', + 'download_id': 'dl-torrent-1', + 'track_info': {'name': 'Money', 'artists': [{'name': 'Artist'}]}, + } + matched_downloads_context[f'torrent::{filename}'] = { + 'original_search_result': {'title': 'Money', 'track_number': 2}, + } + + class _FakeStatus: + file_path = str(wrong) + audio_files = [str(wrong), str(right)] + + class _FakeTorrentClient: + def get_download_status(self, dl_id): + assert dl_id == 'dl-torrent-1' + return _FakeStatus() + + deps, rec = _build_deps( + config=_FakeConfig({'soulseek.transfer_path': str(transfer_dir)}), + download_orchestrator=_FakeTorrentClient(), + run_async=lambda coro: coro, + ) + + pp.run_post_processing_worker('t1', 'b1', deps) + + copied = transfer_dir / '02 - Money.flac' + assert copied.exists() + assert right.exists() + assert any(c[0] == 'post_process' and c[1][2] == str(copied) for c in rec.calls) + + +def test_torrent_release_prefers_task_title_over_release_context(tmp_path): + release_dir = tmp_path / 'release' + release_dir.mkdir() + wrong = release_dir / '09.Harry Styles - Pop.flac' + right = release_dir / '10.Harry Styles - American Girls.flac' + wrong.write_bytes(b'wrong') + right.write_bytes(b'right') + transfer_dir = tmp_path / 'transfer' + + filename = 'http://prowlarr/download?id=1||Harry Styles - Kiss All The Time' + download_tasks['t1'] = { + 'status': 'post_processing', + 'filename': filename, + 'username': 'torrent', + 'download_id': 'dl-torrent-1', + 'track_info': {'name': 'American Girls', 'artists': [{'name': 'Harry Styles'}]}, + } + matched_downloads_context[f'torrent::{filename}'] = { + 'original_search_result': {'title': 'Pop', 'clean_title': 'Pop', 'track_number': 9}, + } + + class _FakeStatus: + file_path = str(wrong) + audio_files = [str(wrong), str(right)] + + class _FakeTorrentClient: + def get_download_status(self, dl_id): + assert dl_id == 'dl-torrent-1' + return _FakeStatus() + + deps, rec = _build_deps( + config=_FakeConfig({'soulseek.transfer_path': str(transfer_dir)}), + download_orchestrator=_FakeTorrentClient(), + run_async=lambda coro: coro, + ) + + pp.run_post_processing_worker('t1', 'b1', deps) + + copied = transfer_dir / '10.Harry Styles - American Girls.flac' + assert copied.exists() + assert any(c[0] == 'post_process' and c[1][2] == str(copied) for c in rec.calls) + + +def test_torrent_release_without_matching_file_does_not_fallback_to_generic_search(tmp_path): + release_dir = tmp_path / 'release' + release_dir.mkdir() + wrong = release_dir / '09.Harry Styles - Pop.flac' + wrong.write_bytes(b'wrong') + transfer_dir = tmp_path / 'transfer' + + filename = 'http://prowlarr/download?id=1||Harry Styles - Kiss All The Time' + download_tasks['t1'] = { + 'status': 'post_processing', + 'filename': filename, + 'username': 'torrent', + 'download_id': 'dl-torrent-1', + 'track_info': {'name': 'American Girls', 'artists': [{'name': 'Harry Styles'}]}, + } + matched_downloads_context[f'torrent::{filename}'] = { + 'original_search_result': {'title': 'Pop', 'clean_title': 'Pop', 'track_number': 9}, + } + + class _FakeStatus: + file_path = str(wrong) + audio_files = [str(wrong)] + + class _FakeTorrentClient: + def get_download_status(self, dl_id): + assert dl_id == 'dl-torrent-1' + return _FakeStatus() + + def _unexpected_search(*args, **kwargs): + raise AssertionError("torrent releases should not fall back to generic file search") + + deps, rec = _build_deps( + config=_FakeConfig({'soulseek.transfer_path': str(transfer_dir)}), + download_orchestrator=_FakeTorrentClient(), + run_async=lambda coro: coro, + find_completed_file=_unexpected_search, + ) + + pp.run_post_processing_worker('t1', 'b1', deps) + + assert download_tasks['t1']['status'] == 'failed' + assert 'No matching audio file' in download_tasks['t1']['error_message'] + assert any(c[0] == 'on_complete' and c[1] == ('b1', 't1', False) for c in rec.calls) + assert not list(transfer_dir.glob('*')) + + def test_fuzzy_context_matching_when_exact_key_missing(monkeypatch): """When exact key isn't in matched_downloads_context, worker tries fuzzy match constrained to same Soulseek username.""" diff --git a/tests/downloads/test_downloads_staging.py b/tests/downloads/test_downloads_staging.py index e75e633c..70e86b6b 100644 --- a/tests/downloads/test_downloads_staging.py +++ b/tests/downloads/test_downloads_staging.py @@ -278,6 +278,180 @@ def test_explicit_album_context_uses_real_data(tmp_path): assert ctx['staging_source'] is True +def test_staging_context_falls_back_to_matched_file_track_number(tmp_path): + """Album-bundle staging can recover numbering from the selected audio file.""" + src_file = tmp_path / 'staging' / '03 - Backseat Freestyle.flac' + src_file.parent.mkdir() + src_file.touch() + + deps = _build_deps( + transfer_path=str(tmp_path / 'transfer'), + staging_files=[ + { + 'full_path': str(src_file), + 'title': 'Backseat Freestyle', + 'artist': 'Kendrick Lamar', + 'track_number': 3, + 'disc_number': 1, + }, + ], + ) + _seed_task('t6b', track_info={ + '_is_explicit_album_download': True, + '_explicit_album_context': {'id': 'alb', 'name': 'good kid, m.A.A.d city (Deluxe)'}, + '_explicit_artist_context': {'id': 'art', 'name': 'Kendrick Lamar'}, + }) + + ds.try_staging_match( + 't6b', 'b1', + _Track(name='Backseat Freestyle', artists=['Kendrick Lamar']), + deps, + ) + + ctx = matched_downloads_context['staging_t6b'] + assert ctx['original_search_result']['track_number'] == 3 + assert ctx['original_search_result']['disc_number'] == 1 + + +def test_private_album_bundle_staging_overrides_default_track_info_number(tmp_path): + """Private release staging trusts the selected file number over weak task defaults.""" + src_file = tmp_path / 'staging' / '04-kendrick_lamar-the_art_of_peer_pressure.flac' + src_file.parent.mkdir() + src_file.touch() + + def get_batch_field(_batch_id, field): + if field == 'album_bundle_source': + return 'torrent' + if field == 'album_bundle_private_staging': + return True + return None + + deps = _build_deps( + transfer_path=str(tmp_path / 'transfer'), + staging_files=[ + { + 'full_path': str(src_file), + 'title': 'The Art of Peer Pressure', + 'artist': 'Kendrick Lamar', + }, + ], + get_batch_field=get_batch_field, + ) + _seed_task('t6c', track_info={ + '_is_explicit_album_download': True, + '_explicit_album_context': {'id': 'alb', 'name': 'good kid, m.A.A.d city (Deluxe)'}, + '_explicit_artist_context': {'id': 'art', 'name': 'Kendrick Lamar'}, + 'track_number': 1, + }) + + ds.try_staging_match( + 't6c', 'b1', + _Track(name='The Art of Peer Pressure', artists=['Kendrick Lamar']), + deps, + ) + + ctx = matched_downloads_context['staging_t6c'] + assert ctx['track_info']['track_number'] == 4 + assert ctx['original_search_result']['track_number'] == 4 + assert ctx['original_search_result']['username'] == 'torrent' + assert ctx['original_search_result']['filename'] == str(src_file) + + +def test_staging_title_match_accepts_feature_suffix_from_release_file(tmp_path): + """Album releases can include featured artists in filenames.""" + src_file = tmp_path / 'staging' / '05-kendrick_lamar-money_trees_(feat._jay_rock).flac' + src_file.parent.mkdir() + src_file.touch() + + deps = _build_deps( + transfer_path=str(tmp_path / 'transfer'), + staging_files=[ + { + 'full_path': str(src_file), + 'title': 'money_trees_(feat._jay_rock)', + 'artist': 'Kendrick Lamar', + 'track_number': 5, + }, + ], + ) + _seed_task('t_feature', track_info={ + '_is_explicit_album_download': True, + '_explicit_album_context': {'id': 'alb', 'name': 'good kid, m.A.A.d city (Deluxe)'}, + '_explicit_artist_context': {'id': 'art', 'name': 'Kendrick Lamar'}, + }) + + result = ds.try_staging_match( + 't_feature', 'b1', + _Track(name='Money Trees', artists=['Kendrick Lamar']), + deps, + ) + + assert result is True + assert matched_downloads_context['staging_t_feature']['track_info']['track_number'] == 5 + + +def test_staging_title_match_accepts_bonus_track_against_release_file(tmp_path): + """Expected bonus labels should not block matching the actual release file.""" + src_file = tmp_path / 'staging' / '13-kendrick_lamar-the_recipe_(feat._dr._dre).flac' + src_file.parent.mkdir() + src_file.touch() + + deps = _build_deps( + transfer_path=str(tmp_path / 'transfer'), + staging_files=[ + { + 'full_path': str(src_file), + 'title': 'the_recipe_(feat._dr._dre)', + 'artist': 'Kendrick Lamar', + 'track_number': 13, + }, + ], + ) + _seed_task('t_bonus', track_info={ + '_is_explicit_album_download': True, + '_explicit_album_context': {'id': 'alb', 'name': 'good kid, m.A.A.d city (Deluxe)'}, + '_explicit_artist_context': {'id': 'art', 'name': 'Kendrick Lamar'}, + }) + + result = ds.try_staging_match( + 't_bonus', 'b1', + _Track(name='The Recipe (Bonus Track)', artists=['Kendrick Lamar']), + deps, + ) + + assert result is True + assert matched_downloads_context['staging_t_bonus']['track_info']['track_number'] == 13 + + +def test_staging_title_match_keeps_wrong_versions_separate(tmp_path): + """Do not strip remix/extended wording when matching staged release files.""" + src_file = tmp_path / 'staging' / '17-kendrick_lamar-swimming_pools_(drank)_(black_hippy_remix).flac' + src_file.parent.mkdir() + src_file.touch() + + deps = _build_deps( + transfer_path=str(tmp_path / 'transfer'), + staging_files=[ + { + 'full_path': str(src_file), + 'title': 'swimming_pools_(drank)_(black_hippy_remix)', + 'artist': 'Kendrick Lamar', + 'track_number': 17, + }, + ], + ) + _seed_task('t_wrong_version') + + result = ds.try_staging_match( + 't_wrong_version', 'b1', + _Track(name='Swimming Pools (Drank) (Extended Version)', artists=['Kendrick Lamar']), + deps, + ) + + assert result is False + assert 'staging_t_wrong_version' not in matched_downloads_context + + def test_fallback_context_synthesizes_from_track(tmp_path): """Without explicit context, synthesizes spotify_artist/album from the track.""" src_file = tmp_path / 'staging' / 'Hello.flac' diff --git a/tests/downloads/test_downloads_status.py b/tests/downloads/test_downloads_status.py index cc9a5d15..706f9131 100644 --- a/tests/downloads/test_downloads_status.py +++ b/tests/downloads/test_downloads_status.py @@ -40,6 +40,8 @@ def _build_deps( make_key=None, submit_pp=None, cached_transfers=None, + download_orchestrator=None, + run_async=None, ): submitted = [] @@ -53,6 +55,8 @@ def _build_deps( make_context_key=make_key or (lambda u, f: f"{u}::{f}"), submit_post_processing=submit_pp or _default_submit, get_cached_transfer_data=cached_transfers or (lambda: {}), + download_orchestrator=download_orchestrator, + run_async=run_async, ) return deps, submitted @@ -286,6 +290,128 @@ def test_post_processing_status_progress_is_95(): assert out['tasks'][0]['progress'] == 95 +def test_auto_torrent_without_live_entry_uses_engine_success_fallback(): + class _Record: + state = 'Completed, Succeeded' + progress = 100 + + class _Orchestrator: + def get_download_status(self, download_id): + assert download_id == 'dl1' + return _Record() + + deps, submitted = _build_deps( + download_orchestrator=_Orchestrator(), + run_async=lambda value: value, + ) + download_tasks['t1'] = { + 'track_index': 0, + 'status': 'downloading', + 'track_info': {}, + 'filename': 'song.flac', + 'username': 'torrent', + 'download_id': 'dl1', + } + batch = {'phase': 'downloading', 'queue': ['t1']} + out = st.build_batch_status_data('b1', batch, {}, deps) + assert out['tasks'][0]['status'] == 'post_processing' + assert download_tasks['t1']['status'] == 'post_processing' + assert submitted == [('t1', 'b1')] + + +def test_auto_torrent_prefers_engine_success_over_live_entry(): + class _Record: + state = 'Completed, Succeeded' + progress = 100 + + class _Orchestrator: + def get_download_status(self, download_id): + assert download_id == 'dl1' + return _Record() + + deps, submitted = _build_deps( + download_orchestrator=_Orchestrator(), + run_async=lambda value: value, + ) + download_tasks['t1'] = { + 'track_index': 0, + 'status': 'downloading', + 'track_info': {}, + 'filename': 'song.flac', + 'username': 'torrent', + 'download_id': 'dl1', + } + live = {'torrent::song.flac': { + 'state': 'InProgress, Downloading', + 'percentComplete': 100, + }} + batch = {'phase': 'downloading', 'queue': ['t1']} + out = st.build_batch_status_data('b1', batch, live, deps) + assert out['tasks'][0]['status'] == 'post_processing' + assert download_tasks['t1']['status'] == 'post_processing' + assert submitted == [('t1', 'b1')] + + +def test_auto_torrent_engine_failure_does_not_bypass_monitor_retry(): + class _Record: + state = 'Completed, Errored' + progress = 100 + error = 'client failed' + + class _Orchestrator: + def get_download_status(self, download_id): + assert download_id == 'dl1' + return _Record() + + deps, submitted = _build_deps( + download_orchestrator=_Orchestrator(), + run_async=lambda value: value, + ) + download_tasks['t1'] = { + 'track_index': 0, + 'status': 'downloading', + 'track_info': {}, + 'filename': 'song.flac', + 'username': 'torrent', + 'download_id': 'dl1', + } + batch = {'phase': 'downloading', 'queue': ['t1']} + out = st.build_batch_status_data('b1', batch, {}, deps) + assert out['tasks'][0]['status'] == 'downloading' + assert download_tasks['t1']['status'] == 'downloading' + assert submitted == [] + + +def test_auto_torrent_engine_success_recovers_premature_failed_task(): + class _Record: + state = 'Completed, Succeeded' + progress = 100 + + class _Orchestrator: + def get_download_status(self, download_id): + assert download_id == 'dl1' + return _Record() + + deps, submitted = _build_deps( + download_orchestrator=_Orchestrator(), + run_async=lambda value: value, + ) + download_tasks['t1'] = { + 'track_index': 0, + 'status': 'failed', + 'track_info': {}, + 'filename': 'release-url||Release', + 'username': 'torrent', + 'download_id': 'dl1', + 'error_message': 'premature failure', + } + batch = {'phase': 'downloading', 'queue': ['t1']} + out = st.build_batch_status_data('b1', batch, {}, deps) + assert out['tasks'][0]['status'] == 'post_processing' + assert download_tasks['t1']['status'] == 'post_processing' + assert submitted == [('t1', 'b1')] + + # --------------------------------------------------------------------------- # Safety valve (stuck task handling) # --------------------------------------------------------------------------- diff --git a/tests/downloads/test_downloads_task_worker.py b/tests/downloads/test_downloads_task_worker.py index 11ccdfca..ea605d6e 100644 --- a/tests/downloads/test_downloads_task_worker.py +++ b/tests/downloads/test_downloads_task_worker.py @@ -5,14 +5,16 @@ from __future__ import annotations import pytest from core.downloads import task_worker as tw -from core.runtime_state import download_tasks +from core.runtime_state import download_batches, download_tasks @pytest.fixture(autouse=True) def reset_state(): download_tasks.clear() + download_batches.clear() yield download_tasks.clear() + download_batches.clear() # --------------------------------------------------------------------------- @@ -37,7 +39,7 @@ class _FakeClient: orchestrator); plain attrs (hybrid_order, hybrid_primary, etc.) are set as attributes so getattr() lookups still resolve them.""" _CLIENT_NAMES = {'soulseek', 'youtube', 'tidal', 'qobuz', 'hifi', - 'deezer_dl', 'lidarr', 'soundcloud'} + 'deezer_dl', 'lidarr', 'soundcloud', 'torrent', 'usenet'} def __init__(self, results=None, mode='soulseek', subclients=None): self._results = results if results is not None else [] @@ -54,7 +56,7 @@ class _FakeClient: def client(self, name): return self._client_map.get(name) - async def search(self, query, timeout=30): + async def search(self, query, timeout=30, exclude_sources=None): self.search_calls.append((query, timeout)) return (self._results, None) @@ -194,6 +196,33 @@ def test_staging_match_hit_returns_immediately(): assert rec.calls == [] +def test_private_torrent_album_staging_miss_skips_per_track_search(): + _seed_task(track_info={ + 'id': 'sp-1', 'name': 'Money Trees', 'artists': ['Kendrick Lamar'], + 'album': 'good kid, m.A.A.d city (Deluxe)', 'duration_ms': 387000, + }) + download_batches['b1'] = { + 'album_bundle_private_staging': True, + 'album_bundle_state': 'staged', + 'album_bundle_source': 'torrent', + } + client = _FakeClient(results=['should-not-search'], mode='torrent') + rec = _Recorder() + deps, _ = _build_deps( + soulseek=client, + matching=_FakeMatchEngine(queries=['Kendrick Lamar Money Trees']), + try_staging_match=lambda *a, **kw: False, + on_download_completed=rec('done'), + ) + + tw.download_track_worker('t1', 'b1', deps) + + assert client.search_calls == [] + assert download_tasks['t1']['status'] == 'not_found' + assert 'staged torrent album release' in download_tasks['t1']['error_message'] + assert ('done', ('b1', 't1', False), {}) in rec.calls + + # --------------------------------------------------------------------------- # Search loop happy path # --------------------------------------------------------------------------- @@ -219,6 +248,25 @@ def test_first_query_success_returns_after_storing_source(): assert download_tasks['t1']['status'] == 'searching' +def test_torrent_mode_uses_album_release_after_track_queries(): + _seed_task(track_info={ + 'id': 'sp-1', 'name': 'Money', 'artists': ['Pink Floyd'], + 'album': 'The Dark Side of the Moon', 'duration_ms': 383000, + }) + client = _FakeClient(results=[], mode='torrent') + rec = _Recorder() + deps, _ = _build_deps( + soulseek=client, + matching=_FakeMatchEngine(queries=['Pink Floyd Money']), + on_download_completed=rec('done'), + ) + + tw.download_track_worker('t1', 'b1', deps) + + assert client.search_calls[0][0] == 'Pink Floyd Money' + assert client.search_calls[-1][0] == 'Pink Floyd The Dark Side of the Moon' + + def test_no_results_marks_not_found_and_calls_completion(): _seed_task() rec = _Recorder() diff --git a/tests/downloads/test_downloads_validation.py b/tests/downloads/test_downloads_validation.py index ddb5346a..1acdf91c 100644 --- a/tests/downloads/test_downloads_validation.py +++ b/tests/downloads/test_downloads_validation.py @@ -123,3 +123,47 @@ def test_keeps_tidal_candidate_inside_integrity_duration_tolerance(monkeypatch): result = get_valid_candidates([tidal], expected, 'Artist Song') assert result == [tidal] + + +def test_rejects_torrent_title_match_from_wrong_artist(monkeypatch): + monkeypatch.setattr(validation, 'matching_engine', _MatchingEngine()) + expected = _Track(duration_ms=180_000, name='The Man I Need', artists=('Olivia Dean',)) + wrong_artist = _Candidate( + username='torrent', + duration=None, + title='The Man I Need', + artist='Tinkabelle', + ) + + assert get_valid_candidates([wrong_artist], expected, 'Olivia Dean The Man I Need') == [] + + +def test_keeps_torrent_title_match_from_expected_artist(monkeypatch): + monkeypatch.setattr(validation, 'matching_engine', _MatchingEngine()) + expected = _Track(duration_ms=180_000, name='The Man I Need', artists=('Olivia Dean',)) + correct_artist = _Candidate( + username='torrent', + duration=None, + title='The Man I Need', + artist='Olivia Dean', + ) + + result = get_valid_candidates([correct_artist], expected, 'Olivia Dean The Man I Need') + + assert result == [correct_artist] + + +def test_keeps_torrent_title_match_when_artist_is_indexer_fallback(monkeypatch): + monkeypatch.setattr(validation, 'matching_engine', _MatchingEngine()) + expected = _Track(duration_ms=180_000, name='The Man I Need', artists=('Olivia Dean',)) + candidate = _Candidate( + username='torrent', + duration=None, + title='The Man I Need', + artist='Indexer', + ) + candidate._source_metadata = {'indexer': 'Indexer'} + + result = get_valid_candidates([candidate], expected, 'Olivia Dean The Man I Need') + + assert result == [candidate] diff --git a/tests/imports/test_import_side_effects.py b/tests/imports/test_import_side_effects.py index 455d2d4f..136ee765 100644 --- a/tests/imports/test_import_side_effects.py +++ b/tests/imports/test_import_side_effects.py @@ -2,6 +2,8 @@ import os import sqlite3 from types import SimpleNamespace +import pytest + from core.imports import side_effects @@ -364,6 +366,42 @@ def test_library_history_labels_auto_import(monkeypatch): assert captured["title"] == "Auto-Imported Track" +@pytest.mark.parametrize( + ("username", "expected"), + [ + ("torrent", "Torrent"), + ("usenet", "Usenet"), + ("staging", "Staging"), + ], +) +def test_library_history_labels_release_and_staging_sources(monkeypatch, username, expected): + """Release/staging imports should not fall through to the Soulseek label.""" + captured = {} + + class _DBStub: + def add_library_history_entry(self, **kwargs): + captured.update(kwargs) + + monkeypatch.setattr(side_effects, "get_database", lambda: _DBStub()) + + context = { + "track_info": { + "name": "Imported Track", + "artists": [{"name": "Some Artist"}], + "album": "Some Album", + }, + "original_search_result": { + "username": username, + "filename": "source-file.flac", + }, + "_final_processed_path": "/library/some-album/01.flac", + } + + side_effects.record_library_history_download(context) + + assert captured["download_source"] == expected + + # --------------------------------------------------------------------------- # Album duration parity — must equal sum of all track durations, not whatever # the first imported track happened to be. diff --git a/tests/test_manual_pick_no_auto_retry.py b/tests/test_manual_pick_no_auto_retry.py index efa88359..9c70f0f8 100644 --- a/tests/test_manual_pick_no_auto_retry.py +++ b/tests/test_manual_pick_no_auto_retry.py @@ -165,3 +165,132 @@ def test_monitor_waits_for_post_processing_before_batch_success(monkeypatch): dm.download_batches.update(previous_batches) +def test_monitor_matches_release_download_by_id_when_filename_changes(monkeypatch): + """Torrent/usenet rows can expose the completed audio filename, not + the original indexer URL/title stored on the task. The monitor must + still claim the completed release by stable download_id. + """ + monkeypatch.setattr(dm, '_make_context_key', lambda u, f: f"{u}::{f}") + monkeypatch.setattr(dm.WebUIDownloadMonitor, '_validate_worker_counts', lambda self: None) + + submitted = [] + + class FakeExecutor: + def submit(self, func, task_id, batch_id): + submitted.append((func, task_id, batch_id)) + + def fake_post_processing_worker(task_id, batch_id): + return None + + monkeypatch.setattr(dm, 'missing_download_executor', FakeExecutor()) + monkeypatch.setattr(dm, '_run_post_processing_worker', fake_post_processing_worker) + monkeypatch.setattr(dm, '_on_download_completed', lambda *args: None) + + with dm.tasks_lock: + previous_tasks = dict(dm.download_tasks) + previous_batches = dict(dm.download_batches) + dm.download_tasks.clear() + dm.download_batches.clear() + dm.download_tasks['task-1'] = { + 'track_info': {'name': 'Ran To Atlanta'}, + 'username': 'torrent', + 'filename': 'http://prowlarr/download?id=123||Drake - ICEMAN', + 'status': 'downloading', + 'download_id': 'torrent-1', + 'status_change_time': time.time(), + } + dm.download_batches['batch-1'] = {'queue': ['task-1']} + + try: + monitor = dm.WebUIDownloadMonitor() + monitor.monitoring = True + monitor.monitored_batches.add('batch-1') + monkeypatch.setattr( + monitor, + '_get_live_transfers', + lambda: { + 'download_id::torrent-1': { + 'id': 'torrent-1', + 'username': 'torrent', + 'filename': '01. Drake - Make Them Cry.flac', + 'state': 'Completed, Succeeded', + 'size': 100, + 'bytesTransferred': 100, + } + }, + ) + + monitor._check_all_downloads() + + assert submitted == [(fake_post_processing_worker, 'task-1', 'batch-1')] + assert dm.download_tasks['task-1']['status'] == 'post_processing' + finally: + with dm.tasks_lock: + dm.download_tasks.clear() + dm.download_tasks.update(previous_tasks) + dm.download_batches.clear() + dm.download_batches.update(previous_batches) + + +def test_monitor_recovers_premature_failed_release_download(monkeypatch): + monkeypatch.setattr(dm, '_make_context_key', lambda u, f: f"{u}::{f}") + monkeypatch.setattr(dm.WebUIDownloadMonitor, '_validate_worker_counts', lambda self: None) + + submitted = [] + + class FakeExecutor: + def submit(self, func, task_id, batch_id): + submitted.append((func, task_id, batch_id)) + + def fake_post_processing_worker(task_id, batch_id): + return None + + monkeypatch.setattr(dm, 'missing_download_executor', FakeExecutor()) + monkeypatch.setattr(dm, '_run_post_processing_worker', fake_post_processing_worker) + monkeypatch.setattr(dm, '_on_download_completed', lambda *args: None) + + with dm.tasks_lock: + previous_tasks = dict(dm.download_tasks) + previous_batches = dict(dm.download_batches) + dm.download_tasks.clear() + dm.download_batches.clear() + dm.download_tasks['task-1'] = { + 'track_info': {'name': 'DAISIES'}, + 'username': 'torrent', + 'filename': 'http://prowlarr/download?id=123||Justin Bieber - Swag', + 'status': 'failed', + 'download_id': 'torrent-1', + 'status_change_time': time.time(), + } + dm.download_batches['batch-1'] = {'queue': ['task-1']} + + try: + monitor = dm.WebUIDownloadMonitor() + monitor.monitoring = True + monitor.monitored_batches.add('batch-1') + monkeypatch.setattr( + monitor, + '_get_live_transfers', + lambda: { + 'download_id::torrent-1': { + 'id': 'torrent-1', + 'username': 'torrent', + 'filename': '02. Justin Bieber - DAISIES.flac', + 'state': 'Completed, Succeeded', + 'size': 100, + 'bytesTransferred': 100, + } + }, + ) + + monitor._check_all_downloads() + + assert submitted == [(fake_post_processing_worker, 'task-1', 'batch-1')] + assert dm.download_tasks['task-1']['status'] == 'post_processing' + finally: + with dm.tasks_lock: + dm.download_tasks.clear() + dm.download_tasks.update(previous_tasks) + dm.download_batches.clear() + dm.download_batches.update(previous_batches) + diff --git a/web_server.py b/web_server.py index 0e7e2d8e..d6269314 100644 --- a/web_server.py +++ b/web_server.py @@ -2115,10 +2115,15 @@ def get_status(): _status_cache_timestamps['media_server'] = current_time # else: use cached value - # Test Soulseek - only if it's the active source or in the hybrid order - if current_time - _status_cache_timestamps['soulseek'] > STATUS_CACHE_TTL: - download_mode = config_manager.get('download_source.mode', 'hybrid') - hybrid_order = config_manager.get('download_source.hybrid_order', ['hifi', 'youtube', 'soulseek']) + download_mode = config_manager.get('download_source.mode', 'hybrid') + hybrid_order = config_manager.get('download_source.hybrid_order', ['hifi', 'youtube', 'soulseek']) + if isinstance(hybrid_order, str): + hybrid_order = [hybrid_order] + source_cache_key = f"{download_mode}:{','.join(str(s) for s in (hybrid_order or []))}" + + # Test Soulseek/download source only when the cached source selection is still current. + if (current_time - _status_cache_timestamps['soulseek'] > STATUS_CACHE_TTL or + _status_cache['soulseek'].get('source_cache_key') != source_cache_key): soulseek_relevant = (download_mode == 'soulseek' or (download_mode == 'hybrid' and 'soulseek' in hybrid_order)) @@ -2130,11 +2135,24 @@ def get_status(): is_serverless = (download_mode in serverless_sources or (download_mode == 'hybrid' and hybrid_order and any(s in serverless_sources for s in hybrid_order))) + external_client_sources = ('torrent', 'usenet') + external_client_relevant = ( + download_mode in external_client_sources or + (download_mode == 'hybrid' and + hybrid_order and any(s in external_client_sources for s in hybrid_order)) + ) # Serverless check first — avoids slow slskd timeout when YouTube/HiFi are in hybrid order if is_serverless: soulseek_status = True soulseek_response_time = 0 + elif external_client_relevant and download_orchestrator: + soulseek_start = time.time() + try: + soulseek_status = run_async(download_orchestrator.check_connection()) + except Exception: + soulseek_status = False + soulseek_response_time = (time.time() - soulseek_start) * 1000 elif soulseek_relevant and download_orchestrator: soulseek_start = time.time() try: @@ -2148,13 +2166,17 @@ def get_status(): _status_cache['soulseek'] = { 'connected': soulseek_status, - 'response_time': round(soulseek_response_time, 1) + 'response_time': round(soulseek_response_time, 1), + 'source_cache_key': source_cache_key, } _status_cache_timestamps['soulseek'] = current_time # Include download source mode so frontend can update labels - download_mode = config_manager.get('download_source.mode', 'hybrid') _status_cache['soulseek']['source'] = download_mode + soulseek_data = { + key: value for key, value in _status_cache['soulseek'].items() + if key != 'source_cache_key' + } # Count active downloads for nav badge active_dl_count = 0 @@ -2167,7 +2189,7 @@ def get_status(): 'metadata_source': metadata_status['metadata_source'], 'spotify': metadata_status['spotify'], 'media_server': _status_cache['media_server'], - 'soulseek': _status_cache['soulseek'], + 'soulseek': soulseek_data, 'active_media_server': active_server, 'enrichment': _get_enrichment_status(), 'active_downloads': active_dl_count, @@ -5887,6 +5909,14 @@ def _find_completed_file_robust(download_dir, api_filename, transfer_dir=None): from difflib import SequenceMatcher from unidecode import unidecode + audio_extensions = { + '.mp3', '.flac', '.m4a', '.aac', '.ogg', '.opus', '.wav', '.wma', '.alac', + '.aiff', '.aif', '.dsf', '.dff', '.ape', + } + + def _is_audio_candidate(path): + return os.path.splitext(str(path or ''))[1].lower() in audio_extensions + # YOUTUBE/TIDAL SUPPORT: Handle encoded filename format "id||title" # Extract just the title part for file matching if '||' in api_filename: @@ -5921,9 +5951,12 @@ def _find_completed_file_robust(download_dir, api_filename, transfer_dir=None): # Skip quarantine folder — contains known-wrong files from AcoustID verification dirs[:] = [d for d in dirs if d != 'ss_quarantine'] for file in files: + file_path = os.path.join(root, file) + if not _is_audio_candidate(file_path): + continue + # Direct basename match if os.path.basename(file) == target_basename: - file_path = os.path.join(root, file) # Fast path: if path aligns with expected directory structure, return now if api_dir_parts and _path_matches_api_dirs(file_path): logger.info(f"Found path-confirmed match in {location_name}: {file_path}") @@ -5940,7 +5973,6 @@ def _find_completed_file_robust(download_dir, api_filename, transfer_dir=None): file_stem, file_ext_part = os.path.splitext(file) stripped_stem = re.sub(r'_\d{10,}$', '', file_stem) if stripped_stem != file_stem and stripped_stem + file_ext_part == target_basename: - file_path = os.path.join(root, file) if api_dir_parts and _path_matches_api_dirs(file_path): logger.info(f"Found path-confirmed dedup match in {location_name}: {file_path}") return file_path, 1.0 @@ -5956,7 +5988,7 @@ def _find_completed_file_robust(download_dir, api_filename, transfer_dir=None): if similarity > highest_fuzzy_similarity: highest_fuzzy_similarity = similarity - best_fuzzy_path = os.path.join(root, file) + best_fuzzy_path = file_path # Return best exact match (disambiguated by path), or fall back to fuzzy if exact_matches: diff --git a/webui/static/downloads.js b/webui/static/downloads.js index 678c1796..7df12aff 100644 --- a/webui/static/downloads.js +++ b/webui/static/downloads.js @@ -3300,6 +3300,34 @@ function closeCandidatesModal() { } } +function _downloadModalBundleProgressPercent(bundle) { + if (!bundle) return 0; + const raw = bundle.progress_percent ?? bundle.progress ?? 0; + let progress = Number(raw); + if (!Number.isFinite(progress)) progress = 0; + if (progress <= 1) progress *= 100; + return Math.max(0, Math.min(100, Math.round(progress))); +} + +function _downloadModalFormatBytes(bytes) { + const value = Number(bytes); + if (!Number.isFinite(value) || value <= 0) return ''; + const units = ['B', 'KB', 'MB', 'GB', 'TB']; + let size = value; + let unit = 0; + while (size >= 1024 && unit < units.length - 1) { + size /= 1024; + unit += 1; + } + const decimals = size >= 10 || unit === 0 ? 0 : 1; + return `${size.toFixed(decimals)} ${units[unit]}`; +} + +function _downloadModalFormatSpeed(bytesPerSecond) { + const formatted = _downloadModalFormatBytes(bytesPerSecond); + return formatted ? `${formatted}/s` : ''; +} + function processModalStatusUpdate(playlistId, data) { // This function contains ALL the existing polling logic from startModalDownloadPolling // Extracted so it can be called from both individual and batched polling @@ -3348,6 +3376,35 @@ function processModalStatusUpdate(playlistId, data) { // Auto-save M3U file for playlists after analysis autoSavePlaylistM3U(playlistId); } + } else if (data.phase === 'album_downloading') { + const analysisFill = document.getElementById(`analysis-progress-fill-${playlistId}`); + const analysisText = document.getElementById(`analysis-progress-text-${playlistId}`); + if (analysisFill) analysisFill.style.width = '100%'; + if (analysisText) analysisText.textContent = 'Analysis complete!'; + + const bundle = data.album_bundle || {}; + const percent = _downloadModalBundleProgressPercent(bundle); + const source = bundle.source ? `${bundle.source} ` : ''; + const release = bundle.release ? ` - ${bundle.release}` : ''; + const speed = _downloadModalFormatSpeed(bundle.speed); + const size = _downloadModalFormatBytes(bundle.size); + const detail = speed || size ? ` (${[speed, size].filter(Boolean).join(' of ')})` : ''; + const downloadFill = document.getElementById(`download-progress-fill-${playlistId}`); + const downloadText = document.getElementById(`download-progress-text-${playlistId}`); + if (downloadFill) downloadFill.style.width = `${percent}%`; + if (downloadText) { + downloadText.textContent = `${source}album ${bundle.state || 'downloading'} ${percent}%${release}${detail}`; + } + + const modal = document.getElementById(`download-missing-modal-${playlistId}`); + if (modal) { + modal.querySelectorAll('[id^="download-"]').forEach(statusEl => { + if (!statusEl.id.startsWith(`download-${playlistId}-`)) return; + if (!statusEl.textContent || statusEl.textContent === '-' || statusEl.textContent.includes('Pending')) { + statusEl.textContent = 'Waiting for album bundle'; + } + }); + } } else if (data.phase === 'downloading' || data.phase === 'complete' || data.phase === 'error') { console.debug(`📊 [Status Update] Processing ${data.phase} phase for playlistId: ${playlistId}, tasks: ${(data.tasks || []).length}`); diff --git a/webui/static/shared-helpers.js b/webui/static/shared-helpers.js index 9840fea3..05ab4fa2 100644 --- a/webui/static/shared-helpers.js +++ b/webui/static/shared-helpers.js @@ -3377,8 +3377,8 @@ function updateServiceStatus(service, statusData, spotifyStatus = null) { // Update download source title on dashboard card if (service === 'soulseek' && statusData.source) { - const sourceNames = { soulseek: 'Soulseek', youtube: 'YouTube', tidal: 'Tidal', qobuz: 'Qobuz', hifi: 'HiFi', deezer_dl: 'Deezer', lidarr: 'Lidarr', soundcloud: 'SoundCloud', hybrid: 'Hybrid' }; - const displayName = sourceNames[statusData.source] || 'Soulseek'; + const sourceNames = { soulseek: 'Soulseek', youtube: 'YouTube', tidal: 'Tidal', qobuz: 'Qobuz', hifi: 'HiFi', deezer_dl: 'Deezer', amazon: 'Amazon', lidarr: 'Lidarr', soundcloud: 'SoundCloud', torrent: 'Torrent', usenet: 'Usenet', hybrid: 'Hybrid' }; + const displayName = sourceNames[statusData.source] || 'Download Source'; const titleEl = document.getElementById('download-source-title'); if (titleEl) titleEl.textContent = displayName; } @@ -3422,8 +3422,8 @@ function updateSidebarServiceStatus(service, statusData, spotifyStatus = null) { // Update download source name based on configured mode if (service === 'soulseek' && statusData.source) { - const sourceNames = { soulseek: 'Soulseek', youtube: 'YouTube', tidal: 'Tidal', qobuz: 'Qobuz', hifi: 'HiFi', deezer_dl: 'Deezer', lidarr: 'Lidarr', soundcloud: 'SoundCloud', hybrid: 'Hybrid' }; - const displayName = sourceNames[statusData.source] || 'Soulseek'; + const sourceNames = { soulseek: 'Soulseek', youtube: 'YouTube', tidal: 'Tidal', qobuz: 'Qobuz', hifi: 'HiFi', deezer_dl: 'Deezer', amazon: 'Amazon', lidarr: 'Lidarr', soundcloud: 'SoundCloud', torrent: 'Torrent', usenet: 'Usenet', hybrid: 'Hybrid' }; + const displayName = sourceNames[statusData.source] || 'Download Source'; const sidebarName = document.getElementById('download-source-name'); if (sidebarName) sidebarName.textContent = displayName; } From b6b83c8bf89559103d8a5b74153523babe80c145 Mon Sep 17 00:00:00 2001 From: Broque Thomas <26755000+Nezreka@users.noreply.github.com> Date: Thu, 21 May 2026 15:19:43 -0700 Subject: [PATCH 17/25] Polish torrent and usenet download UX Clarifies album-bundle progress text in the download modal and active downloads panel so release-first downloads read as downloading a release, then matching tracks after staging. Adds waiting-state copy and tooltips for rows blocked on release staging, plus source-specific library history badge styling for Torrent, Usenet, Staging, and Auto-Import. --- webui/static/downloads.js | 52 +++++++++++++++++++++++++++++----- webui/static/pages-extra.js | 51 ++++++++++++++++++++++++++++----- webui/static/style.css | 31 ++++++++++++++++++++ webui/static/wishlist-tools.js | 5 +++- 4 files changed, 124 insertions(+), 15 deletions(-) diff --git a/webui/static/downloads.js b/webui/static/downloads.js index 7df12aff..36142faf 100644 --- a/webui/static/downloads.js +++ b/webui/static/downloads.js @@ -3328,6 +3328,46 @@ function _downloadModalFormatSpeed(bytesPerSecond) { return formatted ? `${formatted}/s` : ''; } +function _downloadModalSourceLabel(source) { + const labels = { + torrent: 'Torrent', + usenet: 'Usenet', + soulseek: 'Soulseek', + youtube: 'YouTube', + tidal: 'Tidal', + qobuz: 'Qobuz', + hifi: 'HiFi', + deezer_dl: 'Deezer', + amazon: 'Amazon', + lidarr: 'Lidarr', + soundcloud: 'SoundCloud' + }; + const key = String(source || '').toLowerCase(); + return labels[key] || (source ? String(source) : 'Release'); +} + +function _downloadModalBundleStateLabel(state) { + const labels = { + searching: 'searching for release', + downloading: 'downloading release', + staged: 'matching tracks', + failed: 'release failed' + }; + const key = String(state || '').toLowerCase(); + return labels[key] || (state ? String(state).replace(/_/g, ' ') : 'downloading release'); +} + +function _downloadModalBundleProgressText(bundle) { + const percent = _downloadModalBundleProgressPercent(bundle); + const source = _downloadModalSourceLabel(bundle && bundle.source); + const state = _downloadModalBundleStateLabel(bundle && bundle.state); + const release = bundle && bundle.release ? ` - ${bundle.release}` : ''; + const speed = _downloadModalFormatSpeed(bundle && bundle.speed); + const size = _downloadModalFormatBytes(bundle && bundle.size); + const detail = speed || size ? ` (${[speed, size].filter(Boolean).join(' of ')})` : ''; + return `${source} ${state} ${percent}%${release}${detail}`; +} + function processModalStatusUpdate(playlistId, data) { // This function contains ALL the existing polling logic from startModalDownloadPolling // Extracted so it can be called from both individual and batched polling @@ -3384,16 +3424,12 @@ function processModalStatusUpdate(playlistId, data) { const bundle = data.album_bundle || {}; const percent = _downloadModalBundleProgressPercent(bundle); - const source = bundle.source ? `${bundle.source} ` : ''; - const release = bundle.release ? ` - ${bundle.release}` : ''; - const speed = _downloadModalFormatSpeed(bundle.speed); - const size = _downloadModalFormatBytes(bundle.size); - const detail = speed || size ? ` (${[speed, size].filter(Boolean).join(' of ')})` : ''; const downloadFill = document.getElementById(`download-progress-fill-${playlistId}`); const downloadText = document.getElementById(`download-progress-text-${playlistId}`); if (downloadFill) downloadFill.style.width = `${percent}%`; if (downloadText) { - downloadText.textContent = `${source}album ${bundle.state || 'downloading'} ${percent}%${release}${detail}`; + downloadText.textContent = _downloadModalBundleProgressText(bundle); + downloadText.title = 'SoulSync downloads one album release first, then matches the selected tracks from the staged files.'; } const modal = document.getElementById(`download-missing-modal-${playlistId}`); @@ -3401,7 +3437,9 @@ function processModalStatusUpdate(playlistId, data) { modal.querySelectorAll('[id^="download-"]').forEach(statusEl => { if (!statusEl.id.startsWith(`download-${playlistId}-`)) return; if (!statusEl.textContent || statusEl.textContent === '-' || statusEl.textContent.includes('Pending')) { - statusEl.textContent = 'Waiting for album bundle'; + statusEl.textContent = 'Waiting for release'; + statusEl.classList.add('album-bundle-waiting'); + statusEl.title = 'The album release is downloading first. Tracks will move to processing once SoulSync can match files from it.'; } }); } diff --git a/webui/static/pages-extra.js b/webui/static/pages-extra.js index 915040f1..47a65710 100644 --- a/webui/static/pages-extra.js +++ b/webui/static/pages-extra.js @@ -2473,6 +2473,46 @@ function _adlFormatSpeed(bytesPerSecond) { return formatted ? `${formatted}/s` : ''; } +function _adlSourceLabel(source) { + const labels = { + torrent: 'Torrent', + usenet: 'Usenet', + soulseek: 'Soulseek', + youtube: 'YouTube', + tidal: 'Tidal', + qobuz: 'Qobuz', + hifi: 'HiFi', + deezer_dl: 'Deezer', + amazon: 'Amazon', + lidarr: 'Lidarr', + soundcloud: 'SoundCloud' + }; + const key = String(source || '').toLowerCase(); + return labels[key] || (source ? String(source) : 'Release'); +} + +function _adlBundleStateLabel(state) { + const labels = { + searching: 'searching for release', + downloading: 'downloading release', + staged: 'matching tracks', + failed: 'release failed' + }; + const key = String(state || '').toLowerCase(); + return labels[key] || (state ? String(state).replace(/_/g, ' ') : 'downloading release'); +} + +function _adlBundleProgressText(bundle) { + const pct = _adlBundleProgressPercent(bundle); + const source = _adlSourceLabel(bundle && bundle.source); + const state = _adlBundleStateLabel(bundle && bundle.state); + const release = bundle && bundle.release ? ` - ${bundle.release}` : ''; + const speed = _adlFormatSpeed(bundle && bundle.speed); + const size = _adlFormatBytes(bundle && bundle.size); + const detail = speed || size ? ` (${[speed, size].filter(Boolean).join(' of ')})` : ''; + return `${source} ${state} ${pct}%${release}${detail}`; +} + async function adlClearCompleted() { try { const resp = await fetch('/api/downloads/clear-completed', { method: 'POST' }); @@ -2565,12 +2605,7 @@ function _adlRenderBatchPanel() { phaseText = 'Analyzing...'; phaseIcon = ''; } else if (batch.phase === 'album_downloading') { - const source = albumBundle && albumBundle.source ? `${albumBundle.source} ` : ''; - const release = albumBundle && albumBundle.release ? ` - ${albumBundle.release}` : ''; - const speed = _adlFormatSpeed(albumBundle && albumBundle.speed); - const size = _adlFormatBytes(albumBundle && albumBundle.size); - const detail = speed || size ? ` (${[speed, size].filter(Boolean).join(' of ')})` : ''; - phaseText = `${source}album ${albumBundle && albumBundle.state ? albumBundle.state : 'downloading'} ${bundleProgress}%${release}${detail}`; + phaseText = _adlBundleProgressText(albumBundle); phaseIcon = ''; } else if (batch.phase === 'downloading') { phaseText = `${batch.completed}/${total} tracks`; @@ -2635,7 +2670,9 @@ function _adlRenderBatchPanel() {
`; }).join(''); } else { - tracksHtml = '
No tracks loaded
'; + tracksHtml = batch.phase === 'album_downloading' + ? '
Downloading one release first. Track matching starts after staging.
' + : '
No tracks loaded
'; } } diff --git a/webui/static/style.css b/webui/static/style.css index cad3e987..d6c531ed 100644 --- a/webui/static/style.css +++ b/webui/static/style.css @@ -10333,6 +10333,25 @@ body.helper-mode-active #dashboard-activity-feed:hover { border: 1px solid rgba(100, 200, 130, 0.2); } +.library-history-badge.download.source-torrent { + color: #5dade2; + background: rgba(93, 173, 226, 0.1); + border-color: rgba(93, 173, 226, 0.25); +} + +.library-history-badge.download.source-usenet { + color: #a78bfa; + background: rgba(167, 139, 250, 0.1); + border-color: rgba(167, 139, 250, 0.25); +} + +.library-history-badge.download.source-staging, +.library-history-badge.download.source-auto-import { + color: rgba(255, 255, 255, 0.55); + background: rgba(255, 255, 255, 0.05); + border-color: rgba(255, 255, 255, 0.12); +} + .library-history-badge.import { color: rgba(120, 160, 230, 0.9); background: rgba(120, 160, 230, 0.1); @@ -19885,6 +19904,11 @@ body.helper-mode-active #dashboard-activity-feed:hover { inset 0 1px 2px rgba(255, 255, 255, 0.3); } +.album-bundle-waiting { + color: rgba(255, 255, 255, 0.55); + font-style: italic; +} + /* Track Table Section */ .download-tracks-section { /* Premium glassmorphic card matching other sections */ @@ -58357,6 +58381,13 @@ body.reduce-effects *::after { .adl-batch-track-status.failed { color: #ef4444; } .adl-batch-track-status.queued { color: rgba(255, 255, 255, 0.3); } +.adl-batch-release-note { + padding: 6px 0; + color: rgba(255, 255, 255, 0.38); + font-size: 0.7rem; + line-height: 1.4; +} + /* Mini progress bar per track */ .adl-batch-track-progress { width: 100%; diff --git a/webui/static/wishlist-tools.js b/webui/static/wishlist-tools.js index 3807102a..fd8eee6b 100644 --- a/webui/static/wishlist-tools.js +++ b/webui/static/wishlist-tools.js @@ -3449,7 +3449,10 @@ function renderHistoryEntry(entry) { const parts = []; if (entry.download_source) parts.push(entry.download_source); if (entry.quality) parts.push(entry.quality); - badge = parts.map(p => `${escapeHtml(p)}`).join(''); + badge = parts.map(p => { + const cls = String(p || '').toLowerCase().replace(/[^a-z0-9]+/g, '-').replace(/^-|-$/g, ''); + return `${escapeHtml(p)}`; + }).join(''); } else if (entry.event_type === 'import' && entry.server_source) { const sourceName = { plex: 'Plex', jellyfin: 'Jellyfin', navidrome: 'Navidrome' }[entry.server_source] || entry.server_source; badge = `${escapeHtml(sourceName)}`; From dee200b26749767c74e600d6fe07b50d3798a7a8 Mon Sep 17 00:00:00 2001 From: Broque Thomas <26755000+Nezreka@users.noreply.github.com> Date: Thu, 21 May 2026 15:28:48 -0700 Subject: [PATCH 18/25] Add torrent usenet PR notes and test updates Adds the PR description for the torrent/usenet release-staging feature and updates drifted tests for the new plugin registry entries and search exclude_sources signature. Verified with the focused pytest command covering cancellation and default registry source registration. --- pr_description.md | 76 +++++++++++++++++++ tests/downloads/test_downloads_task_worker.py | 2 +- tests/test_download_plugin_conformance.py | 1 + 3 files changed, 78 insertions(+), 1 deletion(-) create mode 100644 pr_description.md diff --git a/pr_description.md b/pr_description.md new file mode 100644 index 00000000..83fda10a --- /dev/null +++ b/pr_description.md @@ -0,0 +1,76 @@ +## Summary + +Adds torrent and usenet as release-oriented download sources backed by Prowlarr and the configured downloader clients. These sources can download full releases, stage the resulting audio files, and let SoulSync match/import the requested tracks through the existing post-processing pipeline. + +This PR also improves the torrent/usenet user experience with clearer release-download progress, source-aware service labels, library history provenance, and safer staged-release matching for album files that include featured-artist or bonus-track filename noise. + +## Scope + +- Adds torrent and usenet plugin support for release downloads. +- Adds album-bundle staging for single-source torrent/usenet album downloads. +- Keeps hybrid album downloads on per-track-capable sources by excluding torrent/usenet from hybrid album per-track searches. +- Allows torrent/usenet in hybrid for non-album track downloads, such as playlist singles and wishlist tracks. +- Selects the requested audio file from completed release folders instead of importing the first audio file blindly. +- Reports live album-bundle progress to the Downloads page and download modal. +- Records torrent/usenet provenance in library history. +- Adds UI polish for release-first download states. + +## Behavior Gates + +- Users who do not select torrent or usenet as a download source should not hit the new download paths. +- Single-source `torrent` / `usenet` album downloads use the release staging flow. +- Hybrid album downloads skip torrent/usenet and continue through the existing per-track source chain. +- Hybrid non-album downloads may use torrent/usenet when they are included in the hybrid order. + +## Notable Implementation Details + +- Torrent/usenet release downloads use private per-batch staging folders under the configured album-bundle staging root. +- Post-processing receives the real source label (`torrent` / `usenet`) so library history no longer falls back to `Soulseek`. +- Staging matching strips only conservative noise like `(feat. Artist)` and `(Bonus Track)` while preserving meaningful version text like `remix`, `extended`, `live`, and `acoustic`. +- When a staged release does not contain a requested track, the task is marked not found instead of repeatedly searching/downloading the same release. +- Completed release downloads expose all discovered audio files so post-processing can choose the best matching track. + +## Testing + +Manually tested: + +- Torrent-only album download for `good kid, m.A.A.d city (Deluxe)`. +- Torrent playlist/single-track flow. +- Album-bundle progress in the download modal and Downloads page. +- Torrent source labeling in library history for new imports. +- Staging match behavior for featured-artist filenames and bonus-track labels. +- Quarantine behavior for wrong-version matches. + +Automated tests run during development: + +```bash +.venv/bin/python -m pytest \ + tests/downloads/test_downloads_status.py \ + tests/test_album_bundle_dispatch.py \ + tests/downloads/test_downloads_staging.py \ + tests/test_torrent_usenet_plugins.py +``` + +```bash +.venv/bin/python -m pytest \ + tests/downloads/test_downloads_validation.py \ + tests/test_manual_pick_no_auto_retry.py \ + tests/downloads/test_downloads_post_processing.py \ + tests/downloads/test_downloads_task_worker.py \ + tests/imports/test_import_side_effects.py +``` + +Focused checks also passed for: + +- staged release feature suffix matching +- bonus-track title matching +- wrong-version separation +- private torrent album staging miss handling +- torrent/usenet history source labels + +## Reviewer Notes + +- This is intentionally gated behind torrent/usenet source selection, but it is still a new release-oriented download path and should be considered beta/experimental for first release. +- Remote downloader setups need SoulSync to be able to read the downloader save path. Local/all-in-one setups should be the easiest path. +- Existing library history rows that were previously recorded as `Soulseek` are not backfilled by this PR. +- Release matching is naturally fuzzier than track-native sources, so reviewer focus should stay on false positives, version handling, and staged-file selection. diff --git a/tests/downloads/test_downloads_task_worker.py b/tests/downloads/test_downloads_task_worker.py index ea605d6e..e5156b4a 100644 --- a/tests/downloads/test_downloads_task_worker.py +++ b/tests/downloads/test_downloads_task_worker.py @@ -319,7 +319,7 @@ def test_cancellation_mid_query_returns_without_completion(): _seed_task() rec = _Recorder() - def _cancel_during_search(query, timeout=30): + def _cancel_during_search(query, timeout=30, exclude_sources=None): download_tasks['t1']['status'] = 'cancelled' async def _empty(): diff --git a/tests/test_download_plugin_conformance.py b/tests/test_download_plugin_conformance.py index 4f380076..68483adc 100644 --- a/tests/test_download_plugin_conformance.py +++ b/tests/test_download_plugin_conformance.py @@ -85,6 +85,7 @@ def test_default_registry_registers_all_sources(): expected = { 'soulseek', 'youtube', 'tidal', 'qobuz', 'hifi', 'deezer', 'lidarr', 'soundcloud', 'amazon', + 'torrent', 'usenet', } assert set(registry.names()) == expected From 8012f41ef78e3565d9e88f7ac091572a02267d39 Mon Sep 17 00:00:00 2001 From: Broque Thomas <26755000+Nezreka@users.noreply.github.com> Date: Thu, 21 May 2026 16:49:12 -0700 Subject: [PATCH 19/25] fix(album-completeness): block cross-artist auto-fill MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Reported bug: filling Jamiroquai's "Light Years" single pulled in Gut's "Light Years" album tracks (different artist, completely different genre — track titles like "Wound Fuck" and "Eat My Cum" made the contamination obvious). The Album Completeness auto-fill was the only file-copying path with a loose 0.50 SequenceMatcher artist gate, which let unrelated candidates through whenever the title matched well. Two-stage defense now sits on the only album-fill code path (_fix_incomplete_album in core/repair_worker.py): - Stage 1 — _album_fill_target_artist_allows_track. Pre-search gate: before doing any library lookup for a missing track, refuse to operate if the missing track's source artist(s) don't match the target album's artist. Compilation albums (album_artist in {'various artists', 'various', 'soundtrack'}) bypass the gate so legitimate VA releases still work. Empty source-artist metadata also bypasses for backward compat with older missing-track records that don't carry per-track artist. - Stage 2 — _album_fill_artist_names_match. Replaces the old 0.50 SequenceMatcher with an alias-aware 0.82 threshold that uses core.matching.artist_aliases when available (handles diacritic variants like Beyoncé/Beyonce and known stage names) with a normalized-similarity fallback if the aliases module isn't importable. Skipped candidates are logged at debug so a later support ticket can show what was rejected and why. Tests in tests/test_repair_worker_album_fill.py reproduce the exact reported scenario: target album "Light Years" by Gut + missing track from a Jamiroquai source → skipped with a logged warning, no copy attempted, wishlist not poisoned. Second test covers Stage 2 directly with a wrong-artist library candidate. Existing test_perform_album_fill_copy_branch still passes. Note: this fix prevents NEW cross-artist contamination via Album Completeness. It does not clean up the data anomaly that made Gut's library entry appear to have a "Light Years" album in the first place — that's a separate data-quality issue worth investigating if it recurs. --- core/repair_worker.py | 91 +++++++++++++- tests/test_repair_worker_album_fill.py | 161 +++++++++++++++++++++++++ webui/static/helper.js | 1 + 3 files changed, 251 insertions(+), 2 deletions(-) diff --git a/core/repair_worker.py b/core/repair_worker.py index 227d144a..9af2ff30 100644 --- a/core/repair_worker.py +++ b/core/repair_worker.py @@ -35,6 +35,64 @@ logger = get_logger("repair_worker") AUDIO_EXTENSIONS = {'.mp3', '.flac', '.ogg', '.opus', '.m4a', '.aac', '.wav', '.wma', '.aiff', '.aif'} +def _album_fill_artist_names_match(expected_artist: str, candidate_artist: str) -> bool: + """Strict artist gate for Album Completeness auto-fill. + + Auto-fill moves/copies files into an existing album, so artist identity + must outrank album/title similarity. Use the alias-aware matcher when it + is available, then fall back to conservative normalized similarity. + """ + expected = (expected_artist or '').strip() + candidate = (candidate_artist or '').strip() + if not expected or not candidate: + return False + + try: + from core.matching.artist_aliases import artist_names_match + matched, score = artist_names_match(expected, candidate, threshold=0.82) + if matched: + return True + if score < 0.82: + return False + except Exception as alias_err: + logger.debug("artist_names_match unavailable, using fallback: %s", alias_err) + + try: + from core.matching_engine import MusicMatchingEngine + engine = MusicMatchingEngine() + expected_norm = engine.clean_artist(expected) + candidate_norm = engine.clean_artist(candidate) + except Exception: + expected_norm = expected.lower() + candidate_norm = candidate.lower() + + if not expected_norm or not candidate_norm: + return False + return expected_norm == candidate_norm or SequenceMatcher(None, expected_norm, candidate_norm).ratio() >= 0.82 + + +def _album_fill_target_artist_allows_track(album_artist: str, track_artists: List[str]) -> bool: + """Return whether a source track can be auto-filled into an album artist. + + Compilation-style album artists are allowed to contain varied track + artists. Normal albums require at least one source track artist to match + the target album artist before any local candidate is considered. + """ + album_artist = (album_artist or '').strip() + if not album_artist: + return False + + normalized_album_artist = album_artist.lower().strip() + if normalized_album_artist in {'various artists', 'various', 'soundtrack'}: + return True + + source_artists = [str(a).strip() for a in (track_artists or []) if str(a or '').strip()] + if not source_artists: + return True + + return any(_album_fill_artist_names_match(album_artist, artist) for artist in source_artists) + + def _split_acoustid_credit(credit: str) -> List[str]: """Split an AcoustID artist credit into individual contributor names. @@ -2097,6 +2155,21 @@ class RepairWorker: track_details.append({'track': track_name, 'status': 'skipped', 'reason': 'no track name'}) continue + if not _album_fill_target_artist_allows_track(artist_name, track_artists): + skipped_count += 1 + logger.warning( + "Album auto-fill skipped '%s': source artist(s) %s do not match target album artist '%s'", + track_name, track_artists, artist_name, + ) + track_details.append({ + 'track': track_name, + 'status': 'skipped', + 'reason': 'source artist does not match target album artist', + 'source_artists': track_artists, + 'target_artist': artist_name, + }) + continue + # Search library for this track candidates = self.db.search_tracks(title=track_name, artist=artist_search, limit=20) @@ -2117,8 +2190,22 @@ class RepairWorker: # Artist match (more lenient) cand_artist = getattr(cand, 'artist_name', '') or '' - artist_sim = SequenceMatcher(None, artist_search.lower(), cand_artist.lower()).ratio() - if artist_sim < 0.50: + candidate_artist_fields = [ + cand_artist, + getattr(cand, 'track_artist', '') or '', + ] + expected_artist_names = track_artists or [artist_name] + if not any( + _album_fill_artist_names_match(expected, candidate) + for expected in expected_artist_names + for candidate in candidate_artist_fields + ): + logger.debug( + "Album auto-fill rejected candidate '%s' by '%s' for expected artist(s) %s", + getattr(cand, 'title', ''), + cand_artist, + expected_artist_names, + ) continue # Quality gate diff --git a/tests/test_repair_worker_album_fill.py b/tests/test_repair_worker_album_fill.py index 57a44bcd..ee1bf28e 100644 --- a/tests/test_repair_worker_album_fill.py +++ b/tests/test_repair_worker_album_fill.py @@ -44,6 +44,167 @@ if "config.settings" not in sys.modules: from core.repair_worker import RepairWorker +def test_incomplete_album_auto_fill_skips_source_artist_mismatch(tmp_path): + """Album Completeness must never fill a target album with another artist.""" + existing_path = tmp_path / "Gut" / "Light Years" / "01 - Wound Fuck.flac" + existing_path.parent.mkdir(parents=True) + existing_path.write_bytes(b"existing") + + candidate_path = tmp_path / "Jamiroquai" / "Light Years.flac" + candidate_path.parent.mkdir(parents=True) + candidate_path.write_bytes(b"candidate") + + class _FakeDB: + def __init__(self): + self.search_calls = [] + self.wishlist = [] + + def get_tracks_by_album(self, album_id): + if album_id == "target-album": + return [ + SimpleNamespace( + id="target-existing-1", + album_id="target-album", + artist_id="gut", + title="Wound Fuck", + track_number=1, + file_path=str(existing_path), + bitrate=9999, + ), + ] + return [] + + def search_tracks(self, title="", artist="", limit=50, server_source=None): + self.search_calls.append((title, artist, limit, server_source)) + return [ + SimpleNamespace( + id="candidate-1", + album_id="single-album", + artist_id="jamiroquai", + artist_name="Jamiroquai", + title="Light Years", + track_number=1, + file_path=str(candidate_path), + bitrate=9999, + ), + ] + + def add_to_wishlist(self, *args, **kwargs): + self.wishlist.append((args, kwargs)) + + worker = RepairWorker.__new__(RepairWorker) + worker.db = _FakeDB() + worker.transfer_folder = str(tmp_path) + worker._config_manager = None + + result = worker._fix_incomplete_album( + "album", + "target-album", + None, + { + "album_id": "target-album", + "album_title": "Light Years", + "artist": "Gut", + "missing_tracks": [ + { + "name": "Light Years", + "track_number": 2, + "disc_number": 1, + "source": "spotify", + "source_track_id": "sp-light-years", + "artists": ["Jamiroquai"], + }, + ], + }, + ) + + assert result["success"] is False + assert result["fixed"] == 0 + assert result["skipped"] == 1 + assert result["details"][0]["reason"] == "source artist does not match target album artist" + assert worker.db.search_calls == [] + assert worker.db.wishlist == [] + + +def test_incomplete_album_auto_fill_rejects_wrong_artist_candidate(tmp_path): + """Exact title is not enough when the candidate artist differs.""" + existing_path = tmp_path / "album" / "01 - Existing.flac" + existing_path.parent.mkdir(parents=True) + existing_path.write_bytes(b"existing") + candidate_path = tmp_path / "wrong" / "02 - Light Years.flac" + candidate_path.parent.mkdir(parents=True) + candidate_path.write_bytes(b"wrong") + + class _FakeDB: + def __init__(self): + self.wishlist = [] + + def get_tracks_by_album(self, album_id): + if album_id == "target-album": + return [ + SimpleNamespace( + id="target-existing-1", + album_id="target-album", + artist_id="jamiroquai", + title="Existing", + track_number=1, + file_path=str(existing_path), + bitrate=9999, + ), + ] + return [] + + def search_tracks(self, title="", artist="", limit=50, server_source=None): + return [ + SimpleNamespace( + id="candidate-1", + album_id="wrong-album", + artist_id="gut", + artist_name="Gut", + title="Light Years", + track_number=1, + file_path=str(candidate_path), + bitrate=9999, + ), + ] + + def add_to_wishlist(self, *args, **kwargs): + self.wishlist.append((args, kwargs)) + + worker = RepairWorker.__new__(RepairWorker) + worker.db = _FakeDB() + worker.transfer_folder = str(tmp_path) + worker._config_manager = None + + result = worker._fix_incomplete_album( + "album", + "target-album", + None, + { + "album_id": "target-album", + "album_title": "Light Years", + "artist": "Jamiroquai", + "spotify_album_id": "sp-album", + "expected_tracks": 2, + "missing_tracks": [ + { + "name": "Light Years", + "track_number": 2, + "disc_number": 1, + "source": "spotify", + "source_track_id": "sp-light-years", + "artists": ["Jamiroquai"], + }, + ], + }, + ) + + assert result["success"] is True + assert result["fixed"] == 0 + assert result["wishlisted"] == 1 + assert worker.db.wishlist + + def test_perform_album_fill_copy_branch_generates_track_id(tmp_path, monkeypatch): src_path = tmp_path / "source.flac" src_path.write_bytes(b"fake-audio") diff --git a/webui/static/helper.js b/webui/static/helper.js index a48529f1..0d57a8b2 100644 --- a/webui/static/helper.js +++ b/webui/static/helper.js @@ -3415,6 +3415,7 @@ function closeHelperSearch() { const WHATS_NEW = { '2.6.0': [ { unreleased: true }, + { title: 'Fix: Album Completeness no longer cross-contaminates artists', desc: 'reported case: filling a Jamiroquai "Light Years" single brought in tracks from Gut\'s "Light Years" album — completely different artist, completely different genre. Root cause: the auto-fill artist gate was a loose 0.50 SequenceMatcher threshold that could let unrelated candidates through. Two new defenses now block this entirely. Stage 1 skips the whole track-fill operation up front if the missing track\'s source artist doesn\'t match the target album artist — compilation albums (various artists / soundtrack) still pass through. Stage 2 replaces the per-candidate 0.50 SequenceMatcher with an alias-aware 0.82 strict matcher that handles diacritics (Beyoncé/Beyonce) and known artist aliases. Both stages logged with structured warnings so future mismatches are diagnosable from the logs.' }, { title: 'Code review refactor pass on the torrent / usenet flow', desc: 'cleanup commits before review. Lifted the shared album-pick + staging-collision helpers out of torrent.py into a new core/download_plugins/album_bundle.py module so usenet.py no longer reaches into a sibling plugin\'s private surface. Lifted the ~90-line inline album-bundle gate out of run_full_missing_tracks_process into core/downloads/album_bundle_dispatch.py with a clean pure-predicate / inject-deps design so the gate is unit-testable in isolation. Replaced the staging matcher\'s direct download_batches import with an injected get_batch_field accessor on StagingDeps. Made the 6h poll timeout configurable via download_source.album_bundle_timeout_seconds. Added atomic .tmp + rename copy so the Auto-Import worker can never observe a partial audio file during the album-bundle copy loop. 49 new tests across album_bundle, album_bundle_dispatch, and staging-provenance modules pin the contracts.' }, { title: 'Hybrid mode skips torrent / usenet on album batches', desc: 'follow-up to the album-bundle flow. When download mode is set to Hybrid AND the batch is an album, the per-track search loop now silently strips torrent and usenet from the source chain — they\'re release-level sources that can\'t match per-track meaningfully, and the album-bundle handling only fires in single-source mode. Without the skip, hybrid + torrent-first would fire N redundant Prowlarr searches per album and rely on Auto-Import sweeping Staging to recover. Cleaner now: hybrid falls straight through to per-track-compatible sources (Soulseek / streaming) for albums, and torrent / usenet still get a shot for single-track / wishlist / basic-search use cases where per-track makes sense.' }, { title: 'Album-bundle flow for torrent / usenet downloads', desc: 'fixes the core architectural problem with indexer-based sources. Prowlarr returns release-level torrents — searching per-track for "Luther (with SZA)" against the GNX album torrent scores near-zero and the orchestrator rejects every candidate. New gated flow: when downloading an album AND torrent or usenet is the single active source (not hybrid), SoulSync now does ONE Prowlarr search for the whole release, picks the best torrent (prefers FLAC, high seeders, reasonable size — drops single-track torrents that snuck in), hands it to your torrent / usenet client, walks the resulting audio files (extracting .zip/.rar/.tar if needed), and drops them all into the staging folder. The existing per-track staging matcher then imports each one to the library by fuzzy title match — same path as the Auto-Import flow. Gate is strictly opt-in: per-track flow is completely untouched for hybrid mode, non-album downloads, and every other source. 5 new tests cover the album picker (seeded-FLAC preference, size floor for single-track torrents, fallback when all candidates are small) and the staging path collision handler.' }, From f1d4f78e0eb38b2856f1304e195bd7313e4781e2 Mon Sep 17 00:00:00 2001 From: Broque Thomas <26755000+Nezreka@users.noreply.github.com> Date: Thu, 21 May 2026 17:41:54 -0700 Subject: [PATCH 20/25] Repair stale media schema during refresh Ensure upgraded databases have the tracks.file_size and albums.api_track_count columns after all legacy migrations run. Add defensive repair paths for Jellyfin track imports and album track-count caching so stale schemas self-heal instead of dropping full-refresh track imports. Tests cover legacy schema repair and api_track_count self-repair. --- core/worker_utils.py | 11 +++++ database/music_database.py | 44 ++++++++++++++++++ tests/test_library_disk_usage.py | 47 ++++++++++++++++++++ tests/test_worker_utils_album_track_count.py | 20 +++++++++ 4 files changed, 122 insertions(+) diff --git a/core/worker_utils.py b/core/worker_utils.py index d6f42523..e572c702 100644 --- a/core/worker_utils.py +++ b/core/worker_utils.py @@ -64,6 +64,17 @@ def set_album_api_track_count(cursor, album_id, count): (count, album_id), ) except Exception as e: + if "api_track_count" in str(e) and "no such column" in str(e).lower(): + try: + cursor.execute("ALTER TABLE albums ADD COLUMN api_track_count INTEGER DEFAULT NULL") + cursor.execute( + "UPDATE albums SET api_track_count = ? WHERE id = ?", + (count, album_id), + ) + logger.info("Repaired missing api_track_count column while caching album track count") + return + except Exception as repair_error: + e = repair_error logger.warning( "Failed to cache api_track_count for album %s: %s", album_id, e ) diff --git a/database/music_database.py b/database/music_database.py index 02cc5246..c6980623 100644 --- a/database/music_database.py +++ b/database/music_database.py @@ -766,6 +766,8 @@ class MusicDatabase: except Exception as ps_err: logger.error(f"Personalized-playlist schema init failed: {ps_err}") + self._ensure_core_media_schema_columns(cursor) + conn.commit() logger.info("Database initialized successfully") @@ -775,6 +777,29 @@ class MusicDatabase: self._init_manual_library_match_table() + def _ensure_core_media_schema_columns(self, cursor): + """Repair required media-library columns that older migrations may miss. + + A few legacy migrations rebuild artists/albums/tracks in place. Newer + installs get these columns from CREATE TABLE, but upgraded databases can + occasionally miss one if a previous migration path failed or was marked + complete before the column existed. + """ + try: + cursor.execute("PRAGMA table_info(tracks)") + track_cols = {c[1] for c in cursor.fetchall()} + if track_cols and 'file_size' not in track_cols: + cursor.execute("ALTER TABLE tracks ADD COLUMN file_size INTEGER") + logger.info("Repaired missing file_size column on tracks table") + + cursor.execute("PRAGMA table_info(albums)") + album_cols = {c[1] for c in cursor.fetchall()} + if album_cols and 'api_track_count' not in album_cols: + cursor.execute("ALTER TABLE albums ADD COLUMN api_track_count INTEGER DEFAULT NULL") + logger.info("Repaired missing api_track_count column on albums table") + except Exception as e: + logger.error("Error repairing core media schema columns: %s", e) + def _add_mirrored_playlist_explored_column(self, cursor): """Add explored_at column to mirrored_playlists to persist explore badge.""" try: @@ -5499,6 +5524,25 @@ class MusicDatabase: return True except Exception as e: + error_text = str(e).lower() + if ( + 'file_size' in error_text + and ('no such column' in error_text or 'no column named' in error_text) + and retry_count < max_retries - 1 + ): + try: + repair_conn = conn if 'conn' in locals() else self._get_connection() + repair_cursor = repair_conn.cursor() + self._ensure_core_media_schema_columns(repair_cursor) + repair_conn.commit() + if repair_conn is not conn: + repair_conn.close() + retry_count += 1 + logger.info("Repaired missing file_size column while importing media track; retrying") + continue + except Exception as schema_error: + logger.error("Failed to repair tracks.file_size during track import: %s", schema_error) + retry_count += 1 if "database is locked" in str(e).lower() and retry_count < max_retries: logger.warning(f"Database locked on track '{getattr(track_obj, 'title', 'Unknown')}', retrying {retry_count}/{max_retries}...") diff --git a/tests/test_library_disk_usage.py b/tests/test_library_disk_usage.py index 00707aa7..1214da8c 100644 --- a/tests/test_library_disk_usage.py +++ b/tests/test_library_disk_usage.py @@ -69,6 +69,53 @@ def test_file_size_column_exists_after_init(db: MusicDatabase) -> None: assert 'file_size' in cols +def test_legacy_media_schema_repairs_required_refresh_columns(tmp_path: Path) -> None: + """Upgraded installs can have old library tables plus migration markers. + Startup must repair the columns full refresh writes later.""" + db_path = tmp_path / 'legacy_missing_media_columns.db' + conn = sqlite3.connect(str(db_path)) + cur = conn.cursor() + cur.execute("CREATE TABLE metadata (key TEXT PRIMARY KEY, value TEXT)") + cur.execute("INSERT INTO metadata (key, value) VALUES ('id_columns_migrated', 'true')") + cur.execute(""" + CREATE TABLE artists ( + id TEXT PRIMARY KEY, + name TEXT NOT NULL + ) + """) + cur.execute(""" + CREATE TABLE albums ( + id TEXT PRIMARY KEY, + artist_id TEXT NOT NULL, + title TEXT NOT NULL + ) + """) + cur.execute(""" + CREATE TABLE tracks ( + id TEXT PRIMARY KEY, + album_id TEXT NOT NULL, + artist_id TEXT NOT NULL, + title TEXT NOT NULL, + file_path TEXT, + bitrate INTEGER + ) + """) + conn.commit() + conn.close() + + repaired = MusicDatabase(database_path=str(db_path)) + conn = repaired._get_connection() + cur = conn.cursor() + cur.execute("PRAGMA table_info(tracks)") + track_cols = {row[1] for row in cur.fetchall()} + cur.execute("PRAGMA table_info(albums)") + album_cols = {row[1] for row in cur.fetchall()} + conn.close() + + assert 'file_size' in track_cols + assert 'api_track_count' in album_cols + + def test_existing_tracks_have_null_file_size_after_migration(db: MusicDatabase) -> None: """Backward-compat: rows inserted via the OLD schema (no file_size) must still be readable, and querying file_size returns NULL — not diff --git a/tests/test_worker_utils_album_track_count.py b/tests/test_worker_utils_album_track_count.py index 4b988d70..cf96b535 100644 --- a/tests/test_worker_utils_album_track_count.py +++ b/tests/test_worker_utils_album_track_count.py @@ -1,6 +1,8 @@ """Tests for `worker_utils.set_album_api_track_count` — the shared helper enrichment workers call to cache authoritative track counts.""" +import sqlite3 + from core.worker_utils import set_album_api_track_count @@ -114,3 +116,21 @@ def test_swallows_cursor_execute_errors(): cursor = _BrokenCursor() # Should not raise. set_album_api_track_count(cursor, "album-z", 10) + + +def test_repairs_missing_api_track_count_column(): + conn = sqlite3.connect(":memory:") + cursor = conn.cursor() + cursor.execute("CREATE TABLE albums (id TEXT PRIMARY KEY, title TEXT)") + cursor.execute("INSERT INTO albums (id, title) VALUES ('album-z', 'Album')") + + set_album_api_track_count(cursor, "album-z", 10) + + cursor.execute("PRAGMA table_info(albums)") + cols = {row[1] for row in cursor.fetchall()} + cursor.execute("SELECT api_track_count FROM albums WHERE id = 'album-z'") + row = cursor.fetchone() + conn.close() + + assert 'api_track_count' in cols + assert row[0] == 10 From b9af4ef4efa6f4a67113d7c26c27f0e452af80dc Mon Sep 17 00:00:00 2001 From: Broque Thomas <26755000+Nezreka@users.noreply.github.com> Date: Thu, 21 May 2026 17:50:30 -0700 Subject: [PATCH 21/25] Handle transient SQLite IO during maintenance Keep full refresh moving when post-clear VACUUM hits a transient disk I/O error, and retry clear_server_data once when the clear step itself sees the same transient SQLite failure. Retry metadata cache maintenance writes once on transient disk I/O errors so first-attempt cache jobs do not fail when an immediate retry would succeed. Tests cover best-effort VACUUM, clear retry behavior, and cache maintenance retry behavior. --- core/metadata/cache.py | 230 +++++++++--------- database/music_database.py | 87 ++++--- .../metadata/test_cache_maintenance_retry.py | 44 ++++ tests/test_database_io_resilience.py | 98 ++++++++ 4 files changed, 316 insertions(+), 143 deletions(-) create mode 100644 tests/metadata/test_cache_maintenance_retry.py create mode 100644 tests/test_database_io_resilience.py diff --git a/core/metadata/cache.py b/core/metadata/cache.py index 2974c5eb..b49daae5 100644 --- a/core/metadata/cache.py +++ b/core/metadata/cache.py @@ -8,6 +8,7 @@ Transparent to callers: check cache before API call, store after success. import json import logging import threading +import time from datetime import datetime from typing import Optional, Dict, List, Tuple @@ -49,6 +50,33 @@ class MetadataCache: from database.music_database import get_database return get_database() + @staticmethod + def _is_transient_sqlite_io_error(exc: Exception) -> bool: + return 'disk i/o error' in str(exc).lower() + + def _run_maintenance_write(self, label: str, operation, default: int = 0) -> int: + """Run a maintenance write with one retry for transient SQLite I/O.""" + for attempt in range(2): + try: + db = self._get_db() + conn = db._get_connection() + try: + return operation(conn) + finally: + conn.close() + except Exception as e: + if self._is_transient_sqlite_io_error(e) and attempt == 0: + logger.warning( + "%s hit transient SQLite disk I/O error; retrying once: %s", + label, + e, + ) + time.sleep(0.25) + continue + logger.error("%s error: %s", label, e) + return default + return default + # ─── Entity Methods ─────────────────────────────────────────────── def get_entity(self, source: str, entity_type: str, entity_id: str) -> Optional[dict]: @@ -566,137 +594,113 @@ class MetadataCache: def evict_expired(self) -> int: """Delete entries that have exceeded their TTL. Returns count of evicted entries.""" - try: - db = self._get_db() - conn = db._get_connection() - try: - cursor = conn.cursor() + def _operation(conn): + cursor = conn.cursor() - # Entities - cursor.execute(""" - DELETE FROM metadata_cache_entities - WHERE julianday('now') - julianday(updated_at) > ttl_days - """) - entity_count = cursor.rowcount + # Entities + cursor.execute(""" + DELETE FROM metadata_cache_entities + WHERE julianday('now') - julianday(updated_at) > ttl_days + """) + entity_count = cursor.rowcount - # Searches - cursor.execute(""" - DELETE FROM metadata_cache_searches - WHERE julianday('now') - julianday(created_at) > ttl_days - """) - search_count = cursor.rowcount + # Searches + cursor.execute(""" + DELETE FROM metadata_cache_searches + WHERE julianday('now') - julianday(created_at) > ttl_days + """) + search_count = cursor.rowcount - conn.commit() - total = entity_count + search_count - if total > 0: - logger.info(f"Evicted {total} expired cache entries ({entity_count} entities, {search_count} searches)") - return total - finally: - conn.close() - except Exception as e: - logger.error(f"Cache eviction error: {e}") - return 0 + conn.commit() + total = entity_count + search_count + if total > 0: + logger.info(f"Evicted {total} expired cache entries ({entity_count} entities, {search_count} searches)") + return total + + return self._run_maintenance_write("Cache eviction", _operation) def clean_junk_entities(self) -> int: """Delete cached entities with empty/placeholder names.""" - try: - db = self._get_db() - conn = db._get_connection() - try: - cursor = conn.cursor() - junk_names = "', '".join(self._JUNK_NAMES - {''}) # exclude empty, handled separately - cursor.execute(f""" - DELETE FROM metadata_cache_entities - WHERE (name IS NULL - OR TRIM(name) = '' - OR LOWER(TRIM(name)) IN ('{junk_names}')) - AND entity_id NOT LIKE '%\\_features' ESCAPE '\\' - AND entity_id NOT LIKE '%\\_tracks' ESCAPE '\\' - """) - count = cursor.rowcount - conn.commit() - if count > 0: - logger.info(f"Cleaned {count} junk entities from cache") - return count - finally: - conn.close() - except Exception as e: - logger.error(f"Junk cleanup error: {e}") - return 0 + def _operation(conn): + cursor = conn.cursor() + junk_names = "', '".join(self._JUNK_NAMES - {''}) # exclude empty, handled separately + cursor.execute(f""" + DELETE FROM metadata_cache_entities + WHERE (name IS NULL + OR TRIM(name) = '' + OR LOWER(TRIM(name)) IN ('{junk_names}')) + AND entity_id NOT LIKE '%\\_features' ESCAPE '\\' + AND entity_id NOT LIKE '%\\_tracks' ESCAPE '\\' + """) + count = cursor.rowcount + conn.commit() + if count > 0: + logger.info(f"Cleaned {count} junk entities from cache") + return count + + return self._run_maintenance_write("Junk cleanup", _operation) def clean_orphaned_searches(self) -> int: """Delete search results where <50% of referenced entities still exist.""" - try: - db = self._get_db() - conn = db._get_connection() - try: - cursor = conn.cursor() - cursor.execute("SELECT id, source, search_type, result_ids FROM metadata_cache_searches") - rows = cursor.fetchall() + def _operation(conn): + cursor = conn.cursor() + cursor.execute("SELECT id, source, search_type, result_ids FROM metadata_cache_searches") + rows = cursor.fetchall() - dead_ids = [] - for row in rows: - try: - result_ids = json.loads(row['result_ids'] or '[]') - except (json.JSONDecodeError, TypeError): - dead_ids.append(row['id']) - continue + dead_ids = [] + for row in rows: + try: + result_ids = json.loads(row['result_ids'] or '[]') + except (json.JSONDecodeError, TypeError): + dead_ids.append(row['id']) + continue - if not result_ids: - dead_ids.append(row['id']) - continue + if not result_ids: + dead_ids.append(row['id']) + continue - # Check how many referenced entities still exist - placeholders = ','.join('?' * len(result_ids)) - cursor.execute(f""" - SELECT COUNT(*) FROM metadata_cache_entities - WHERE source = ? AND entity_type = ? AND entity_id IN ({placeholders}) - """, [row['source'], row['search_type']] + list(result_ids)) - found = cursor.fetchone()[0] + # Check how many referenced entities still exist + placeholders = ','.join('?' * len(result_ids)) + cursor.execute(f""" + SELECT COUNT(*) FROM metadata_cache_entities + WHERE source = ? AND entity_type = ? AND entity_id IN ({placeholders}) + """, [row['source'], row['search_type']] + list(result_ids)) + found = cursor.fetchone()[0] - if found < len(result_ids) * 0.5: - dead_ids.append(row['id']) + if found < len(result_ids) * 0.5: + dead_ids.append(row['id']) - if dead_ids: - # Delete in chunks to stay under SQLite variable limit - for i in range(0, len(dead_ids), 400): - chunk = dead_ids[i:i + 400] - placeholders = ','.join('?' * len(chunk)) - cursor.execute(f"DELETE FROM metadata_cache_searches WHERE id IN ({placeholders})", chunk) - conn.commit() + if dead_ids: + # Delete in chunks to stay under SQLite variable limit + for i in range(0, len(dead_ids), 400): + chunk = dead_ids[i:i + 400] + placeholders = ','.join('?' * len(chunk)) + cursor.execute(f"DELETE FROM metadata_cache_searches WHERE id IN ({placeholders})", chunk) + conn.commit() - count = len(dead_ids) - if count > 0: - logger.info(f"Cleaned {count} orphaned search results from cache") - return count - finally: - conn.close() - except Exception as e: - logger.error(f"Orphan search cleanup error: {e}") - return 0 + count = len(dead_ids) + if count > 0: + logger.info(f"Cleaned {count} orphaned search results from cache") + return count + + return self._run_maintenance_write("Orphan search cleanup", _operation) def clean_stale_musicbrainz_nulls(self, max_age_days: int = 30) -> int: """Delete MusicBrainz cache entries where lookup found nothing (null MBID) and age > max_age_days.""" - try: - db = self._get_db() - conn = db._get_connection() - try: - cursor = conn.cursor() - cursor.execute(""" - DELETE FROM musicbrainz_cache - WHERE musicbrainz_id IS NULL - AND julianday('now') - julianday(last_updated) > ? - """, (max_age_days,)) - count = cursor.rowcount - conn.commit() - if count > 0: - logger.info(f"Cleaned {count} stale MusicBrainz null entries (>{max_age_days} days)") - return count - finally: - conn.close() - except Exception as e: - logger.error(f"MusicBrainz null cleanup error: {e}") - return 0 + def _operation(conn): + cursor = conn.cursor() + cursor.execute(""" + DELETE FROM musicbrainz_cache + WHERE musicbrainz_id IS NULL + AND julianday('now') - julianday(last_updated) > ? + """, (max_age_days,)) + count = cursor.rowcount + conn.commit() + if count > 0: + logger.info(f"Cleaned {count} stale MusicBrainz null entries (>{max_age_days} days)") + return count + + return self._run_maintenance_write("MusicBrainz null cleanup", _operation) def get_health_stats(self) -> dict: """Return cache health statistics for the repair dashboard. diff --git a/database/music_database.py b/database/music_database.py index c6980623..0171d88e 100644 --- a/database/music_database.py +++ b/database/music_database.py @@ -4591,45 +4591,72 @@ class MusicDatabase: # VACUUM to actually shrink the database file and reclaim disk space logger.info("Vacuuming database to reclaim disk space...") - cursor.execute("VACUUM") + self._vacuum_best_effort(cursor) logger.info("All database data cleared and file compacted") except Exception as e: logger.error(f"Error clearing database: {e}") raise + + def _vacuum_best_effort(self, cursor): + """Run VACUUM without making the caller fail if compaction hiccups.""" + try: + cursor.execute("VACUUM") + except Exception as e: + logger.warning( + "Database VACUUM failed after data was already cleared; continuing without compaction: %s", + e, + ) + + @staticmethod + def _is_transient_sqlite_io_error(exc: Exception) -> bool: + return "disk i/o error" in str(exc).lower() def clear_server_data(self, server_source: str): """Clear data for specific server only (server-aware full refresh)""" - try: - with self._get_connection() as conn: - cursor = conn.cursor() - - # Delete only data from the specified server - # Order matters: tracks -> albums -> artists (foreign key constraints) - cursor.execute("DELETE FROM tracks WHERE server_source = ?", (server_source,)) - tracks_deleted = cursor.rowcount - - cursor.execute("DELETE FROM albums WHERE server_source = ?", (server_source,)) - albums_deleted = cursor.rowcount - - cursor.execute("DELETE FROM artists WHERE server_source = ?", (server_source,)) - artists_deleted = cursor.rowcount - - conn.commit() - - # Only VACUUM if we deleted a significant amount of data - if tracks_deleted > 1000 or albums_deleted > 100: - logger.info("Vacuuming database to reclaim disk space...") - cursor.execute("VACUUM") - - logger.info(f"Cleared {server_source} data: {artists_deleted} artists, {albums_deleted} albums, {tracks_deleted} tracks") - - # Note: Watchlist and wishlist are preserved as they are server-agnostic - - except Exception as e: - logger.error(f"Error clearing {server_source} database data: {e}") - raise + for attempt in range(2): + try: + with self._get_connection() as conn: + cursor = conn.cursor() + + # Delete only data from the specified server + # Order matters: tracks -> albums -> artists (foreign key constraints) + cursor.execute("DELETE FROM tracks WHERE server_source = ?", (server_source,)) + tracks_deleted = cursor.rowcount + + cursor.execute("DELETE FROM albums WHERE server_source = ?", (server_source,)) + albums_deleted = cursor.rowcount + + cursor.execute("DELETE FROM artists WHERE server_source = ?", (server_source,)) + artists_deleted = cursor.rowcount + + conn.commit() + + # Only VACUUM if we deleted a significant amount of data + if tracks_deleted > 1000 or albums_deleted > 100: + logger.info("Vacuuming database to reclaim disk space...") + self._vacuum_best_effort(cursor) + + logger.info( + f"Cleared {server_source} data: {artists_deleted} artists, " + f"{albums_deleted} albums, {tracks_deleted} tracks" + ) + + # Note: Watchlist and wishlist are preserved as they are server-agnostic + return + + except Exception as e: + if self._is_transient_sqlite_io_error(e) and attempt == 0: + logger.warning( + "Transient disk I/O error clearing %s database data; retrying once: %s", + server_source, + e, + ) + time.sleep(0.25) + continue + logger.error(f"Error clearing {server_source} database data: {e}") + raise def cleanup_orphaned_records(self) -> Dict[str, int]: """Remove artists and albums that have no associated tracks""" diff --git a/tests/metadata/test_cache_maintenance_retry.py b/tests/metadata/test_cache_maintenance_retry.py new file mode 100644 index 00000000..dc5ee85c --- /dev/null +++ b/tests/metadata/test_cache_maintenance_retry.py @@ -0,0 +1,44 @@ +import sqlite3 +from types import SimpleNamespace + +import core.metadata.cache as cache_module +from core.metadata.cache import MetadataCache + + +def test_maintenance_write_retries_once_after_disk_io(monkeypatch): + cache = MetadataCache() + attempts = [] + + class _Conn: + def close(self): + pass + + monkeypatch.setattr(cache, "_get_db", lambda: SimpleNamespace(_get_connection=lambda: _Conn())) + monkeypatch.setattr(cache_module.time, "sleep", lambda _seconds: None) + + def _operation(_conn): + attempts.append(1) + if len(attempts) == 1: + raise sqlite3.OperationalError("disk I/O error") + return 9 + + assert cache._run_maintenance_write("Cache eviction", _operation) == 9 + assert len(attempts) == 2 + + +def test_maintenance_write_does_not_retry_non_io_errors(monkeypatch): + cache = MetadataCache() + attempts = [] + + class _Conn: + def close(self): + pass + + monkeypatch.setattr(cache, "_get_db", lambda: SimpleNamespace(_get_connection=lambda: _Conn())) + + def _operation(_conn): + attempts.append(1) + raise sqlite3.OperationalError("syntax error") + + assert cache._run_maintenance_write("Cache eviction", _operation) == 0 + assert len(attempts) == 1 diff --git a/tests/test_database_io_resilience.py b/tests/test_database_io_resilience.py new file mode 100644 index 00000000..37da1cc7 --- /dev/null +++ b/tests/test_database_io_resilience.py @@ -0,0 +1,98 @@ +import sqlite3 + +from database.music_database import MusicDatabase + + +def test_clear_server_data_does_not_fail_when_vacuum_hits_disk_io(): + db = object.__new__(MusicDatabase) + + class _Cursor: + rowcount = 0 + + def __init__(self): + self.calls = [] + + def execute(self, query, params=None): + self.calls.append((query, params)) + if query == "VACUUM": + raise sqlite3.OperationalError("disk I/O error") + if "tracks" in query: + self.rowcount = 1500 + elif "albums" in query: + self.rowcount = 200 + elif "artists" in query: + self.rowcount = 20 + + class _Conn: + def __init__(self): + self.cursor_obj = _Cursor() + self.commits = 0 + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc, tb): + return False + + def cursor(self): + return self.cursor_obj + + def commit(self): + self.commits += 1 + + conn = _Conn() + db._get_connection = lambda: conn + + db.clear_server_data("jellyfin") + + assert conn.commits == 1 + assert any(call[0] == "VACUUM" for call in conn.cursor_obj.calls) + + +def test_clear_server_data_retries_transient_disk_io_before_commit(monkeypatch): + db = object.__new__(MusicDatabase) + connections = [] + + class _Cursor: + rowcount = 0 + + def __init__(self, fail_first_delete=False): + self.fail_first_delete = fail_first_delete + self.calls = [] + + def execute(self, query, params=None): + self.calls.append((query, params)) + if self.fail_first_delete and "DELETE FROM tracks" in query: + self.fail_first_delete = False + raise sqlite3.OperationalError("disk I/O error") + self.rowcount = 1 + + class _Conn: + def __init__(self, fail_first_delete=False): + self.cursor_obj = _Cursor(fail_first_delete=fail_first_delete) + self.commits = 0 + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc, tb): + return False + + def cursor(self): + return self.cursor_obj + + def commit(self): + self.commits += 1 + + def _connect(): + conn = _Conn(fail_first_delete=not connections) + connections.append(conn) + return conn + + db._get_connection = _connect + monkeypatch.setattr("database.music_database.time.sleep", lambda _seconds: None) + + db.clear_server_data("jellyfin") + + assert len(connections) == 2 + assert connections[1].commits == 1 From fae13226e5a530646b918f3ce5c543c8abdd9de4 Mon Sep 17 00:00:00 2001 From: Broque Thomas <26755000+Nezreka@users.noreply.github.com> Date: Thu, 21 May 2026 18:04:10 -0700 Subject: [PATCH 22/25] Check HiFi download capability via manifests Probe public HiFi instances with the same trackManifests endpoint used by real downloads instead of the legacy /track endpoint. This prevents compatible instances from being falsely labeled search-only in Settings. Centralize HiFi instance capability checks in HiFiClient and reuse manifest URI parsing with the download path. Tests cover manifest-based capability detection, no legacy /track probe, and limited instances without a manifest URI. --- core/hifi_client.py | 78 +++++++++++++++++++++++++--- tests/downloads/test_hifi_pinning.py | 75 ++++++++++++++++++++++++++ web_server.py | 29 +---------- 3 files changed, 148 insertions(+), 34 deletions(-) diff --git a/core/hifi_client.py b/core/hifi_client.py index 8fa51cdd..82163f19 100644 --- a/core/hifi_client.py +++ b/core/hifi_client.py @@ -244,6 +244,75 @@ class HiFiClient(DownloadSourcePlugin): return data.get('version') or data.get('data', {}).get('version') return None + @staticmethod + def _extract_manifest_uri(data: Any) -> Optional[str]: + try: + inner = data.get('data', data) if isinstance(data, dict) else data + attrs = inner.get('data', {}).get('attributes', {}) + return attrs.get('uri') + except (AttributeError, KeyError): + return None + + def check_instance_capabilities(self, url: str, timeout: int = 5) -> Dict[str, Any]: + """Probe one public HiFi instance using the endpoints SoulSync needs.""" + entry = { + 'url': url, + 'status': 'unknown', + 'version': None, + 'can_search': False, + 'can_download': False, + } + try: + root = self.session.get( + f'{url}/', + timeout=timeout, + headers={'Accept': 'application/json'}, + ) + if not root.ok: + entry['status'] = f'error (HTTP {root.status_code})' + return entry + + data = root.json() + entry['version'] = data.get('version') or data.get('data', {}).get('version') + entry['status'] = 'online' + + search = self.session.get( + f'{url}/search/', + params={'s': 'test', 'limit': 1}, + timeout=timeout, + ) + entry['can_search'] = search.ok + + manifest = self.session.get( + f'{url}/trackManifests/', + params={ + 'id': '1550546', + 'formats': 'FLAC', + 'usage': 'DOWNLOAD', + 'manifestType': 'HLS', + 'adaptive': 'true', + 'uriScheme': 'HTTPS', + }, + timeout=timeout, + ) + entry['can_download'] = ( + manifest.ok + and bool(self._extract_manifest_uri(manifest.json())) + ) + if not manifest.ok: + entry['download_error'] = f'HTTP {manifest.status_code}' + elif not entry['can_download']: + entry['download_error'] = 'No HLS manifest URI' + except http_requests.exceptions.SSLError: + entry['status'] = 'ssl_error' + except http_requests.exceptions.ConnectTimeout: + entry['status'] = 'timeout' + except http_requests.exceptions.ConnectionError: + entry['status'] = 'offline' + except Exception as e: + entry['status'] = f'error ({type(e).__name__})' + return entry + def search_tracks(self, title: str = None, artist: str = None, album: str = None, limit: int = 20) -> List[Dict]: params = {'limit': limit} @@ -435,12 +504,9 @@ class HiFiClient(DownloadSourcePlugin): if not data: return None - try: - inner = data.get('data', data) if isinstance(data, dict) else data - attrs = inner.get('data', {}).get('attributes', {}) - uri = attrs.get('uri') - except (AttributeError, KeyError) as e: - logger.warning(f"Failed to extract playlist URI from manifest response: {e}") + uri = self._extract_manifest_uri(data) + if uri is None: + logger.warning("Failed to extract playlist URI from manifest response") return None if not uri: diff --git a/tests/downloads/test_hifi_pinning.py b/tests/downloads/test_hifi_pinning.py index 24ca15f1..de04b62d 100644 --- a/tests/downloads/test_hifi_pinning.py +++ b/tests/downloads/test_hifi_pinning.py @@ -102,3 +102,78 @@ def test_cancel_download_marks_cancelled(hifi_client_with_engine): ok = _run_async(client.cancel_download('dl-1', None, remove=False)) assert ok is True assert engine.get_record('hifi', 'dl-1')['state'] == 'Cancelled' + + +def test_instance_capability_probe_uses_track_manifests_not_legacy_track(): + class _Response: + def __init__(self, *, ok=True, status_code=200, payload=None): + self.ok = ok + self.status_code = status_code + self._payload = payload or {} + + def json(self): + return self._payload + + class _Session: + def __init__(self): + self.calls = [] + + def get(self, url, **kwargs): + self.calls.append((url, kwargs)) + if url.endswith('/search/'): + return _Response(payload={'items': []}) + if url.endswith('/trackManifests/'): + return _Response(payload={ + 'data': { + 'data': { + 'attributes': { + 'uri': 'https://cdn.example/playlist.m3u8', + }, + }, + }, + }) + if url.endswith('/'): + return _Response(payload={'version': 'test'}) + return _Response(ok=False, status_code=404) + + client = HiFiClient.__new__(HiFiClient) + client.session = _Session() + + result = client.check_instance_capabilities('https://hifi.example') + + called_urls = [url for url, _ in client.session.calls] + assert result['can_search'] is True + assert result['can_download'] is True + assert any(url.endswith('/trackManifests/') for url in called_urls) + assert not any(url.endswith('/track') for url in called_urls) + + +def test_instance_capability_probe_reports_manifest_without_uri_as_limited(): + class _Response: + ok = True + status_code = 200 + + def __init__(self, payload): + self._payload = payload + + def json(self): + return self._payload + + class _Session: + def get(self, url, **kwargs): + if url.endswith('/search/'): + return _Response({'items': []}) + if url.endswith('/trackManifests/'): + return _Response({'data': {'data': {'attributes': {}}}}) + if url.endswith('/'): + return _Response({'version': 'test'}) + return _Response({'data': {'data': {'attributes': {}}}}) + + client = HiFiClient.__new__(HiFiClient) + client.session = _Session() + + result = client.check_instance_capabilities('https://hifi.example') + + assert result['can_search'] is True + assert result['can_download'] is False + assert result['download_error'] == 'No HLS manifest URI' diff --git a/web_server.py b/web_server.py index d6269314..e9319471 100644 --- a/web_server.py +++ b/web_server.py @@ -19779,39 +19779,12 @@ def soundcloud_status(): @app.route('/api/hifi/instances', methods=['GET']) def hifi_instances(): """Check availability of all HiFi API instances.""" - import requests as req try: hifi = download_orchestrator.client("hifi") instances = list(hifi._instances) results = [] for url in instances: - entry = {'url': url, 'status': 'unknown', 'version': None, 'can_search': False, 'can_download': False} - try: - # Check root for version - r = req.get(f'{url}/', timeout=5, headers={'Accept': 'application/json'}) - if r.ok: - data = r.json() - entry['version'] = data.get('version') - entry['status'] = 'online' - # Check search - sr = req.get(f'{url}/search', params={'s': 'test', 'limit': 1}, timeout=5) - entry['can_search'] = sr.ok - # Check track (download capability) - tr = req.get(f'{url}/track', params={'id': '1550546', 'quality': 'LOSSLESS'}, timeout=5) - entry['can_download'] = tr.ok - if not tr.ok: - entry['download_error'] = f'HTTP {tr.status_code}' - else: - entry['status'] = f'error (HTTP {r.status_code})' - except req.exceptions.SSLError: - entry['status'] = 'ssl_error' - except req.exceptions.ConnectTimeout: - entry['status'] = 'timeout' - except req.exceptions.ConnectionError: - entry['status'] = 'offline' - except Exception as e: - entry['status'] = f'error ({type(e).__name__})' - results.append(entry) + results.append(hifi.check_instance_capabilities(url)) return jsonify({'instances': results, 'active': hifi._get_instance()}) except Exception as e: return jsonify({'error': str(e)}), 500 From 763888e6713b5a2044b57c98bedb31e3bf3e80eb Mon Sep 17 00:00:00 2001 From: Broque Thomas <26755000+Nezreka@users.noreply.github.com> Date: Thu, 21 May 2026 18:11:14 -0700 Subject: [PATCH 23/25] Support legacy HiFi track manifests Add fallback support for public hifi-api instances that expose playback through /track/ instead of /trackManifests/. The capability checker now accepts either manifest shape, and downloads can use direct URLs decoded from the legacy base64 manifest. Tests cover legacy instance capability detection and download-manifest fallback while preserving the newer trackManifests path. --- core/hifi_client.py | 99 ++++++++++++++++++++++++++-- tests/downloads/test_hifi_pinning.py | 77 +++++++++++++++++++++- 2 files changed, 168 insertions(+), 8 deletions(-) diff --git a/core/hifi_client.py b/core/hifi_client.py index 82163f19..b6c854ee 100644 --- a/core/hifi_client.py +++ b/core/hifi_client.py @@ -23,6 +23,8 @@ import re import uuid import time import shutil +import json +import base64 import subprocess import threading from typing import List, Optional, Dict, Any, Tuple @@ -75,6 +77,13 @@ HLS_QUALITY_MAP = { HLS_MAP_TAG_RE = re.compile(r'#EXT-X-MAP:.*URI="([^"]+)"') +TRACK_ENDPOINT_QUALITY_MAP = { + 'hires': 'HI_RES_LOSSLESS', + 'lossless': 'LOSSLESS', + 'high': 'HIGH', + 'low': 'LOW', +} + # Default public hifi-api instances (ordered by preference) DEFAULT_INSTANCES = [ 'https://triton.squid.wtf', @@ -253,6 +262,37 @@ class HiFiClient(DownloadSourcePlugin): except (AttributeError, KeyError): return None + @staticmethod + def _extract_track_manifest_urls(data: Any) -> List[str]: + try: + inner = data.get('data', data) if isinstance(data, dict) else data + manifest_b64 = inner.get('manifest') + if not manifest_b64: + return [] + manifest = json.loads(base64.b64decode(manifest_b64)) + if manifest.get('encryptionType') not in (None, 'NONE'): + return [] + urls = manifest.get('urls') or [] + return [url for url in urls if isinstance(url, str) and url] + except Exception as e: + logger.debug("Failed to extract legacy HiFi track manifest URLs: %s", e) + return [] + + @staticmethod + def _extension_from_track_manifest(data: Any, fallback: str) -> str: + try: + inner = data.get('data', data) if isinstance(data, dict) else data + manifest = json.loads(base64.b64decode(inner.get('manifest') or '')) + mime = (manifest.get('mimeType') or '').lower() + codecs = (manifest.get('codecs') or '').lower() + if 'flac' in mime or 'flac' in codecs: + return 'flac' + if 'mp4' in mime or 'aac' in codecs: + return 'm4a' + except Exception: + pass + return fallback + def check_instance_capabilities(self, url: str, timeout: int = 5) -> Dict[str, Any]: """Probe one public HiFi instance using the endpoints SoulSync needs.""" entry = { @@ -302,7 +342,23 @@ class HiFiClient(DownloadSourcePlugin): if not manifest.ok: entry['download_error'] = f'HTTP {manifest.status_code}' elif not entry['can_download']: - entry['download_error'] = 'No HLS manifest URI' + legacy = self.session.get( + f'{url}/track/', + params={'id': '1550546', 'quality': 'LOSSLESS'}, + timeout=timeout, + ) + entry['can_download'] = ( + legacy.ok + and bool(self._extract_track_manifest_urls(legacy.json())) + ) + if not legacy.ok: + entry['download_error'] = f'HTTP {legacy.status_code}' + elif not entry['can_download']: + entry['download_error'] = 'No playable manifest URL' + else: + entry['download_probe'] = 'track' + else: + entry['download_probe'] = 'trackManifests' except http_requests.exceptions.SSLError: entry['status'] = 'ssl_error' except http_requests.exceptions.ConnectTimeout: @@ -502,12 +558,12 @@ class HiFiClient(DownloadSourcePlugin): data = self._api_get('/trackManifests/', params=params, timeout=20) if not data: - return None + return self._get_legacy_track_manifest(track_id, quality) uri = self._extract_manifest_uri(data) if uri is None: logger.warning("Failed to extract playlist URI from manifest response") - return None + return self._get_legacy_track_manifest(track_id, quality) if not uri: logger.warning(f"No playlist URI in manifest for track {track_id}") @@ -554,6 +610,28 @@ class HiFiClient(DownloadSourcePlugin): 'quality': quality, } + def _get_legacy_track_manifest(self, track_id: int, quality: str = 'lossless') -> Optional[Dict]: + q_info = HLS_QUALITY_MAP.get(quality, HLS_QUALITY_MAP['lossless']) + api_quality = TRACK_ENDPOINT_QUALITY_MAP.get(quality, 'LOSSLESS') + data = self._api_get('/track/', params={'id': track_id, 'quality': api_quality}, timeout=20) + if not data: + return None + + direct_urls = self._extract_track_manifest_urls(data) + if not direct_urls: + logger.warning(f"No playable URL in legacy HiFi manifest for track {track_id}") + return None + + extension = self._extension_from_track_manifest(data, q_info['extension']) + logger.info(f"HiFi legacy track manifest for track {track_id}: " + f"{len(direct_urls)} direct URL(s) ({quality})") + return { + 'direct_urls': direct_urls, + 'extension': extension, + 'codec': q_info['codec'], + 'quality': quality, + } + def _demux_flac(self, input_path: Path, output_path: Path) -> None: ffmpeg = shutil.which('ffmpeg') if not ffmpeg: @@ -685,7 +763,13 @@ class HiFiClient(DownloadSourcePlugin): return None manifest_info = self._get_hls_manifest(track_id, quality=q_key) - if not manifest_info or not manifest_info.get('segment_uris'): + if ( + not manifest_info + or ( + not manifest_info.get('segment_uris') + and not manifest_info.get('direct_urls') + ) + ): logger.warning(f"No HLS manifest at quality {q_key}, trying next") continue @@ -694,16 +778,17 @@ class HiFiClient(DownloadSourcePlugin): out_filename = f"{safe_name}.{extension}" out_path = self.download_path / out_filename - is_flac = q_key in ('hires', 'lossless') + is_direct = bool(manifest_info.get('direct_urls')) + is_flac = q_key in ('hires', 'lossless') and not is_direct intermediate_path = out_path.with_suffix('.m4a') if is_flac else out_path try: init_uri = manifest_info.get('init_uri') - segment_uris = manifest_info['segment_uris'] + segment_uris = manifest_info.get('segment_uris') or manifest_info.get('direct_urls') or [] total_segments = len(segment_uris) + (1 if init_uri else 0) logger.info(f"Downloading from HiFi ({q_key}): {out_filename} " - f"({total_segments} segments)") + f"({total_segments} {'URL(s)' if is_direct else 'segments'})") downloaded = 0 speed_start = time.time() diff --git a/tests/downloads/test_hifi_pinning.py b/tests/downloads/test_hifi_pinning.py index de04b62d..d5fc9a8b 100644 --- a/tests/downloads/test_hifi_pinning.py +++ b/tests/downloads/test_hifi_pinning.py @@ -176,4 +176,79 @@ def test_instance_capability_probe_reports_manifest_without_uri_as_limited(): assert result['can_search'] is True assert result['can_download'] is False - assert result['download_error'] == 'No HLS manifest URI' + assert result['download_error'] == 'No playable manifest URL' + + +def test_instance_capability_probe_accepts_legacy_track_manifest(): + import base64 + import json + + class _Response: + def __init__(self, payload, *, ok=True, status_code=200): + self._payload = payload + self.ok = ok + self.status_code = status_code + + def json(self): + return self._payload + + class _Session: + def __init__(self): + self.calls = [] + + def get(self, url, **kwargs): + self.calls.append((url, kwargs)) + if url.endswith('/search/'): + return _Response({'items': []}) + if url.endswith('/trackManifests/'): + return _Response({'data': {'data': {'attributes': {}}}}) + if url.endswith('/track/'): + manifest = base64.b64encode(json.dumps({ + 'mimeType': 'audio/flac', + 'codecs': 'flac', + 'encryptionType': 'NONE', + 'urls': ['https://cdn.example/track.flac'], + }).encode()).decode() + return _Response({'data': {'manifest': manifest}}) + if url.endswith('/'): + return _Response({'version': 'test'}) + return _Response({}, ok=False, status_code=404) + + client = HiFiClient.__new__(HiFiClient) + client.session = _Session() + + result = client.check_instance_capabilities('https://hifi.example') + + assert result['can_search'] is True + assert result['can_download'] is True + assert result['download_probe'] == 'track' + assert any(url.endswith('/track/') for url, _ in client.session.calls) + + +def test_get_hls_manifest_falls_back_to_legacy_track_endpoint(): + import base64 + import json + + client = HiFiClient.__new__(HiFiClient) + calls = [] + + def _fake_api_get(path, params=None, timeout=15): + calls.append((path, params)) + if path == '/trackManifests/': + return None + manifest = base64.b64encode(json.dumps({ + 'mimeType': 'audio/flac', + 'codecs': 'flac', + 'encryptionType': 'NONE', + 'urls': ['https://cdn.example/track.flac'], + }).encode()).decode() + return {'data': {'manifest': manifest}} + + client._api_get = _fake_api_get + + result = client._get_hls_manifest(123, quality='lossless') + + assert result['direct_urls'] == ['https://cdn.example/track.flac'] + assert result['extension'] == 'flac' + assert calls[0][0] == '/trackManifests/' + assert calls[1][0] == '/track/' From 274a1ed34a6d6d4c5934aecc4d88bdd6d12c6ddd Mon Sep 17 00:00:00 2001 From: Broque Thomas <26755000+Nezreka@users.noreply.github.com> Date: Thu, 21 May 2026 18:21:22 -0700 Subject: [PATCH 24/25] Bump release to 2.5.9 and add changelog Update project version and release notes for 2.5.9. Changes: update .github/workflows/docker-publish.yml default/version_tag prompt to 2.5.9, bump _SOULSYNC_BASE_VERSION in web_server.py to 2.5.9, and replace the WHATS_NEW entry in webui/static/helper.js with detailed 2.5.9 release notes and a new VERSION_MODAL_SECTIONS entry. Also update the helper.js fallback for latest whats-new version to 2.5.9. --- .github/workflows/docker-publish.yml | 4 ++-- web_server.py | 2 +- webui/static/helper.js | 23 ++++++++++++++++++++--- 3 files changed, 23 insertions(+), 6 deletions(-) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index f049a16e..ae96209f 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -9,9 +9,9 @@ on: workflow_dispatch: inputs: version_tag: - description: 'Version tag (e.g. 2.5.8)' + description: 'Version tag (e.g. 2.5.9)' required: true - default: '2.5.8' + default: '2.5.9' jobs: build-and-push: diff --git a/web_server.py b/web_server.py index e9319471..3966f93c 100644 --- a/web_server.py +++ b/web_server.py @@ -40,7 +40,7 @@ logger = setup_logging(_log_level, _log_path) # App version — single source of truth for backup metadata, system-info, update check, etc. # Semver: MAJOR.MINOR.PATCH. Bump at each dev→main release. -_SOULSYNC_BASE_VERSION = "2.5.8" +_SOULSYNC_BASE_VERSION = "2.5.9" def _build_version_string(): """Append short commit hash to version when available (e.g. 2.35+abc1234).""" diff --git a/webui/static/helper.js b/webui/static/helper.js index 0d57a8b2..2935a313 100644 --- a/webui/static/helper.js +++ b/webui/static/helper.js @@ -3413,8 +3413,13 @@ function closeHelperSearch() { // projects that span multiple commits before shipping. Strip the flag at // release time and add a real `date:` line at the top of the version block. const WHATS_NEW = { - '2.6.0': [ - { unreleased: true }, + '2.5.9': [ + { date: 'May 21, 2026 — 2.5.9 release' }, + { title: 'Torrent and Usenet release downloads', desc: 'torrent and usenet are now real download sources backed by Prowlarr plus your configured downloader client. album downloads use a release-first staging flow so SoulSync downloads one release, watches live progress, then imports the matching tracks from the staged files. hybrid mode keeps torrent / usenet out of album batches, but still lets them participate for playlist singles and wishlist tracks.' }, + { title: 'Fix: HiFi public instance compatibility', desc: 'HiFi instance probing now understands both supported manifest shapes: the newer /trackManifests-style flow and the public hifi-api /track/ legacy flow. instances that can search and return a playable HLS manifest are no longer mislabeled as Search only. browser-openable pages can still be offline from the API side, so HTTP 502 / Offline labels are still shown honestly.' }, + { title: 'Fix: Jellyfin full refresh imports tracks on older databases', desc: 'full refresh could import artists and albums but fail every Jellyfin track on upgraded databases that were missing newer media columns. startup repair now adds the missing tracks.file_size and albums.api_track_count columns before refresh work runs, so old databases can accept new Jellyfin track rows again.' }, + { title: 'Fix: transient SQLite disk I/O errors retry cleanly', desc: 'cache maintenance and the Tools page full-refresh job now retry short-lived SQLite disk I/O failures around destructive maintenance writes. this targets the reported pattern where the first run failed with disk I/O error and an immediate second run succeeded.' }, + { title: 'Fix: Album Completeness blocks wrong-artist fills', desc: 'artist matching now wins before album/single title matching. if Album Completeness is filling a Jamiroquai track, a same-title release by a different artist is rejected instead of being allowed through by a loose title match.' }, { title: 'Fix: Album Completeness no longer cross-contaminates artists', desc: 'reported case: filling a Jamiroquai "Light Years" single brought in tracks from Gut\'s "Light Years" album — completely different artist, completely different genre. Root cause: the auto-fill artist gate was a loose 0.50 SequenceMatcher threshold that could let unrelated candidates through. Two new defenses now block this entirely. Stage 1 skips the whole track-fill operation up front if the missing track\'s source artist doesn\'t match the target album artist — compilation albums (various artists / soundtrack) still pass through. Stage 2 replaces the per-candidate 0.50 SequenceMatcher with an alias-aware 0.82 strict matcher that handles diacritics (Beyoncé/Beyonce) and known artist aliases. Both stages logged with structured warnings so future mismatches are diagnosable from the logs.' }, { title: 'Code review refactor pass on the torrent / usenet flow', desc: 'cleanup commits before review. Lifted the shared album-pick + staging-collision helpers out of torrent.py into a new core/download_plugins/album_bundle.py module so usenet.py no longer reaches into a sibling plugin\'s private surface. Lifted the ~90-line inline album-bundle gate out of run_full_missing_tracks_process into core/downloads/album_bundle_dispatch.py with a clean pure-predicate / inject-deps design so the gate is unit-testable in isolation. Replaced the staging matcher\'s direct download_batches import with an injected get_batch_field accessor on StagingDeps. Made the 6h poll timeout configurable via download_source.album_bundle_timeout_seconds. Added atomic .tmp + rename copy so the Auto-Import worker can never observe a partial audio file during the album-bundle copy loop. 49 new tests across album_bundle, album_bundle_dispatch, and staging-provenance modules pin the contracts.' }, { title: 'Hybrid mode skips torrent / usenet on album batches', desc: 'follow-up to the album-bundle flow. When download mode is set to Hybrid AND the batch is an album, the per-track search loop now silently strips torrent and usenet from the source chain — they\'re release-level sources that can\'t match per-track meaningfully, and the album-bundle handling only fires in single-source mode. Without the skip, hybrid + torrent-first would fire N redundant Prowlarr searches per album and rely on Auto-Import sweeping Staging to recover. Cleaner now: hybrid falls straight through to per-track-compatible sources (Soulseek / streaming) for albums, and torrent / usenet still get a shot for single-track / wishlist / basic-search use cases where per-track makes sense.' }, @@ -3486,6 +3491,18 @@ const WHATS_NEW = { // Section shape: { title, description, features: [bullet strings], // usage_note?: 'optional hint shown at the bottom' } const VERSION_MODAL_SECTIONS = [ + { + title: "2.5.9 Release Stability Pass", + description: "this release ties together the new release-based download sources and a set of fixes from real user reports: HiFi instance detection, Jellyfin full refreshes, transient SQLite disk I/O failures, and wrong-artist Album Completeness fills.", + features: [ + "Torrent and Usenet are now available as Prowlarr-backed download sources with release staging, live progress, and source-aware history labels", + "HiFi instances are probed by actually checking whether they can return a playable manifest, including legacy public hifi-api instances", + "Jellyfin full refresh repairs older media databases before importing tracks, so stale schemas no longer drop every track row", + "Cache maintenance and full refresh retry transient SQLite disk I/O errors instead of failing the whole job on the first attempt", + "Album Completeness rejects same-title releases from the wrong artist before importing anything", + ], + usage_note: "version 2.5.9 focuses on safer matching and making the new release-based sources usable without disturbing existing source flows", + }, { title: "Torrent and Usenet Are Now Live Download Sources", description: "the long-awaited payoff. Two new entries in the Download Source dropdown — Torrent Only and Usenet Only — both backed by your Prowlarr indexers. Searches go through Prowlarr filtered by protocol, picked releases get shipped to your configured torrent client (qBit / Transmission / Deluge) or usenet client (SABnzbd / NZBGet), and the resulting files flow through SoulSync's matching pipeline like any other source.", @@ -3870,7 +3887,7 @@ function _getLatestWhatsNewVersion() { const versions = Object.keys(WHATS_NEW) .filter(v => _compareVersions(v, buildVer) <= 0) .sort((a, b) => _compareVersions(b, a)); - return versions[0] || '2.5.8'; + return versions[0] || '2.5.9'; } function openWhatsNew() { From 048e4e85d5ceb6bbb9f6a6d2b2efbc259cc331ff Mon Sep 17 00:00:00 2001 From: Broque Thomas <26755000+Nezreka@users.noreply.github.com> Date: Thu, 21 May 2026 18:24:56 -0700 Subject: [PATCH 25/25] Log exception when inferring HiFi manifest ext Add a debug log in the exception handler that occurs while inferring legacy HiFi track manifest extensions. Previously exceptions were silently ignored; this change records the error message via logger.debug to aid debugging without changing behavior. --- core/hifi_client.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/hifi_client.py b/core/hifi_client.py index b6c854ee..242983ea 100644 --- a/core/hifi_client.py +++ b/core/hifi_client.py @@ -289,8 +289,8 @@ class HiFiClient(DownloadSourcePlugin): return 'flac' if 'mp4' in mime or 'aac' in codecs: return 'm4a' - except Exception: - pass + except Exception as e: + logger.debug("Failed to infer legacy HiFi track manifest extension: %s", e) return fallback def check_instance_capabilities(self, url: str, timeout: int = 5) -> Dict[str, Any]: