commit
dcd4d426c5
58 changed files with 6488 additions and 325 deletions
4
.github/workflows/docker-publish.yml
vendored
4
.github/workflows/docker-publish.yml
vendored
|
|
@ -9,9 +9,9 @@ on:
|
|||
workflow_dispatch:
|
||||
inputs:
|
||||
version_tag:
|
||||
description: 'Version tag (e.g. 2.5.8)'
|
||||
description: 'Version tag (e.g. 2.5.9)'
|
||||
required: true
|
||||
default: '2.5.8'
|
||||
default: '2.5.9'
|
||||
|
||||
jobs:
|
||||
build-and-push:
|
||||
|
|
|
|||
|
|
@ -480,11 +480,19 @@ class ConfigManager:
|
|||
"search_min_delay_seconds": 0,
|
||||
},
|
||||
"download_source": {
|
||||
"mode": "soulseek", # Options: "soulseek", "youtube", "tidal", "qobuz", "hifi", "hybrid"
|
||||
"mode": "soulseek", # Options: "soulseek", "youtube", "tidal", "qobuz", "hifi", "hybrid", "torrent", "usenet"
|
||||
"hybrid_primary": "soulseek", # Legacy: primary source for hybrid mode
|
||||
"hybrid_secondary": "youtube", # Legacy: fallback source for hybrid mode
|
||||
"hybrid_order": [], # Ordered list of sources for hybrid mode (overrides primary/secondary)
|
||||
"stream_source": "youtube", # Options: "youtube" (instant, default), "active" (use download source; falls back to youtube if soulseek)
|
||||
# Album-bundle (torrent / usenet single-source) poll tuning.
|
||||
# Downloader is polled every N seconds until the release
|
||||
# lands; whole job aborts at the timeout. Defaults match
|
||||
# the previous hard-coded constants. Users on slow private
|
||||
# trackers / large box sets can extend the timeout without
|
||||
# editing source.
|
||||
"album_bundle_poll_interval_seconds": 2.0,
|
||||
"album_bundle_timeout_seconds": 6 * 60 * 60, # 6 hours
|
||||
},
|
||||
"tidal_download": {
|
||||
"quality": "lossless", # Options: "low", "high", "lossless", "hires"
|
||||
|
|
|
|||
238
core/archive_pipeline.py
Normal file
238
core/archive_pipeline.py
Normal file
|
|
@ -0,0 +1,238 @@
|
|||
"""Archive extraction + audio-file discovery for torrent / usenet downloads.
|
||||
|
||||
The torrent and usenet download plugins need a uniform way to:
|
||||
|
||||
1. Walk the downloader's save directory and find every audio file in it.
|
||||
2. If the directory contains an archive (``.zip`` / ``.rar`` / ``.tar`` /
|
||||
``.7z``), extract it first so the audio files inside become walkable.
|
||||
|
||||
This module is intentionally narrow — no matching, no tagging, no
|
||||
import. The download plugin layer composes this with the existing
|
||||
post-processing / matching pipeline. Lidarr does NOT use this module:
|
||||
Lidarr extracts archives in its own import step before SoulSync sees
|
||||
the files at all. Usenet downloaders (SABnzbd, NZBGet) also auto-
|
||||
extract by default. Torrents are the main case where SoulSync may
|
||||
need to do the extract step itself — most music torrents ship loose,
|
||||
but some bundle the album in a ``.rar`` archive.
|
||||
|
||||
``rarfile`` is an optional dependency. If it isn't installed, archives
|
||||
with ``.rar`` content are skipped with a single warning rather than
|
||||
crashing the download.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import tarfile
|
||||
import zipfile
|
||||
from pathlib import Path
|
||||
from typing import List, Optional
|
||||
|
||||
from utils.logging_config import get_logger
|
||||
|
||||
logger = get_logger("archive_pipeline")
|
||||
|
||||
|
||||
# Same audio-extension set as ``core/imports/file_ops.py`` ``quality_tiers``.
|
||||
# Keep them in sync — if a new format is added to file_ops, add it here too
|
||||
# or the walker will skip it and the download plugin will mark the download
|
||||
# failed even when files arrived.
|
||||
AUDIO_EXTENSIONS = frozenset([
|
||||
# lossless
|
||||
'.flac', '.ape', '.wav', '.alac', '.dsf', '.dff', '.aiff', '.aif',
|
||||
# high lossy
|
||||
'.opus', '.ogg',
|
||||
# standard lossy
|
||||
'.m4a', '.aac',
|
||||
# low lossy
|
||||
'.mp3', '.wma',
|
||||
])
|
||||
|
||||
ARCHIVE_EXTENSIONS = frozenset(['.zip', '.rar', '.tar', '.tar.gz', '.tgz', '.7z'])
|
||||
|
||||
|
||||
def is_archive(path: Path) -> bool:
|
||||
"""True if the file extension looks like a supported archive.
|
||||
|
||||
Compound extensions (``.tar.gz``, ``.tar.bz2``) are detected by
|
||||
checking the last two suffixes joined together — Path.suffix
|
||||
only returns the final suffix.
|
||||
"""
|
||||
if not path.is_file():
|
||||
return False
|
||||
name = path.name.lower()
|
||||
if name.endswith(('.tar.gz', '.tar.bz2', '.tar.xz')):
|
||||
return True
|
||||
return path.suffix.lower() in ARCHIVE_EXTENSIONS
|
||||
|
||||
|
||||
def walk_audio_files(directory: Path) -> List[Path]:
|
||||
"""Recursively scan ``directory`` for audio files. Returns
|
||||
a sorted list of absolute paths. Empty list if the directory
|
||||
doesn't exist or contains no audio.
|
||||
"""
|
||||
if not directory or not directory.exists() or not directory.is_dir():
|
||||
return []
|
||||
out: List[Path] = []
|
||||
for child in directory.rglob('*'):
|
||||
if not child.is_file():
|
||||
continue
|
||||
if child.suffix.lower() in AUDIO_EXTENSIONS:
|
||||
out.append(child.resolve())
|
||||
out.sort()
|
||||
return out
|
||||
|
||||
|
||||
def find_archives_in_dir(directory: Path) -> List[Path]:
|
||||
"""Find every archive file directly inside ``directory`` (one
|
||||
level deep — torrents normally put the archive at the root of
|
||||
their folder; we don't search nested dirs to avoid extracting
|
||||
something we shouldn't).
|
||||
"""
|
||||
if not directory or not directory.exists() or not directory.is_dir():
|
||||
return []
|
||||
return sorted(p for p in directory.iterdir() if is_archive(p))
|
||||
|
||||
|
||||
def extract_archive(archive_path: Path, extract_to: Optional[Path] = None) -> Optional[Path]:
|
||||
"""Extract a single archive in-place (or to ``extract_to`` if
|
||||
given). Returns the directory the archive was extracted into,
|
||||
or ``None`` on failure.
|
||||
|
||||
Supports ``.zip``, ``.tar``/``.tar.gz``/``.tar.bz2``/``.tar.xz``,
|
||||
and ``.rar`` (only when the optional ``rarfile`` library is
|
||||
installed). ``.7z`` is recognised but extraction requires
|
||||
``py7zr``; without it, the call logs and returns None.
|
||||
"""
|
||||
if not archive_path or not archive_path.exists():
|
||||
logger.warning("archive_pipeline: %s does not exist", archive_path)
|
||||
return None
|
||||
dest = extract_to or archive_path.parent
|
||||
dest.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
name = archive_path.name.lower()
|
||||
try:
|
||||
if name.endswith('.zip'):
|
||||
with zipfile.ZipFile(archive_path) as zf:
|
||||
_safe_extract_zip(zf, dest)
|
||||
return dest
|
||||
if name.endswith(('.tar', '.tar.gz', '.tgz', '.tar.bz2', '.tar.xz')):
|
||||
with tarfile.open(archive_path) as tf:
|
||||
_safe_extract_tar(tf, dest)
|
||||
return dest
|
||||
if name.endswith('.rar'):
|
||||
return _extract_rar(archive_path, dest)
|
||||
if name.endswith('.7z'):
|
||||
return _extract_7z(archive_path, dest)
|
||||
except (zipfile.BadZipFile, tarfile.TarError, OSError) as e:
|
||||
logger.error("archive_pipeline: failed to extract %s: %s", archive_path, e)
|
||||
return None
|
||||
logger.warning("archive_pipeline: unknown archive type for %s", archive_path)
|
||||
return None
|
||||
|
||||
|
||||
def extract_all_in_dir(directory: Path) -> List[Path]:
|
||||
"""Find every archive in ``directory`` and extract each in place.
|
||||
Returns the list of directories archives were extracted into
|
||||
(usually all the same — ``directory`` itself). Archives that
|
||||
failed to extract are skipped silently after a warning.
|
||||
"""
|
||||
out: List[Path] = []
|
||||
for archive in find_archives_in_dir(directory):
|
||||
result = extract_archive(archive)
|
||||
if result is not None:
|
||||
out.append(result)
|
||||
return out
|
||||
|
||||
|
||||
def collect_audio_after_extraction(directory: Path) -> List[Path]:
|
||||
"""One-shot helper for the download plugins: extract any archives
|
||||
in the directory, then return the walked audio file list. This is
|
||||
the common pattern — torrent / usenet plugin gets a save_path,
|
||||
calls this, hands the resulting files to the matching pipeline.
|
||||
"""
|
||||
extract_all_in_dir(directory)
|
||||
return walk_audio_files(directory)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Safety helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _safe_extract_zip(zf: zipfile.ZipFile, dest: Path) -> None:
|
||||
"""Extract a zipfile after rejecting any member whose resolved
|
||||
path escapes ``dest`` (path traversal protection).
|
||||
"""
|
||||
dest = dest.resolve()
|
||||
for member in zf.namelist():
|
||||
target = (dest / member).resolve()
|
||||
if dest not in target.parents and target != dest:
|
||||
logger.error("archive_pipeline: refusing path-traversal member %r", member)
|
||||
return
|
||||
zf.extractall(dest)
|
||||
|
||||
|
||||
def _safe_extract_tar(tf: tarfile.TarFile, dest: Path) -> None:
|
||||
"""Same path-traversal protection for tarfiles."""
|
||||
dest = dest.resolve()
|
||||
for member in tf.getmembers():
|
||||
target = (dest / member.name).resolve()
|
||||
if dest not in target.parents and target != dest:
|
||||
logger.error("archive_pipeline: refusing path-traversal member %r", member.name)
|
||||
return
|
||||
# ``filter='data'`` is the Python 3.12+ safe extractor; fall back
|
||||
# to the legacy call on older runtimes.
|
||||
try:
|
||||
tf.extractall(dest, filter='data') # type: ignore[call-arg]
|
||||
except TypeError:
|
||||
tf.extractall(dest)
|
||||
|
||||
|
||||
def _extract_rar(archive_path: Path, dest: Path) -> Optional[Path]:
|
||||
try:
|
||||
import rarfile # type: ignore[import-untyped]
|
||||
except ImportError:
|
||||
logger.warning(
|
||||
"archive_pipeline: cannot extract %s — rarfile library not installed. "
|
||||
"Install with: pip install rarfile (and ensure unrar is on PATH).",
|
||||
archive_path,
|
||||
)
|
||||
return None
|
||||
try:
|
||||
with rarfile.RarFile(archive_path) as rf:
|
||||
dest_resolved = dest.resolve()
|
||||
for name in rf.namelist():
|
||||
target = (dest_resolved / name).resolve()
|
||||
if dest_resolved not in target.parents and target != dest_resolved:
|
||||
logger.error("archive_pipeline: refusing path-traversal rar member %r", name)
|
||||
return None
|
||||
rf.extractall(dest)
|
||||
return dest
|
||||
except Exception as e:
|
||||
logger.error("archive_pipeline: rar extract failed for %s: %s", archive_path, e)
|
||||
return None
|
||||
|
||||
|
||||
def _extract_7z(archive_path: Path, dest: Path) -> Optional[Path]:
|
||||
try:
|
||||
import py7zr # type: ignore[import-untyped]
|
||||
except ImportError:
|
||||
logger.warning(
|
||||
"archive_pipeline: cannot extract %s — py7zr library not installed. "
|
||||
"Install with: pip install py7zr.",
|
||||
archive_path,
|
||||
)
|
||||
return None
|
||||
try:
|
||||
with py7zr.SevenZipFile(archive_path, 'r') as sz:
|
||||
dest_resolved = dest.resolve()
|
||||
for name in sz.getnames():
|
||||
target = (dest_resolved / name).resolve()
|
||||
if dest_resolved not in target.parents and target != dest_resolved:
|
||||
logger.error("archive_pipeline: refusing path-traversal 7z member %r", name)
|
||||
return None
|
||||
sz.extractall(path=dest)
|
||||
return dest
|
||||
except Exception as e:
|
||||
logger.error("archive_pipeline: 7z extract failed for %s: %s", archive_path, e)
|
||||
return None
|
||||
|
|
@ -191,6 +191,12 @@ def run_service_test(service, test_config):
|
|||
'tidal': "Tidal download source ready.",
|
||||
'qobuz': "Qobuz download source ready.",
|
||||
'hifi': "HiFi download source ready.",
|
||||
'deezer_dl': "Deezer download source ready.",
|
||||
'amazon': "Amazon download source ready.",
|
||||
'lidarr': "Lidarr download source ready.",
|
||||
'soundcloud': "SoundCloud download source ready.",
|
||||
'torrent': "Torrent download source ready.",
|
||||
'usenet': "Usenet download source ready.",
|
||||
'hybrid': "Download sources ready (Hybrid mode)."
|
||||
}
|
||||
message = mode_messages.get(download_mode, "Download source connected.")
|
||||
|
|
@ -203,6 +209,12 @@ def run_service_test(service, test_config):
|
|||
'tidal': "Tidal download source not available. Check authentication.",
|
||||
'qobuz': "Qobuz download source not available. Check authentication.",
|
||||
'hifi': "HiFi download source not available. Public API instances may be down.",
|
||||
'deezer_dl': "Deezer download source not available. Check authentication.",
|
||||
'amazon': "Amazon download source not available.",
|
||||
'lidarr': "Lidarr download source not available. Check Lidarr URL and API key.",
|
||||
'soundcloud': "SoundCloud download source not available.",
|
||||
'torrent': "Torrent download source not available. Check Prowlarr and torrent client settings.",
|
||||
'usenet': "Usenet download source not available. Check Prowlarr and usenet client settings.",
|
||||
'hybrid': "Could not connect to download sources. Check configuration."
|
||||
}
|
||||
error = mode_errors.get(download_mode, "Download source connection failed.")
|
||||
|
|
|
|||
|
|
@ -286,11 +286,25 @@ class DownloadOrchestrator:
|
|||
chain = ['soulseek']
|
||||
return chain
|
||||
|
||||
async def search(self, query: str, timeout: int = None, progress_callback=None) -> Tuple[List[TrackResult], List[AlbumResult]]:
|
||||
async def search(self, query: str, timeout: int = None, progress_callback=None,
|
||||
exclude_sources=None) -> Tuple[List[TrackResult], List[AlbumResult]]:
|
||||
"""Search for tracks using configured source(s). Single-source
|
||||
modes route directly; hybrid mode delegates to
|
||||
``engine.search_with_fallback`` which tries the chain in order."""
|
||||
``engine.search_with_fallback`` which tries the chain in order.
|
||||
|
||||
``exclude_sources`` (optional) is an iterable of source names
|
||||
the caller wants filtered out of the hybrid chain. Used by the
|
||||
per-track download worker to skip torrent / usenet for album-
|
||||
context batches — those sources are release-level and don't
|
||||
score meaningfully on per-track titles; the album-bundle flow
|
||||
on the master worker handles them separately when they're the
|
||||
single active source."""
|
||||
if self.mode != 'hybrid':
|
||||
# Single-source mode is opt-in; honour the user's choice even
|
||||
# if it's torrent/usenet on an album batch (the master worker
|
||||
# routes those through the album-bundle flow before per-track
|
||||
# tasks ever fire, so search() being called here would be a
|
||||
# non-album / wishlist / basic-search use case).
|
||||
client = self._client(self.mode)
|
||||
if not client:
|
||||
logger.error(f"{self.registry.display_name(self.mode)} client not available (failed to initialize)")
|
||||
|
|
@ -299,6 +313,19 @@ class DownloadOrchestrator:
|
|||
return await client.search(query, timeout, progress_callback)
|
||||
|
||||
chain = self._resolve_source_chain()
|
||||
if exclude_sources:
|
||||
blocked = {s.lower() for s in exclude_sources if s}
|
||||
filtered = [s for s in chain if s.lower() not in blocked]
|
||||
if filtered != chain:
|
||||
logger.info(
|
||||
"Hybrid search: excluding %s for this query (chain %s -> %s)",
|
||||
sorted(blocked & {s.lower() for s in chain}),
|
||||
" → ".join(chain), " → ".join(filtered) if filtered else "(empty)",
|
||||
)
|
||||
chain = filtered
|
||||
if not chain:
|
||||
logger.warning("Hybrid search exhausted: no eligible sources after exclusion filter")
|
||||
return [], []
|
||||
logger.info(f"Hybrid search ({' → '.join(chain)}): {query}")
|
||||
return await self.engine.search_with_fallback(query, chain, timeout, progress_callback)
|
||||
|
||||
|
|
|
|||
217
core/download_plugins/album_bundle.py
Normal file
217
core/download_plugins/album_bundle.py
Normal file
|
|
@ -0,0 +1,217 @@
|
|||
"""Shared helpers for the album-bundle download flow.
|
||||
|
||||
The torrent and usenet download plugins both implement a
|
||||
``download_album_to_staging`` method that searches Prowlarr for a
|
||||
whole release, hands it to the active downloader, walks the
|
||||
resulting audio files, and copies them into the staging folder. The
|
||||
two implementations share the same release-picker heuristic and the
|
||||
same staging-path collision logic.
|
||||
|
||||
Pulled out of ``core/download_plugins/torrent.py`` so the usenet
|
||||
plugin doesn't have to import private helpers from a sibling
|
||||
plugin (Cin's "no leaky module boundaries" standard).
|
||||
|
||||
Also exposes ``atomic_copy_to_staging`` — the audio file is copied
|
||||
to a ``.tmp.<random>`` sidecar first and atomically renamed onto its
|
||||
final extension. The Auto-Import worker filters by audio extension
|
||||
so the in-flight ``.tmp`` file is never picked up mid-copy, closing
|
||||
the race between the album-bundle copy loop and Auto-Import's
|
||||
folder scan.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import shutil
|
||||
import time
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
from typing import Iterable, Optional
|
||||
|
||||
from config.settings import config_manager
|
||||
from utils.logging_config import get_logger
|
||||
|
||||
logger = get_logger("download_plugins.album_bundle")
|
||||
|
||||
|
||||
# Album-pick size floor / ceiling. Single-track torrents (~10 MB)
|
||||
# are rejected when bigger candidates exist; anything past 3 GB is
|
||||
# treated as suspicious (multi-disc box-set + scans + extras).
|
||||
ALBUM_PICK_MIN_BYTES = 40 * 1024 * 1024
|
||||
ALBUM_PICK_MAX_BYTES = 3 * 1024 * 1024 * 1024
|
||||
|
||||
|
||||
# Quality-score weights for the album-pick heuristic. Mirrors the
|
||||
# tier order in ``core/imports/file_ops.py``'s ``quality_tiers`` —
|
||||
# higher number = preferred.
|
||||
_QUALITY_SCORE = {'flac': 4, 'ogg': 3, 'aac': 2, 'mp3': 1}
|
||||
|
||||
|
||||
# Default poll cadence + timeout for the album-download poll loop.
|
||||
# Both are overridable through config so users with slow trackers
|
||||
# / large box-sets can extend the deadline without editing code.
|
||||
DEFAULT_POLL_INTERVAL_SECONDS = 2.0
|
||||
DEFAULT_POLL_TIMEOUT_SECONDS = 6 * 60 * 60
|
||||
|
||||
|
||||
def get_poll_interval() -> float:
|
||||
"""Return the per-poll sleep duration (seconds). Configurable via
|
||||
``download_source.album_bundle_poll_interval_seconds``."""
|
||||
raw = config_manager.get('download_source.album_bundle_poll_interval_seconds',
|
||||
DEFAULT_POLL_INTERVAL_SECONDS)
|
||||
try:
|
||||
value = float(raw)
|
||||
if value > 0:
|
||||
return value
|
||||
except (TypeError, ValueError):
|
||||
pass
|
||||
return DEFAULT_POLL_INTERVAL_SECONDS
|
||||
|
||||
|
||||
def get_poll_timeout() -> float:
|
||||
"""Return the total deadline for an album-bundle download
|
||||
(seconds). Configurable via
|
||||
``download_source.album_bundle_timeout_seconds``."""
|
||||
raw = config_manager.get('download_source.album_bundle_timeout_seconds',
|
||||
DEFAULT_POLL_TIMEOUT_SECONDS)
|
||||
try:
|
||||
value = float(raw)
|
||||
if value > 0:
|
||||
return value
|
||||
except (TypeError, ValueError):
|
||||
pass
|
||||
return DEFAULT_POLL_TIMEOUT_SECONDS
|
||||
|
||||
|
||||
def quality_score(title: str, quality_guess) -> int:
|
||||
"""Map a release title's inferred quality to a sortable integer.
|
||||
|
||||
``quality_guess`` is the function from each plugin that maps a
|
||||
title string to a quality string ('flac' / 'mp3' / etc.) — passed
|
||||
in so this module doesn't have to import either plugin and risk
|
||||
a circular import."""
|
||||
return _QUALITY_SCORE.get(quality_guess(title) or '', 0)
|
||||
|
||||
|
||||
def pick_best_album_release(candidates, quality_guess) -> Optional[object]:
|
||||
"""Pick the single best torrent / NZB for an album-bundle download.
|
||||
|
||||
Heuristic, in priority order:
|
||||
1. Reasonable album-ish size (40 MB – 3 GB) — drops single-track
|
||||
releases that snuck in and quarantines suspicious giants.
|
||||
2. Higher seeders > lower (dead torrents = dead downloads).
|
||||
Usenet releases use ``grabs`` as a popularity proxy when
|
||||
seeders is None.
|
||||
3. Higher quality (FLAC > AAC > MP3) inferred from title.
|
||||
4. Larger size as tiebreaker (often = higher bitrate).
|
||||
"""
|
||||
if not candidates:
|
||||
return None
|
||||
sized = [c for c in candidates
|
||||
if ALBUM_PICK_MIN_BYTES <= (c.size or 0) <= ALBUM_PICK_MAX_BYTES]
|
||||
pool = sized or list(candidates)
|
||||
if not pool:
|
||||
return None
|
||||
|
||||
def _score(c) -> tuple:
|
||||
seeders = c.seeders if c.seeders is not None else (c.grabs or 0)
|
||||
return (seeders, quality_score(c.title or '', quality_guess), c.size or 0)
|
||||
|
||||
return max(pool, key=_score)
|
||||
|
||||
|
||||
def unique_staging_path(staging_dir: Path, src: Path) -> Path:
|
||||
"""Return a destination path inside ``staging_dir`` that doesn't
|
||||
collide with an existing file. Appends ``_1``, ``_2``, ... before
|
||||
the extension when needed; gives up after 1000 candidates and
|
||||
returns the unsuffixed path so the caller will overwrite (better
|
||||
than infinite loop or crash)."""
|
||||
dest = staging_dir / src.name
|
||||
if not dest.exists():
|
||||
return dest
|
||||
stem = dest.stem
|
||||
suffix = dest.suffix
|
||||
for i in range(1, 1000):
|
||||
candidate = staging_dir / f"{stem}_{i}{suffix}"
|
||||
if not candidate.exists():
|
||||
return candidate
|
||||
return dest
|
||||
|
||||
|
||||
def atomic_copy_to_staging(src: Path, dest: Path) -> bool:
|
||||
"""Copy ``src`` to ``dest`` without exposing a partial file to
|
||||
folder scanners.
|
||||
|
||||
The Auto-Import worker filters by audio extension when scanning
|
||||
Staging — see ``AUDIO_EXTENSIONS`` in ``core/auto_import_worker.py``.
|
||||
Naming the in-flight file ``<dest>.tmp.<random>`` keeps it
|
||||
invisible until the rename atomically swings it to its final
|
||||
extension. ``os.replace`` (used by ``Path.rename`` on Python 3.x)
|
||||
is atomic on the same filesystem, so Auto-Import either sees the
|
||||
file at its final name (complete) or doesn't see it at all
|
||||
(in flight).
|
||||
|
||||
Returns True on success, False on copy / rename failure. Caller
|
||||
is expected to log the failure case so we don't double-log here.
|
||||
"""
|
||||
tmp = dest.with_name(f"{dest.name}.tmp.{uuid.uuid4().hex[:8]}")
|
||||
try:
|
||||
shutil.copy2(src, tmp)
|
||||
except Exception:
|
||||
# Best-effort cleanup of the partial file. If unlink fails
|
||||
# (locked, permissions) we leave it — Auto-Import ignores it
|
||||
# anyway because of the .tmp extension.
|
||||
try:
|
||||
if tmp.exists():
|
||||
tmp.unlink()
|
||||
except Exception as cleanup_exc:
|
||||
logger.debug("album_bundle tmp cleanup failed: %s", cleanup_exc)
|
||||
raise
|
||||
try:
|
||||
tmp.replace(dest)
|
||||
return True
|
||||
except Exception:
|
||||
try:
|
||||
tmp.unlink(missing_ok=True)
|
||||
except Exception as cleanup_exc:
|
||||
logger.debug("album_bundle tmp cleanup failed: %s", cleanup_exc)
|
||||
raise
|
||||
|
||||
|
||||
def copy_audio_files_atomically(
|
||||
sources: Iterable[Path], staging_dir: Path,
|
||||
) -> list:
|
||||
"""Convenience wrapper: pick a non-colliding staging path for
|
||||
each source, copy via ``atomic_copy_to_staging``. Returns the
|
||||
list of final destination paths (as strings). Files that fail
|
||||
to copy are logged and skipped; the caller decides what to do
|
||||
with a partial result."""
|
||||
staging_dir.mkdir(parents=True, exist_ok=True)
|
||||
out: list = []
|
||||
for src in sources:
|
||||
dest = unique_staging_path(staging_dir, src)
|
||||
try:
|
||||
atomic_copy_to_staging(src, dest)
|
||||
out.append(str(dest))
|
||||
except Exception as e:
|
||||
logger.warning("[album_bundle] Failed to stage %s -> %s: %s", src, dest, e)
|
||||
return out
|
||||
|
||||
|
||||
# Re-export so callers don't have to remember which module owns
|
||||
# what. The ``time`` import is kept so plugins can ``from
|
||||
# core.download_plugins.album_bundle import time`` if they want to,
|
||||
# avoiding a second std-lib import line for a single use.
|
||||
__all__ = [
|
||||
"ALBUM_PICK_MIN_BYTES",
|
||||
"ALBUM_PICK_MAX_BYTES",
|
||||
"DEFAULT_POLL_INTERVAL_SECONDS",
|
||||
"DEFAULT_POLL_TIMEOUT_SECONDS",
|
||||
"atomic_copy_to_staging",
|
||||
"copy_audio_files_atomically",
|
||||
"get_poll_interval",
|
||||
"get_poll_timeout",
|
||||
"pick_best_album_release",
|
||||
"quality_score",
|
||||
"time",
|
||||
"unique_staging_path",
|
||||
]
|
||||
|
|
@ -40,6 +40,8 @@ from core.download_plugins.base import DownloadSourcePlugin
|
|||
# orchestrator did.
|
||||
from core.amazon_download_client import AmazonDownloadClient
|
||||
from core.deezer_download_client import DeezerDownloadClient
|
||||
from core.download_plugins.torrent import TorrentDownloadPlugin
|
||||
from core.download_plugins.usenet import UsenetDownloadPlugin
|
||||
from core.hifi_client import HiFiClient
|
||||
from core.lidarr_download_client import LidarrDownloadClient
|
||||
from core.qobuz_client import QobuzClient
|
||||
|
|
@ -190,5 +192,7 @@ def build_default_registry() -> DownloadPluginRegistry:
|
|||
aliases=('deezer_dl',)))
|
||||
registry.register(PluginSpec(name='lidarr', factory=LidarrDownloadClient, display_name='Lidarr'))
|
||||
registry.register(PluginSpec(name='soundcloud',factory=SoundcloudClient, display_name='SoundCloud'))
|
||||
registry.register(PluginSpec(name='torrent', factory=TorrentDownloadPlugin, display_name='Torrent (Prowlarr)'))
|
||||
registry.register(PluginSpec(name='usenet', factory=UsenetDownloadPlugin, display_name='Usenet (Prowlarr)'))
|
||||
|
||||
return registry
|
||||
|
|
|
|||
666
core/download_plugins/torrent.py
Normal file
666
core/download_plugins/torrent.py
Normal file
|
|
@ -0,0 +1,666 @@
|
|||
"""TorrentDownloadPlugin — composes Prowlarr search + torrent client
|
||||
adapter + archive_pipeline into a uniform download source.
|
||||
|
||||
Two flows:
|
||||
|
||||
**Per-track flow** (basic search, single-track wishlist) —
|
||||
1. ``search(query)`` calls ``ProwlarrClient.search`` filtered to
|
||||
``protocol='torrent'`` results, projects releases into
|
||||
``TrackResult`` / ``AlbumResult`` shaped objects the existing
|
||||
search UI already understands. Encodes the indexer's
|
||||
``downloadUrl`` (or magnet URI) into the filename so
|
||||
``download()`` can recover it.
|
||||
2. ``download(username, filename, ...)`` decodes the URL, asks the
|
||||
active torrent adapter (qBittorrent, Transmission, or Deluge per
|
||||
user's settings) to add it, spawns a background thread that
|
||||
polls the adapter for completion.
|
||||
3. On completion the thread walks the adapter-reported save path
|
||||
via ``archive_pipeline.collect_audio_after_extraction`` and
|
||||
exposes the full audio-file list. Post-processing can then pick
|
||||
the requested track from a completed release instead of importing
|
||||
the first file blindly.
|
||||
|
||||
**Album-bundle flow** (album-context batch downloads — wired in
|
||||
``core/downloads/master.py``) —
|
||||
4. ``download_album_to_staging(album, artist, staging_dir)`` does
|
||||
ONE Prowlarr search for the whole release, picks the best
|
||||
torrent (prefers FLAC, decent seeders, reasonable size),
|
||||
downloads it, extracts archives if needed, copies every audio
|
||||
file into the staging directory. The existing per-track
|
||||
``try_staging_match`` flow then finds + imports each track by
|
||||
fuzzy title match against the staged files. Per-track Prowlarr
|
||||
queries never fire — track titles like "Luther (with SZA)"
|
||||
would match album torrents like "GNX (2024) [FLAC]" at near-
|
||||
zero confidence and break the per-track dispatch.
|
||||
|
||||
Limitations:
|
||||
- ``save_path`` is the torrent client's view of the disk. If
|
||||
SoulSync runs on a different host than qBit / Trans / Deluge,
|
||||
the post-processing pipeline can't see those files. The plugin
|
||||
works fine for the all-on-one-box case (most users); remote
|
||||
setups will need a future sync step (rclone / SMB / Docker
|
||||
bind mount).
|
||||
- Track-level metadata isn't available until after download.
|
||||
Search results carry only the release title + indexer metadata;
|
||||
individual track names are populated when the matching pipeline
|
||||
walks the extracted audio files.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import re
|
||||
import threading
|
||||
import time
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, List, Optional, Tuple
|
||||
|
||||
from config.settings import config_manager
|
||||
from core.archive_pipeline import collect_audio_after_extraction
|
||||
from core.download_plugins.album_bundle import (
|
||||
copy_audio_files_atomically,
|
||||
get_poll_interval,
|
||||
get_poll_timeout,
|
||||
pick_best_album_release,
|
||||
)
|
||||
from core.download_plugins.base import DownloadSourcePlugin
|
||||
from core.download_plugins.types import AlbumResult, DownloadStatus, TrackResult
|
||||
from core.prowlarr_client import (
|
||||
DEFAULT_MUSIC_CATEGORIES,
|
||||
ProwlarrClient,
|
||||
ProwlarrSearchResult,
|
||||
)
|
||||
from core.torrent_clients import get_active_adapter as get_active_torrent_adapter
|
||||
from utils.async_helpers import run_async
|
||||
from utils.logging_config import get_logger
|
||||
|
||||
logger = get_logger("download_plugins.torrent")
|
||||
|
||||
|
||||
# Separator used to encode the download URL inside the filename
|
||||
# field. Same convention Lidarr / YouTube use for embedding their
|
||||
# own opaque identifiers — ``<download_url>||<display>``.
|
||||
_FILENAME_SEP = '||'
|
||||
|
||||
# Adapter states that count as the download being on-disk and
|
||||
# safe to walk. ``seeding`` and ``completed`` both mean the
|
||||
# bits are there; the user can pause seeding manually if they
|
||||
# don't want to keep sharing.
|
||||
_COMPLETE_STATES = frozenset(['seeding', 'completed'])
|
||||
|
||||
# Poll cadence / timeout — both pull from config via the shared
|
||||
# album_bundle helpers so users can extend the deadline for slow
|
||||
# trackers without editing source. Kept as module aliases so the
|
||||
# per-track flow at the bottom of this file can still import them
|
||||
# under the legacy names without re-reading config every loop.
|
||||
_POLL_TIMEOUT_SECONDS = get_poll_timeout()
|
||||
_POLL_INTERVAL_SECONDS = get_poll_interval()
|
||||
|
||||
|
||||
class TorrentDownloadPlugin(DownloadSourcePlugin):
|
||||
"""Torrent download source backed by Prowlarr + an active
|
||||
torrent client adapter."""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self._prowlarr = ProwlarrClient()
|
||||
# Track every download we've kicked off. Keyed by our own
|
||||
# uuid — NOT the adapter's hash — because the orchestrator
|
||||
# owns the lifecycle and we need a stable id even before
|
||||
# the adapter has assigned one.
|
||||
self.active_downloads: Dict[str, Dict[str, Any]] = {}
|
||||
self._lock = threading.Lock()
|
||||
self.shutdown_check = None
|
||||
|
||||
def set_shutdown_check(self, check_callable):
|
||||
self.shutdown_check = check_callable
|
||||
|
||||
def reload_settings(self) -> None:
|
||||
self._prowlarr.reload_settings()
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Lifecycle
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def is_configured(self) -> bool:
|
||||
if not self._prowlarr.is_configured():
|
||||
return False
|
||||
adapter = get_active_torrent_adapter()
|
||||
return bool(adapter and adapter.is_configured())
|
||||
|
||||
async def check_connection(self) -> bool:
|
||||
if not self._prowlarr.is_configured():
|
||||
return False
|
||||
adapter = get_active_torrent_adapter()
|
||||
if not adapter or not adapter.is_configured():
|
||||
return False
|
||||
# Probe both sides. A torrent download is useless if either
|
||||
# the indexer or the downloader is unreachable.
|
||||
prowlarr_ok = await self._prowlarr.check_connection()
|
||||
if not prowlarr_ok:
|
||||
return False
|
||||
return await adapter.check_connection()
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Search
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
async def search(
|
||||
self,
|
||||
query: str,
|
||||
timeout: Optional[int] = None,
|
||||
progress_callback=None,
|
||||
) -> Tuple[List[TrackResult], List[AlbumResult]]:
|
||||
if not self._prowlarr.is_configured():
|
||||
return ([], [])
|
||||
try:
|
||||
indexer_ids = _parse_indexer_id_filter()
|
||||
results = await self._prowlarr.search(
|
||||
query,
|
||||
categories=DEFAULT_MUSIC_CATEGORIES,
|
||||
indexer_ids=indexer_ids,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.error("Torrent plugin search failed: %s", e)
|
||||
return ([], [])
|
||||
return self._project_results(results)
|
||||
|
||||
def _project_results(
|
||||
self, results: List[ProwlarrSearchResult]
|
||||
) -> Tuple[List[TrackResult], List[AlbumResult]]:
|
||||
"""Turn Prowlarr releases into TrackResult / AlbumResult
|
||||
shaped objects. One TrackResult + one AlbumResult per
|
||||
release — Prowlarr search hits are at the release level,
|
||||
not the track level, so we can't synthesise track listings
|
||||
without downloading the actual torrent."""
|
||||
tracks: List[TrackResult] = []
|
||||
albums: List[AlbumResult] = []
|
||||
for result in results:
|
||||
if result.protocol != 'torrent':
|
||||
continue
|
||||
download_url = result.magnet_uri or result.download_url
|
||||
if not download_url:
|
||||
continue
|
||||
filename = f"{download_url}{_FILENAME_SEP}{result.title}"
|
||||
quality = _guess_quality_from_title(result.title)
|
||||
parsed_artist, parsed_title = _parse_release_title(result.title)
|
||||
tr = TrackResult(
|
||||
username='torrent',
|
||||
filename=filename,
|
||||
size=result.size,
|
||||
bitrate=None,
|
||||
duration=None,
|
||||
quality=quality,
|
||||
# Torrent results don't have per-uploader slot / queue
|
||||
# data the way Soulseek does. Fill with neutral values
|
||||
# so the quality_score doesn't punish them artificially.
|
||||
free_upload_slots=max(1, result.seeders or 0),
|
||||
upload_speed=0,
|
||||
queue_length=0,
|
||||
# Pre-fill artist + title so TrackResult.__post_init__
|
||||
# doesn't auto-parse the filename — our filename starts
|
||||
# with the indexer download URL, which would otherwise
|
||||
# show up as "by download?apikey=..." in the UI.
|
||||
artist=parsed_artist or result.indexer_name or 'Torrent',
|
||||
title=parsed_title or result.title,
|
||||
album=parsed_title or None,
|
||||
track_number=None,
|
||||
_source_metadata={
|
||||
'indexer': result.indexer_name,
|
||||
'indexer_id': result.indexer_id,
|
||||
'seeders': result.seeders,
|
||||
'leechers': result.leechers,
|
||||
'grabs': result.grabs,
|
||||
'protocol': 'torrent',
|
||||
},
|
||||
)
|
||||
tracks.append(tr)
|
||||
albums.append(AlbumResult(
|
||||
username='torrent',
|
||||
album_path=f"torrent/{result.guid}",
|
||||
album_title=parsed_title or result.title,
|
||||
artist=parsed_artist or None,
|
||||
track_count=1, # unknown until download finishes
|
||||
total_size=result.size,
|
||||
tracks=[tr],
|
||||
dominant_quality=quality,
|
||||
year=None,
|
||||
))
|
||||
return tracks, albums
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Download
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
async def download(
|
||||
self,
|
||||
username: str,
|
||||
filename: str,
|
||||
file_size: int = 0,
|
||||
) -> Optional[str]:
|
||||
if not self.is_configured():
|
||||
return None
|
||||
download_url, display_name = _decode_filename(filename)
|
||||
if not download_url:
|
||||
logger.error("Torrent download missing URL in filename: %r", filename)
|
||||
return None
|
||||
|
||||
download_id = str(uuid.uuid4())
|
||||
with self._lock:
|
||||
self.active_downloads[download_id] = {
|
||||
'id': download_id,
|
||||
'filename': filename,
|
||||
'username': 'torrent',
|
||||
'display_name': display_name,
|
||||
'state': 'Initializing',
|
||||
'progress': 0.0,
|
||||
'size': file_size,
|
||||
'transferred': 0,
|
||||
'speed': 0,
|
||||
'file_path': None,
|
||||
'audio_files': [],
|
||||
'torrent_hash': None,
|
||||
'error': None,
|
||||
}
|
||||
|
||||
thread = threading.Thread(
|
||||
target=self._download_thread,
|
||||
args=(download_id, download_url, display_name),
|
||||
daemon=True,
|
||||
name=f'torrent-dl-{download_id[:8]}',
|
||||
)
|
||||
thread.start()
|
||||
return download_id
|
||||
|
||||
def _download_thread(self, download_id: str, download_url: str, display_name: str) -> None:
|
||||
"""Background worker: hand the URL to the active adapter,
|
||||
poll until done, then walk the resulting directory."""
|
||||
adapter = get_active_torrent_adapter()
|
||||
if adapter is None or not adapter.is_configured():
|
||||
self._mark_error(download_id, "No torrent client configured")
|
||||
return
|
||||
|
||||
try:
|
||||
torrent_hash = run_async(adapter.add_torrent(download_url))
|
||||
except Exception as e:
|
||||
self._mark_error(download_id, f"add_torrent failed: {e}")
|
||||
return
|
||||
if not torrent_hash:
|
||||
self._mark_error(download_id, "Torrent client refused the URL")
|
||||
return
|
||||
|
||||
with self._lock:
|
||||
row = self.active_downloads.get(download_id)
|
||||
if row is not None:
|
||||
row['torrent_hash'] = torrent_hash
|
||||
row['state'] = 'InProgress, Downloading'
|
||||
|
||||
deadline = time.monotonic() + _POLL_TIMEOUT_SECONDS
|
||||
last_save_path: Optional[str] = None
|
||||
while time.monotonic() < deadline:
|
||||
if self.shutdown_check and self.shutdown_check():
|
||||
return
|
||||
try:
|
||||
status = run_async(adapter.get_status(torrent_hash))
|
||||
except Exception as e:
|
||||
logger.warning("Torrent poll error for %s: %s", torrent_hash, e)
|
||||
status = None
|
||||
|
||||
if status is None:
|
||||
# Adapter forgot about the torrent — probably user-removed.
|
||||
self._mark_error(download_id, "Torrent disappeared from client")
|
||||
return
|
||||
|
||||
with self._lock:
|
||||
row = self.active_downloads.get(download_id)
|
||||
if row is not None:
|
||||
row['progress'] = status.progress * 100.0
|
||||
row['transferred'] = status.downloaded
|
||||
row['speed'] = status.download_speed
|
||||
row['size'] = status.size or row.get('size', 0)
|
||||
row['state'] = _adapter_state_to_display(status.state)
|
||||
row['error'] = status.error
|
||||
if status.save_path:
|
||||
last_save_path = status.save_path
|
||||
|
||||
if status.state in _COMPLETE_STATES:
|
||||
self._finalize_download(download_id, last_save_path)
|
||||
return
|
||||
if status.state == 'error':
|
||||
self._mark_error(download_id, status.error or "Torrent client reported error")
|
||||
return
|
||||
|
||||
time.sleep(_POLL_INTERVAL_SECONDS)
|
||||
|
||||
self._mark_error(download_id, "Torrent download timed out")
|
||||
|
||||
def _finalize_download(self, download_id: str, save_path: Optional[str]) -> None:
|
||||
"""Adapter said complete. Walk the directory + pick the
|
||||
first audio file as the canonical ``file_path``."""
|
||||
if not save_path:
|
||||
self._mark_error(download_id, "Torrent completed but no save_path reported")
|
||||
return
|
||||
try:
|
||||
audio_files = collect_audio_after_extraction(Path(save_path))
|
||||
except Exception as e:
|
||||
self._mark_error(download_id, f"Post-extract walk failed: {e}")
|
||||
return
|
||||
if not audio_files:
|
||||
self._mark_error(download_id, f"No audio files found in {save_path}")
|
||||
return
|
||||
primary = audio_files[0]
|
||||
with self._lock:
|
||||
row = self.active_downloads.get(download_id)
|
||||
if row is not None:
|
||||
row['state'] = 'Completed, Succeeded'
|
||||
row['progress'] = 100.0
|
||||
row['file_path'] = str(primary)
|
||||
row['audio_files'] = [str(path) for path in audio_files]
|
||||
logger.info("Torrent download complete: %s -> %s (%d audio files)",
|
||||
download_id[:8], primary.name, len(audio_files))
|
||||
|
||||
def _mark_error(self, download_id: str, message: str) -> None:
|
||||
logger.error("Torrent download %s failed: %s", download_id[:8], message)
|
||||
with self._lock:
|
||||
row = self.active_downloads.get(download_id)
|
||||
if row is not None:
|
||||
row['state'] = 'Completed, Errored'
|
||||
row['error'] = message
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Status / lifecycle
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
async def get_all_downloads(self) -> List[DownloadStatus]:
|
||||
with self._lock:
|
||||
rows = list(self.active_downloads.values())
|
||||
return [_row_to_status(r) for r in rows]
|
||||
|
||||
async def get_download_status(self, download_id: str) -> Optional[DownloadStatus]:
|
||||
with self._lock:
|
||||
row = self.active_downloads.get(download_id)
|
||||
if row is None:
|
||||
return None
|
||||
return _row_to_status(row)
|
||||
|
||||
async def cancel_download(
|
||||
self,
|
||||
download_id: str,
|
||||
username: Optional[str] = None,
|
||||
remove: bool = False,
|
||||
) -> bool:
|
||||
adapter = get_active_torrent_adapter()
|
||||
with self._lock:
|
||||
row = self.active_downloads.get(download_id)
|
||||
torrent_hash = row.get('torrent_hash') if row else None
|
||||
if adapter and torrent_hash:
|
||||
try:
|
||||
await adapter.remove(torrent_hash, delete_files=remove)
|
||||
except Exception as e:
|
||||
logger.warning("Torrent cancel via adapter failed: %s", e)
|
||||
with self._lock:
|
||||
if remove:
|
||||
self.active_downloads.pop(download_id, None)
|
||||
else:
|
||||
row = self.active_downloads.get(download_id)
|
||||
if row is not None:
|
||||
row['state'] = 'Cancelled'
|
||||
return True
|
||||
|
||||
async def clear_all_completed_downloads(self) -> bool:
|
||||
with self._lock:
|
||||
for did in list(self.active_downloads.keys()):
|
||||
state = self.active_downloads[did].get('state', '')
|
||||
if state.startswith('Completed') or state == 'Cancelled':
|
||||
self.active_downloads.pop(did, None)
|
||||
return True
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Album-bundle flow
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def download_album_to_staging(
|
||||
self,
|
||||
album_name: str,
|
||||
artist_name: str,
|
||||
staging_dir: str,
|
||||
progress_callback=None,
|
||||
) -> Dict[str, Any]:
|
||||
"""One-shot album download: search Prowlarr for the whole
|
||||
release, pick the best torrent, fetch it, extract if needed,
|
||||
copy every audio file into ``staging_dir`` so the existing
|
||||
``try_staging_match`` flow can hand each track off to the
|
||||
post-processing pipeline.
|
||||
|
||||
``progress_callback`` is called with a dict on each state
|
||||
change so the batch UI can show download progress without
|
||||
waiting for the whole thing.
|
||||
|
||||
Returns ``{'success': bool, 'files': [paths], 'error': str|None}``.
|
||||
"""
|
||||
result: Dict[str, Any] = {'success': False, 'files': [], 'error': None}
|
||||
if not self.is_configured():
|
||||
result['error'] = 'Torrent source not configured'
|
||||
return result
|
||||
|
||||
adapter = get_active_torrent_adapter()
|
||||
if adapter is None or not adapter.is_configured():
|
||||
result['error'] = 'No active torrent client'
|
||||
return result
|
||||
|
||||
def _emit(state: str, **extra) -> None:
|
||||
if progress_callback:
|
||||
payload = {'state': state, **extra}
|
||||
try:
|
||||
progress_callback(payload)
|
||||
except Exception as cb_exc:
|
||||
logger.debug("[Torrent album] progress callback failed: %s", cb_exc)
|
||||
|
||||
# Phase 1: search Prowlarr for the album.
|
||||
query = f"{artist_name} {album_name}".strip()
|
||||
_emit('searching', query=query)
|
||||
try:
|
||||
search_results = run_async(self._prowlarr.search(
|
||||
query, categories=DEFAULT_MUSIC_CATEGORIES,
|
||||
indexer_ids=_parse_indexer_id_filter(),
|
||||
))
|
||||
except Exception as e:
|
||||
result['error'] = f'Prowlarr search failed: {e}'
|
||||
return result
|
||||
|
||||
candidates = [r for r in search_results
|
||||
if r.protocol == 'torrent' and (r.magnet_uri or r.download_url)]
|
||||
if not candidates:
|
||||
result['error'] = f'No torrent results found for "{query}"'
|
||||
return result
|
||||
|
||||
picked = pick_best_album_release(candidates, _guess_quality_from_title)
|
||||
if picked is None:
|
||||
result['error'] = 'No suitable torrent candidate after filtering'
|
||||
return result
|
||||
|
||||
download_url = picked.magnet_uri or picked.download_url
|
||||
logger.info("[Torrent album] Picked '%s' (size=%.1fMB seeders=%s indexer=%s)",
|
||||
picked.title, picked.size / 1_048_576, picked.seeders, picked.indexer_name)
|
||||
_emit('queued', release=picked.title, size=picked.size, seeders=picked.seeders)
|
||||
|
||||
# Phase 2: hand to adapter.
|
||||
try:
|
||||
torrent_id = run_async(adapter.add_torrent(download_url))
|
||||
except Exception as e:
|
||||
result['error'] = f'Torrent client refused the release: {e}'
|
||||
return result
|
||||
if not torrent_id:
|
||||
result['error'] = 'Torrent client refused the release'
|
||||
return result
|
||||
|
||||
# Phase 3: poll until complete.
|
||||
_emit('downloading', release=picked.title)
|
||||
save_path = self._poll_album_download(adapter, torrent_id, picked.title, _emit)
|
||||
if save_path is None:
|
||||
result['error'] = 'Torrent download failed or timed out'
|
||||
return result
|
||||
|
||||
# Phase 4: extract + walk + copy to staging.
|
||||
_emit('staging', release=picked.title)
|
||||
try:
|
||||
audio_files = collect_audio_after_extraction(Path(save_path))
|
||||
except Exception as e:
|
||||
result['error'] = f'Failed to walk audio files: {e}'
|
||||
return result
|
||||
if not audio_files:
|
||||
result['error'] = f'No audio files found in {save_path}'
|
||||
return result
|
||||
|
||||
copied = copy_audio_files_atomically(audio_files, Path(staging_dir))
|
||||
if not copied:
|
||||
result['error'] = 'No audio files copied to staging'
|
||||
return result
|
||||
logger.info("[Torrent album] Staged %d audio files for '%s'", len(copied), album_name)
|
||||
_emit('staged', count=len(copied))
|
||||
result['success'] = True
|
||||
result['files'] = copied
|
||||
return result
|
||||
|
||||
def _poll_album_download(self, adapter, torrent_id, title, emit) -> Optional[str]:
|
||||
"""Poll the adapter until the torrent is complete. Returns
|
||||
the save path or ``None`` on timeout / failure."""
|
||||
deadline = time.monotonic() + _POLL_TIMEOUT_SECONDS
|
||||
last_save_path: Optional[str] = None
|
||||
while time.monotonic() < deadline:
|
||||
if self.shutdown_check and self.shutdown_check():
|
||||
return None
|
||||
try:
|
||||
status = run_async(adapter.get_status(torrent_id))
|
||||
except Exception as e:
|
||||
logger.warning("[Torrent album] Poll error: %s", e)
|
||||
status = None
|
||||
if status is None:
|
||||
logger.error("[Torrent album] '%s' disappeared from client", title)
|
||||
return None
|
||||
emit('downloading', progress=status.progress, downloaded=status.downloaded,
|
||||
speed=status.download_speed)
|
||||
if status.save_path:
|
||||
last_save_path = status.save_path
|
||||
if status.state in _COMPLETE_STATES:
|
||||
return last_save_path
|
||||
if status.state == 'error':
|
||||
logger.error("[Torrent album] '%s' errored: %s", title, status.error)
|
||||
return None
|
||||
time.sleep(_POLL_INTERVAL_SECONDS)
|
||||
logger.error("[Torrent album] '%s' timed out", title)
|
||||
return None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Module-level helpers (pure functions — easy to unit-test)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _decode_filename(filename: str) -> Tuple[Optional[str], str]:
|
||||
"""Pull the encoded download URL out of the ``filename`` string.
|
||||
Returns ``(url, display_name)``. ``url`` is None when the string
|
||||
has no separator."""
|
||||
if not filename or _FILENAME_SEP not in filename:
|
||||
return (None, filename or '')
|
||||
url, display = filename.split(_FILENAME_SEP, 1)
|
||||
return (url, display)
|
||||
|
||||
|
||||
def _parse_release_title(title: str) -> Tuple[str, str]:
|
||||
"""Split a release title into ``(artist, title)`` using the
|
||||
``Artist - Title`` / ``Artist - Album`` convention almost every
|
||||
indexer follows. Returns ``('', title)`` when no dash is found.
|
||||
|
||||
Without this, ``TrackResult.__post_init__`` runs the bare
|
||||
filename through ``parse_filename_metadata`` — and our filename
|
||||
starts with the indexer's download URL, so the auto-parser
|
||||
extracts garbage like ``download?apikey=...`` as the artist
|
||||
and shows it in the search-result UI's "by" line. Pre-filling
|
||||
the artist field short-circuits the auto-parse.
|
||||
"""
|
||||
if not title:
|
||||
return ('', '')
|
||||
# Strip common quality / format tags so the dash split doesn't
|
||||
# eat them — "Artist - Album [FLAC] (2020)" → "Artist", "Album".
|
||||
cleaned = re.sub(r'\s*[\[\(][^\]\)]*[\]\)]\s*$', '', title.strip())
|
||||
# Look for the FIRST " - " (or "-" surrounded by content). Some
|
||||
# release titles have multiple dashes (subtitle dashes); the
|
||||
# first split is the artist/work boundary.
|
||||
parts = re.split(r'\s+-\s+|\s+-(?=\S)|(?<=\S)-\s+', cleaned, maxsplit=1)
|
||||
if len(parts) == 2:
|
||||
artist = parts[0].strip()
|
||||
rest = parts[1].strip()
|
||||
# Reject obvious non-artist prefixes (URLs, hashes, single
|
||||
# punctuation) so we don't propagate garbage.
|
||||
if artist and not re.match(r'^https?:|^[a-f0-9]{32,}$', artist):
|
||||
return (artist, rest or cleaned)
|
||||
return ('', cleaned)
|
||||
|
||||
|
||||
def _guess_quality_from_title(title: str) -> str:
|
||||
"""Read the quality hint from a release title — most music
|
||||
torrents put the encoding right in the name (FLAC, MP3 320,
|
||||
etc.). Falls back to ``'mp3'`` so quality_score doesn't crash."""
|
||||
if not title:
|
||||
return 'mp3'
|
||||
lower = title.lower()
|
||||
if 'flac' in lower:
|
||||
return 'flac'
|
||||
if re.search(r'\b24[\s-]?bit\b', lower) or 'hi-?res' in lower:
|
||||
return 'flac'
|
||||
if 'aac' in lower:
|
||||
return 'aac'
|
||||
if 'ogg' in lower:
|
||||
return 'ogg'
|
||||
return 'mp3'
|
||||
|
||||
|
||||
def _parse_indexer_id_filter() -> List[int]:
|
||||
"""Read the comma-separated indexer-ID allowlist from config.
|
||||
Empty list = search every enabled indexer."""
|
||||
raw = (config_manager.get('prowlarr.indexer_ids', '') or '').strip()
|
||||
if not raw:
|
||||
return []
|
||||
out: List[int] = []
|
||||
for chunk in raw.split(','):
|
||||
chunk = chunk.strip()
|
||||
if not chunk:
|
||||
continue
|
||||
try:
|
||||
out.append(int(chunk))
|
||||
except ValueError:
|
||||
continue
|
||||
return out
|
||||
|
||||
|
||||
def _adapter_state_to_display(state: str) -> str:
|
||||
"""Translate the adapter-uniform state strings into the
|
||||
``'InProgress, Downloading'`` / ``'Completed, Succeeded'``
|
||||
style the existing UI expects (matches Soulseek + Lidarr)."""
|
||||
mapping = {
|
||||
'queued': 'Queued',
|
||||
'downloading': 'InProgress, Downloading',
|
||||
'stalled': 'InProgress, Stalled',
|
||||
'seeding': 'Completed, Succeeded',
|
||||
'completed': 'Completed, Succeeded',
|
||||
'paused': 'Paused',
|
||||
'error': 'Completed, Errored',
|
||||
}
|
||||
return mapping.get(state, state.title())
|
||||
|
||||
|
||||
def _row_to_status(row: Dict[str, Any]) -> DownloadStatus:
|
||||
return DownloadStatus(
|
||||
id=row['id'],
|
||||
filename=row['filename'],
|
||||
username=row['username'],
|
||||
state=row.get('state', 'Unknown'),
|
||||
progress=float(row.get('progress', 0.0)),
|
||||
size=int(row.get('size', 0)),
|
||||
transferred=int(row.get('transferred', 0)),
|
||||
speed=int(row.get('speed', 0)),
|
||||
time_remaining=None,
|
||||
file_path=row.get('file_path'),
|
||||
audio_files=row.get('audio_files') or None,
|
||||
)
|
||||
|
|
@ -197,3 +197,4 @@ class DownloadStatus:
|
|||
speed: int
|
||||
time_remaining: Optional[int] = None
|
||||
file_path: Optional[str] = None
|
||||
audio_files: Optional[List[str]] = None
|
||||
|
|
|
|||
461
core/download_plugins/usenet.py
Normal file
461
core/download_plugins/usenet.py
Normal file
|
|
@ -0,0 +1,461 @@
|
|||
"""UsenetDownloadPlugin — composes Prowlarr search + usenet client
|
||||
adapter + archive_pipeline into a uniform download source.
|
||||
|
||||
Mirrors ``TorrentDownloadPlugin`` in shape and lifecycle (see that
|
||||
module's docstring for the full pipeline rationale). Differences:
|
||||
|
||||
- Search filters Prowlarr results to ``protocol='usenet'``.
|
||||
- ``add_nzb`` replaces ``add_torrent``; for NZBs we usually have
|
||||
a direct HTTP URL the indexer exposes via Prowlarr.
|
||||
- Usenet clients (SABnzbd, NZBGet) typically auto-extract during
|
||||
post-processing, so ``archive_pipeline.collect_audio_after_extraction``
|
||||
usually has nothing to extract and just walks loose files.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import threading
|
||||
import time
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, List, Optional, Tuple
|
||||
|
||||
from core.archive_pipeline import collect_audio_after_extraction
|
||||
from core.download_plugins.album_bundle import (
|
||||
copy_audio_files_atomically,
|
||||
pick_best_album_release,
|
||||
)
|
||||
from core.download_plugins.base import DownloadSourcePlugin
|
||||
from core.download_plugins.torrent import (
|
||||
_adapter_state_to_display,
|
||||
_decode_filename,
|
||||
_guess_quality_from_title,
|
||||
_parse_indexer_id_filter,
|
||||
_parse_release_title,
|
||||
_row_to_status,
|
||||
_COMPLETE_STATES,
|
||||
_FILENAME_SEP,
|
||||
_POLL_INTERVAL_SECONDS,
|
||||
_POLL_TIMEOUT_SECONDS,
|
||||
)
|
||||
from core.download_plugins.types import AlbumResult, DownloadStatus, TrackResult
|
||||
from core.prowlarr_client import (
|
||||
DEFAULT_MUSIC_CATEGORIES,
|
||||
ProwlarrClient,
|
||||
ProwlarrSearchResult,
|
||||
)
|
||||
from core.usenet_clients import get_active_adapter as get_active_usenet_adapter
|
||||
from utils.async_helpers import run_async
|
||||
from utils.logging_config import get_logger
|
||||
|
||||
logger = get_logger("download_plugins.usenet")
|
||||
|
||||
|
||||
class UsenetDownloadPlugin(DownloadSourcePlugin):
|
||||
"""Usenet download source backed by Prowlarr + an active usenet
|
||||
client adapter (SABnzbd or NZBGet)."""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self._prowlarr = ProwlarrClient()
|
||||
self.active_downloads: Dict[str, Dict[str, Any]] = {}
|
||||
self._lock = threading.Lock()
|
||||
self.shutdown_check = None
|
||||
|
||||
def set_shutdown_check(self, check_callable):
|
||||
self.shutdown_check = check_callable
|
||||
|
||||
def reload_settings(self) -> None:
|
||||
self._prowlarr.reload_settings()
|
||||
|
||||
def is_configured(self) -> bool:
|
||||
if not self._prowlarr.is_configured():
|
||||
return False
|
||||
adapter = get_active_usenet_adapter()
|
||||
return bool(adapter and adapter.is_configured())
|
||||
|
||||
async def check_connection(self) -> bool:
|
||||
if not self._prowlarr.is_configured():
|
||||
return False
|
||||
adapter = get_active_usenet_adapter()
|
||||
if not adapter or not adapter.is_configured():
|
||||
return False
|
||||
if not await self._prowlarr.check_connection():
|
||||
return False
|
||||
return await adapter.check_connection()
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Search
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
async def search(
|
||||
self,
|
||||
query: str,
|
||||
timeout: Optional[int] = None,
|
||||
progress_callback=None,
|
||||
) -> Tuple[List[TrackResult], List[AlbumResult]]:
|
||||
if not self._prowlarr.is_configured():
|
||||
return ([], [])
|
||||
try:
|
||||
indexer_ids = _parse_indexer_id_filter()
|
||||
results = await self._prowlarr.search(
|
||||
query,
|
||||
categories=DEFAULT_MUSIC_CATEGORIES,
|
||||
indexer_ids=indexer_ids,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.error("Usenet plugin search failed: %s", e)
|
||||
return ([], [])
|
||||
return self._project_results(results)
|
||||
|
||||
def _project_results(
|
||||
self, results: List[ProwlarrSearchResult]
|
||||
) -> Tuple[List[TrackResult], List[AlbumResult]]:
|
||||
tracks: List[TrackResult] = []
|
||||
albums: List[AlbumResult] = []
|
||||
for result in results:
|
||||
if result.protocol != 'usenet':
|
||||
continue
|
||||
if not result.download_url:
|
||||
continue
|
||||
filename = f"{result.download_url}{_FILENAME_SEP}{result.title}"
|
||||
quality = _guess_quality_from_title(result.title)
|
||||
parsed_artist, parsed_title = _parse_release_title(result.title)
|
||||
tr = TrackResult(
|
||||
username='usenet',
|
||||
filename=filename,
|
||||
size=result.size,
|
||||
bitrate=None,
|
||||
duration=None,
|
||||
quality=quality,
|
||||
# Usenet doesn't expose per-uploader concurrency the way
|
||||
# Soulseek does; fill in neutral non-punishing values.
|
||||
free_upload_slots=1,
|
||||
upload_speed=0,
|
||||
queue_length=0,
|
||||
# Pre-fill artist + title so TrackResult.__post_init__
|
||||
# doesn't auto-parse the filename — same URL-in-filename
|
||||
# gotcha as the torrent plugin.
|
||||
artist=parsed_artist or result.indexer_name or 'Usenet',
|
||||
title=parsed_title or result.title,
|
||||
album=parsed_title or None,
|
||||
track_number=None,
|
||||
_source_metadata={
|
||||
'indexer': result.indexer_name,
|
||||
'indexer_id': result.indexer_id,
|
||||
'grabs': result.grabs,
|
||||
'protocol': 'usenet',
|
||||
},
|
||||
)
|
||||
tracks.append(tr)
|
||||
albums.append(AlbumResult(
|
||||
username='usenet',
|
||||
album_path=f"usenet/{result.guid}",
|
||||
album_title=parsed_title or result.title,
|
||||
artist=parsed_artist or None,
|
||||
track_count=1,
|
||||
total_size=result.size,
|
||||
tracks=[tr],
|
||||
dominant_quality=quality,
|
||||
year=None,
|
||||
))
|
||||
return tracks, albums
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Download
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
async def download(
|
||||
self,
|
||||
username: str,
|
||||
filename: str,
|
||||
file_size: int = 0,
|
||||
) -> Optional[str]:
|
||||
if not self.is_configured():
|
||||
return None
|
||||
nzb_url, display_name = _decode_filename(filename)
|
||||
if not nzb_url:
|
||||
logger.error("Usenet download missing URL in filename: %r", filename)
|
||||
return None
|
||||
|
||||
download_id = str(uuid.uuid4())
|
||||
with self._lock:
|
||||
self.active_downloads[download_id] = {
|
||||
'id': download_id,
|
||||
'filename': filename,
|
||||
'username': 'usenet',
|
||||
'display_name': display_name,
|
||||
'state': 'Initializing',
|
||||
'progress': 0.0,
|
||||
'size': file_size,
|
||||
'transferred': 0,
|
||||
'speed': 0,
|
||||
'file_path': None,
|
||||
'audio_files': [],
|
||||
'job_id': None,
|
||||
'error': None,
|
||||
}
|
||||
|
||||
thread = threading.Thread(
|
||||
target=self._download_thread,
|
||||
args=(download_id, nzb_url),
|
||||
daemon=True,
|
||||
name=f'usenet-dl-{download_id[:8]}',
|
||||
)
|
||||
thread.start()
|
||||
return download_id
|
||||
|
||||
def _download_thread(self, download_id: str, nzb_url: str) -> None:
|
||||
adapter = get_active_usenet_adapter()
|
||||
if adapter is None or not adapter.is_configured():
|
||||
self._mark_error(download_id, "No usenet client configured")
|
||||
return
|
||||
|
||||
try:
|
||||
job_id = run_async(adapter.add_nzb(nzb_url))
|
||||
except Exception as e:
|
||||
self._mark_error(download_id, f"add_nzb failed: {e}")
|
||||
return
|
||||
if not job_id:
|
||||
self._mark_error(download_id, "Usenet client refused the NZB")
|
||||
return
|
||||
|
||||
with self._lock:
|
||||
row = self.active_downloads.get(download_id)
|
||||
if row is not None:
|
||||
row['job_id'] = job_id
|
||||
row['state'] = 'InProgress, Downloading'
|
||||
|
||||
deadline = time.monotonic() + _POLL_TIMEOUT_SECONDS
|
||||
last_save_path: Optional[str] = None
|
||||
while time.monotonic() < deadline:
|
||||
if self.shutdown_check and self.shutdown_check():
|
||||
return
|
||||
try:
|
||||
status = run_async(adapter.get_status(job_id))
|
||||
except Exception as e:
|
||||
logger.warning("Usenet poll error for %s: %s", job_id, e)
|
||||
status = None
|
||||
|
||||
if status is None:
|
||||
self._mark_error(download_id, "Usenet job disappeared from client")
|
||||
return
|
||||
|
||||
with self._lock:
|
||||
row = self.active_downloads.get(download_id)
|
||||
if row is not None:
|
||||
row['progress'] = status.progress * 100.0
|
||||
row['transferred'] = status.downloaded
|
||||
row['speed'] = status.download_speed
|
||||
row['size'] = status.size or row.get('size', 0)
|
||||
row['state'] = _adapter_state_to_display(status.state)
|
||||
row['error'] = status.error
|
||||
if status.save_path:
|
||||
last_save_path = status.save_path
|
||||
|
||||
if status.state in _COMPLETE_STATES:
|
||||
self._finalize_download(download_id, last_save_path)
|
||||
return
|
||||
if status.state == 'failed':
|
||||
self._mark_error(download_id, status.error or "Usenet client reported failure")
|
||||
return
|
||||
|
||||
time.sleep(_POLL_INTERVAL_SECONDS)
|
||||
|
||||
self._mark_error(download_id, "Usenet download timed out")
|
||||
|
||||
def _finalize_download(self, download_id: str, save_path: Optional[str]) -> None:
|
||||
if not save_path:
|
||||
self._mark_error(download_id, "Usenet job completed but no save_path reported")
|
||||
return
|
||||
try:
|
||||
audio_files = collect_audio_after_extraction(Path(save_path))
|
||||
except Exception as e:
|
||||
self._mark_error(download_id, f"Post-extract walk failed: {e}")
|
||||
return
|
||||
if not audio_files:
|
||||
self._mark_error(download_id, f"No audio files found in {save_path}")
|
||||
return
|
||||
primary = audio_files[0]
|
||||
with self._lock:
|
||||
row = self.active_downloads.get(download_id)
|
||||
if row is not None:
|
||||
row['state'] = 'Completed, Succeeded'
|
||||
row['progress'] = 100.0
|
||||
row['file_path'] = str(primary)
|
||||
row['audio_files'] = [str(path) for path in audio_files]
|
||||
logger.info("Usenet download complete: %s -> %s (%d audio files)",
|
||||
download_id[:8], primary.name, len(audio_files))
|
||||
|
||||
def _mark_error(self, download_id: str, message: str) -> None:
|
||||
logger.error("Usenet download %s failed: %s", download_id[:8], message)
|
||||
with self._lock:
|
||||
row = self.active_downloads.get(download_id)
|
||||
if row is not None:
|
||||
row['state'] = 'Completed, Errored'
|
||||
row['error'] = message
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Status / lifecycle
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
async def get_all_downloads(self) -> List[DownloadStatus]:
|
||||
with self._lock:
|
||||
rows = list(self.active_downloads.values())
|
||||
return [_row_to_status(r) for r in rows]
|
||||
|
||||
async def get_download_status(self, download_id: str) -> Optional[DownloadStatus]:
|
||||
with self._lock:
|
||||
row = self.active_downloads.get(download_id)
|
||||
if row is None:
|
||||
return None
|
||||
return _row_to_status(row)
|
||||
|
||||
async def cancel_download(
|
||||
self,
|
||||
download_id: str,
|
||||
username: Optional[str] = None,
|
||||
remove: bool = False,
|
||||
) -> bool:
|
||||
adapter = get_active_usenet_adapter()
|
||||
with self._lock:
|
||||
row = self.active_downloads.get(download_id)
|
||||
job_id = row.get('job_id') if row else None
|
||||
if adapter and job_id:
|
||||
try:
|
||||
await adapter.remove(job_id, delete_files=remove)
|
||||
except Exception as e:
|
||||
logger.warning("Usenet cancel via adapter failed: %s", e)
|
||||
with self._lock:
|
||||
if remove:
|
||||
self.active_downloads.pop(download_id, None)
|
||||
else:
|
||||
row = self.active_downloads.get(download_id)
|
||||
if row is not None:
|
||||
row['state'] = 'Cancelled'
|
||||
return True
|
||||
|
||||
async def clear_all_completed_downloads(self) -> bool:
|
||||
with self._lock:
|
||||
for did in list(self.active_downloads.keys()):
|
||||
state = self.active_downloads[did].get('state', '')
|
||||
if state.startswith('Completed') or state == 'Cancelled':
|
||||
self.active_downloads.pop(did, None)
|
||||
return True
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Album-bundle flow
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def download_album_to_staging(
|
||||
self,
|
||||
album_name: str,
|
||||
artist_name: str,
|
||||
staging_dir: str,
|
||||
progress_callback=None,
|
||||
) -> Dict[str, Any]:
|
||||
"""Usenet sibling of ``TorrentDownloadPlugin.download_album_to_staging``.
|
||||
See that method's docstring for the contract."""
|
||||
result: Dict[str, Any] = {'success': False, 'files': [], 'error': None}
|
||||
if not self.is_configured():
|
||||
result['error'] = 'Usenet source not configured'
|
||||
return result
|
||||
|
||||
adapter = get_active_usenet_adapter()
|
||||
if adapter is None or not adapter.is_configured():
|
||||
result['error'] = 'No active usenet client'
|
||||
return result
|
||||
|
||||
def _emit(state: str, **extra) -> None:
|
||||
if progress_callback:
|
||||
try:
|
||||
progress_callback({'state': state, **extra})
|
||||
except Exception as cb_exc:
|
||||
logger.debug("[Usenet album] progress callback failed: %s", cb_exc)
|
||||
|
||||
query = f"{artist_name} {album_name}".strip()
|
||||
_emit('searching', query=query)
|
||||
try:
|
||||
search_results = run_async(self._prowlarr.search(
|
||||
query, categories=DEFAULT_MUSIC_CATEGORIES,
|
||||
indexer_ids=_parse_indexer_id_filter(),
|
||||
))
|
||||
except Exception as e:
|
||||
result['error'] = f'Prowlarr search failed: {e}'
|
||||
return result
|
||||
|
||||
candidates = [r for r in search_results
|
||||
if r.protocol == 'usenet' and r.download_url]
|
||||
if not candidates:
|
||||
result['error'] = f'No usenet results found for "{query}"'
|
||||
return result
|
||||
|
||||
picked = pick_best_album_release(candidates, _guess_quality_from_title)
|
||||
if picked is None:
|
||||
result['error'] = 'No suitable NZB candidate after filtering'
|
||||
return result
|
||||
|
||||
logger.info("[Usenet album] Picked '%s' (size=%.1fMB grabs=%s indexer=%s)",
|
||||
picked.title, picked.size / 1_048_576, picked.grabs, picked.indexer_name)
|
||||
_emit('queued', release=picked.title, size=picked.size, grabs=picked.grabs)
|
||||
|
||||
try:
|
||||
job_id = run_async(adapter.add_nzb(picked.download_url))
|
||||
except Exception as e:
|
||||
result['error'] = f'Usenet client refused the NZB: {e}'
|
||||
return result
|
||||
if not job_id:
|
||||
result['error'] = 'Usenet client refused the NZB'
|
||||
return result
|
||||
|
||||
_emit('downloading', release=picked.title)
|
||||
save_path = self._poll_album_download(adapter, job_id, picked.title, _emit)
|
||||
if save_path is None:
|
||||
result['error'] = 'Usenet download failed or timed out'
|
||||
return result
|
||||
|
||||
_emit('staging', release=picked.title)
|
||||
try:
|
||||
audio_files = collect_audio_after_extraction(Path(save_path))
|
||||
except Exception as e:
|
||||
result['error'] = f'Failed to walk audio files: {e}'
|
||||
return result
|
||||
if not audio_files:
|
||||
result['error'] = f'No audio files found in {save_path}'
|
||||
return result
|
||||
|
||||
copied = copy_audio_files_atomically(audio_files, Path(staging_dir))
|
||||
if not copied:
|
||||
result['error'] = 'No audio files copied to staging'
|
||||
return result
|
||||
logger.info("[Usenet album] Staged %d audio files for '%s'", len(copied), album_name)
|
||||
_emit('staged', count=len(copied))
|
||||
result['success'] = True
|
||||
result['files'] = copied
|
||||
return result
|
||||
|
||||
def _poll_album_download(self, adapter, job_id, title, emit) -> Optional[str]:
|
||||
deadline = time.monotonic() + _POLL_TIMEOUT_SECONDS
|
||||
last_save_path: Optional[str] = None
|
||||
while time.monotonic() < deadline:
|
||||
if self.shutdown_check and self.shutdown_check():
|
||||
return None
|
||||
try:
|
||||
status = run_async(adapter.get_status(job_id))
|
||||
except Exception as e:
|
||||
logger.warning("[Usenet album] Poll error: %s", e)
|
||||
status = None
|
||||
if status is None:
|
||||
logger.error("[Usenet album] '%s' disappeared from client", title)
|
||||
return None
|
||||
emit('downloading', progress=status.progress, downloaded=status.downloaded,
|
||||
speed=status.download_speed)
|
||||
if status.save_path:
|
||||
last_save_path = status.save_path
|
||||
if status.state in _COMPLETE_STATES:
|
||||
return last_save_path
|
||||
if status.state == 'failed':
|
||||
logger.error("[Usenet album] '%s' failed: %s", title, status.error)
|
||||
return None
|
||||
time.sleep(_POLL_INTERVAL_SECONDS)
|
||||
logger.error("[Usenet album] '%s' timed out", title)
|
||||
return None
|
||||
191
core/downloads/album_bundle_dispatch.py
Normal file
191
core/downloads/album_bundle_dispatch.py
Normal file
|
|
@ -0,0 +1,191 @@
|
|||
"""Album-bundle dispatch for torrent / usenet single-source downloads.
|
||||
|
||||
Lifted from ``run_full_missing_tracks_process`` so the master
|
||||
worker doesn't carry a 90-line inline branch and so the gate logic
|
||||
can be unit-tested in isolation.
|
||||
|
||||
The gate fires only when ALL conditions hold:
|
||||
|
||||
- Batch is an album-context download (``is_album_download`` flag).
|
||||
- Active download source is ``torrent`` or ``usenet`` (single-source
|
||||
mode — hybrid stays per-track to preserve fallback).
|
||||
- Both album-name and artist-name are populated in batch context.
|
||||
- The resolved plugin exposes ``download_album_to_staging``.
|
||||
|
||||
When the gate engages it runs the plugin synchronously (the master
|
||||
worker is already on a thread-pool executor) and mirrors the
|
||||
plugin's lifecycle payloads into the batch state so the Downloads
|
||||
page can render meaningful progress before per-track tasks exist.
|
||||
|
||||
Return semantics: ``True`` means the gate handled the batch — the
|
||||
master worker should stop and not run per-track analysis. ``False``
|
||||
means the gate didn't engage (or engaged-and-fell-back) — caller
|
||||
continues the normal per-track flow.
|
||||
|
||||
The ``BatchStateAccess`` Protocol exists so this module doesn't
|
||||
import ``download_batches`` from runtime_state directly. The
|
||||
caller (master worker) injects accessors so this module stays
|
||||
testable without touching live runtime state.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from pathlib import Path
|
||||
from typing import Any, Callable, Optional, Protocol
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class BatchStateAccess(Protocol):
|
||||
"""Narrow shim around the batch-state dict ops the dispatch needs.
|
||||
|
||||
Two methods to keep the surface small:
|
||||
- ``update_fields(batch_id, fields)`` — atomic merge into the
|
||||
batch dict under tasks_lock.
|
||||
- ``mark_failed(batch_id, error)`` — convenience for the failure
|
||||
path (sets phase + error + album_bundle_state in one shot).
|
||||
"""
|
||||
|
||||
def update_fields(self, batch_id: str, fields: dict) -> None: ...
|
||||
|
||||
def mark_failed(self, batch_id: str, error: str) -> None: ...
|
||||
|
||||
|
||||
# Fields the album-bundle progress callback may carry. Anything in
|
||||
# this set gets mirrored onto the batch row as ``album_bundle_<key>``
|
||||
# so the Downloads page can render it without coupling to the
|
||||
# specific payload shape.
|
||||
_MIRRORED_KEYS = ('progress', 'release', 'speed', 'downloaded',
|
||||
'size', 'seeders', 'grabs', 'count')
|
||||
|
||||
|
||||
def is_eligible(
|
||||
*,
|
||||
mode: str,
|
||||
is_album: bool,
|
||||
album_name: str,
|
||||
artist_name: str,
|
||||
) -> bool:
|
||||
"""Pure predicate: does this batch even qualify for the album
|
||||
flow? Separate from the resolution+run step so tests can pin
|
||||
the gate logic without standing up a plugin."""
|
||||
if not is_album:
|
||||
return False
|
||||
if (mode or '').lower() not in ('torrent', 'usenet'):
|
||||
return False
|
||||
if not (album_name or '').strip():
|
||||
return False
|
||||
if not (artist_name or '').strip():
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def try_dispatch(
|
||||
*,
|
||||
batch_id: str,
|
||||
is_album: bool,
|
||||
album_context: Optional[dict],
|
||||
artist_context: Optional[dict],
|
||||
config_get: Callable[..., Any],
|
||||
plugin_resolver: Callable[[str], Optional[Any]],
|
||||
state: BatchStateAccess,
|
||||
) -> bool:
|
||||
"""Attempt the album-bundle flow. Returns ``True`` iff the
|
||||
master worker should return early (gate engaged and completed
|
||||
— success OR failure). ``False`` means fall through to the
|
||||
normal per-track flow.
|
||||
|
||||
``config_get`` is a callable shaped like ``config_manager.get``;
|
||||
``plugin_resolver`` resolves a source-name string to an
|
||||
initialised plugin instance (or None); ``state`` is the
|
||||
BatchStateAccess shim. Injecting these keeps the module
|
||||
dependency-light + unit-testable.
|
||||
"""
|
||||
mode = (config_get('download_source.mode', 'soulseek') or 'soulseek').lower()
|
||||
album_name = (album_context or {}).get('name') or ''
|
||||
artist_name = (artist_context or {}).get('name') or ''
|
||||
|
||||
if not is_eligible(mode=mode, is_album=is_album,
|
||||
album_name=album_name, artist_name=artist_name):
|
||||
return False
|
||||
|
||||
album_name = album_name.strip()
|
||||
artist_name = artist_name.strip()
|
||||
|
||||
plugin = None
|
||||
try:
|
||||
plugin = plugin_resolver(mode)
|
||||
except Exception as exc:
|
||||
logger.warning("[Album Bundle] Could not resolve %s plugin: %s", mode, exc)
|
||||
|
||||
if plugin is None or not hasattr(plugin, 'download_album_to_staging'):
|
||||
logger.warning(
|
||||
"[Album Bundle] Gate matched but plugin / context unavailable "
|
||||
"(mode=%s album=%r artist=%r plugin=%s) — falling back to per-track flow",
|
||||
mode, album_name, artist_name,
|
||||
type(plugin).__name__ if plugin else None,
|
||||
)
|
||||
return False
|
||||
|
||||
staging_root = config_get(
|
||||
'download_source.album_bundle_staging_path',
|
||||
'storage/album_bundle_staging',
|
||||
) or 'storage/album_bundle_staging'
|
||||
staging_dir = str(Path(staging_root) / _safe_batch_dirname(batch_id))
|
||||
logger.info(
|
||||
"[Album Bundle] Engaging %s album flow for '%s' by '%s' -> %s",
|
||||
mode, album_name, artist_name, staging_dir,
|
||||
)
|
||||
state.update_fields(batch_id, {
|
||||
'phase': 'album_downloading',
|
||||
'album_bundle_state': 'searching',
|
||||
'album_bundle_source': mode,
|
||||
'album_bundle_staging_path': staging_dir,
|
||||
'album_bundle_private_staging': True,
|
||||
})
|
||||
|
||||
def _emit(payload):
|
||||
"""Mirror plugin lifecycle into batch state for UI rendering."""
|
||||
try:
|
||||
fields = {'album_bundle_state': payload.get('state', '')}
|
||||
for key in _MIRRORED_KEYS:
|
||||
if key in payload:
|
||||
fields[f'album_bundle_{key}'] = payload[key]
|
||||
state.update_fields(batch_id, fields)
|
||||
except Exception as exc:
|
||||
logger.debug("[Album Bundle] emit failed: %s", exc)
|
||||
|
||||
try:
|
||||
outcome = plugin.download_album_to_staging(
|
||||
album_name, artist_name, staging_dir, _emit,
|
||||
)
|
||||
except Exception as exc:
|
||||
logger.exception("[Album Bundle] %s plugin raised: %s", mode, exc)
|
||||
outcome = {'success': False, 'error': f'Plugin error: {exc}'}
|
||||
|
||||
if not outcome.get('success'):
|
||||
err = outcome.get('error', 'Album bundle download failed')
|
||||
logger.error("[Album Bundle] %s flow failed for '%s': %s",
|
||||
mode, album_name, err)
|
||||
state.mark_failed(batch_id, err)
|
||||
return True
|
||||
|
||||
logger.info(
|
||||
"[Album Bundle] %s staged %d files for '%s' — handing off to per-track staging matcher",
|
||||
mode, len(outcome.get('files', [])), album_name,
|
||||
)
|
||||
state.update_fields(batch_id, {
|
||||
'phase': 'analysis',
|
||||
'album_bundle_state': 'staged',
|
||||
})
|
||||
# Engaged-and-succeeded: we DON'T early-return because the
|
||||
# per-track flow needs to run to create + complete the per-track
|
||||
# task rows. Those tasks will hit try_staging_match and pull the
|
||||
# files we just staged.
|
||||
return False
|
||||
|
||||
|
||||
def _safe_batch_dirname(batch_id: str) -> str:
|
||||
safe = ''.join(ch if ch.isalnum() or ch in ('-', '_') else '_' for ch in str(batch_id or 'batch'))
|
||||
return safe or 'batch'
|
||||
|
|
@ -26,9 +26,11 @@ Lifted verbatim from web_server.py. Dependencies injected via
|
|||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import shutil
|
||||
import time
|
||||
import traceback
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Any, Callable, Optional
|
||||
|
||||
from core.downloads.history import record_sync_history_completion
|
||||
|
|
@ -42,6 +44,47 @@ from core.runtime_state import (
|
|||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _safe_batch_dirname(batch_id: str) -> str:
|
||||
safe = ''.join(ch if ch.isalnum() or ch in ('-', '_') else '_' for ch in str(batch_id or 'batch'))
|
||||
return safe or 'batch'
|
||||
|
||||
|
||||
def _cleanup_private_album_bundle_staging(batch_id: str, batch: dict) -> None:
|
||||
"""Best-effort cleanup for torrent/usenet private staging copies.
|
||||
|
||||
The torrent/usenet clients keep their own completed download folders.
|
||||
This only removes SoulSync's per-batch copy under album-bundle staging.
|
||||
"""
|
||||
if not batch.get('album_bundle_private_staging'):
|
||||
return
|
||||
if (batch.get('album_bundle_source') or '').lower() not in ('torrent', 'usenet'):
|
||||
return
|
||||
|
||||
staging_path = batch.get('album_bundle_staging_path')
|
||||
if not staging_path:
|
||||
return
|
||||
|
||||
path = Path(staging_path)
|
||||
expected_name = _safe_batch_dirname(batch_id)
|
||||
if path.name != expected_name:
|
||||
logger.warning(
|
||||
"[Album Bundle] Refusing to clean private staging path with unexpected name: %s",
|
||||
staging_path,
|
||||
)
|
||||
return
|
||||
if not path.exists():
|
||||
return
|
||||
if not path.is_dir():
|
||||
logger.warning("[Album Bundle] Refusing to clean non-directory staging path: %s", staging_path)
|
||||
return
|
||||
|
||||
try:
|
||||
shutil.rmtree(path)
|
||||
logger.info("[Album Bundle] Cleaned private staging folder for batch %s: %s", batch_id, staging_path)
|
||||
except Exception as exc:
|
||||
logger.warning("[Album Bundle] Could not clean private staging folder %s: %s", staging_path, exc)
|
||||
|
||||
|
||||
@dataclass
|
||||
class LifecycleDeps:
|
||||
"""Bundle of cross-cutting deps the batch lifecycle needs."""
|
||||
|
|
@ -398,6 +441,7 @@ def on_download_completed(batch_id: str, task_id: str, success: bool, deps: Life
|
|||
|
||||
logger.info(f"[Batch Manager] Batch {batch_id} complete - stopping monitor")
|
||||
deps.download_monitor.stop_monitoring(batch_id)
|
||||
_cleanup_private_album_bundle_staging(batch_id, batch)
|
||||
|
||||
# M3U REGENERATION: Regenerate M3U with real library paths now that
|
||||
# all post-processing (tagging, moving, DB writes) is complete.
|
||||
|
|
@ -606,6 +650,7 @@ def check_batch_completion_v2(batch_id: str, deps: LifecycleDeps) -> Optional[bo
|
|||
|
||||
logger.info(f"[Completion Check V2] Batch {batch_id} complete - stopping monitor")
|
||||
deps.download_monitor.stop_monitoring(batch_id)
|
||||
_cleanup_private_album_bundle_staging(batch_id, batch)
|
||||
|
||||
# REPAIR: Scan all album folders from this batch for track number issues
|
||||
if deps.repair_worker:
|
||||
|
|
|
|||
|
|
@ -35,6 +35,7 @@ from difflib import SequenceMatcher
|
|||
from pathlib import Path
|
||||
from typing import Any, Callable
|
||||
|
||||
from core.downloads import album_bundle_dispatch as _album_bundle_dispatch
|
||||
from core.runtime_state import download_batches, download_tasks, tasks_lock
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
|
@ -270,6 +271,26 @@ class MasterDeps:
|
|||
reset_wishlist_auto_processing: Callable[[], None]
|
||||
|
||||
|
||||
class _BatchStateAccessImpl:
|
||||
"""Concrete ``BatchStateAccess`` for the runtime ``download_batches``
|
||||
dict — wraps the lock + the existing-batch check so the album-
|
||||
bundle dispatcher stays decoupled from runtime_state."""
|
||||
|
||||
def update_fields(self, batch_id: str, fields: dict) -> None:
|
||||
with tasks_lock:
|
||||
row = download_batches.get(batch_id)
|
||||
if row is not None:
|
||||
row.update(fields)
|
||||
|
||||
def mark_failed(self, batch_id: str, error: str) -> None:
|
||||
with tasks_lock:
|
||||
row = download_batches.get(batch_id)
|
||||
if row is not None:
|
||||
row['phase'] = 'failed'
|
||||
row['error'] = error
|
||||
row['album_bundle_state'] = 'failed'
|
||||
|
||||
|
||||
def run_full_missing_tracks_process(batch_id, playlist_id, tracks_json, deps: MasterDeps):
|
||||
"""
|
||||
A master worker that handles the entire missing tracks process:
|
||||
|
|
@ -311,6 +332,23 @@ def run_full_missing_tracks_process(batch_id, playlist_id, tracks_json, deps: Ma
|
|||
if force_download_all:
|
||||
logger.warning(f"[Force Download] Force download mode enabled for batch {batch_id} - treating all tracks as missing")
|
||||
|
||||
# Album-bundle gate for torrent / usenet single-source mode.
|
||||
# See ``core/downloads/album_bundle_dispatch`` for the full
|
||||
# narrow-gate rationale. Returns True iff the master worker
|
||||
# should stop (gate fired and failed); False = engaged-and-
|
||||
# succeeded OR didn't engage, both fall through to per-track.
|
||||
_bundle_state = _BatchStateAccessImpl()
|
||||
if _album_bundle_dispatch.try_dispatch(
|
||||
batch_id=batch_id,
|
||||
is_album=batch_is_album,
|
||||
album_context=batch_album_context,
|
||||
artist_context=batch_artist_context,
|
||||
config_get=deps.config_manager.get,
|
||||
plugin_resolver=deps.download_orchestrator.client,
|
||||
state=_bundle_state,
|
||||
):
|
||||
return
|
||||
|
||||
# Allow duplicate tracks across albums — when enabled, only skip tracks already
|
||||
# owned in THIS album, not tracks owned in other albums
|
||||
allow_duplicates = deps.config_manager.get('wishlist.allow_duplicate_tracks', True)
|
||||
|
|
|
|||
|
|
@ -34,6 +34,32 @@ _start_next_batch_of_downloads = None
|
|||
_orphaned_download_keys = None
|
||||
missing_download_executor = None
|
||||
download_orchestrator = None
|
||||
_RELEASE_SOURCE_NAMES = frozenset(('torrent', 'usenet'))
|
||||
|
||||
|
||||
def _download_id_key(download_id):
|
||||
return f"download_id::{download_id}" if download_id else None
|
||||
|
||||
|
||||
def _is_release_task(task):
|
||||
ti = task.get('track_info') if isinstance(task.get('track_info'), dict) else {}
|
||||
username = task.get('username') or ti.get('username')
|
||||
return username in _RELEASE_SOURCE_NAMES
|
||||
|
||||
|
||||
def _lookup_live_info(task, live_transfers_lookup):
|
||||
ti = task.get('track_info') if isinstance(task.get('track_info'), dict) else {}
|
||||
download_id = task.get('download_id')
|
||||
if _is_release_task(task):
|
||||
by_id = live_transfers_lookup.get(_download_id_key(download_id))
|
||||
if by_id:
|
||||
return by_id
|
||||
|
||||
task_filename = task.get('filename') or ti.get('filename')
|
||||
task_username = task.get('username') or ti.get('username')
|
||||
if not task_filename or not task_username:
|
||||
return None
|
||||
return live_transfers_lookup.get(_make_context_key(task_username, task_filename))
|
||||
|
||||
|
||||
def init(
|
||||
|
|
@ -139,55 +165,64 @@ class WebUIDownloadMonitor:
|
|||
|
||||
for task_id in download_batches[batch_id].get('queue', []):
|
||||
task = download_tasks.get(task_id)
|
||||
if not task or task['status'] not in ['downloading', 'queued']:
|
||||
if not task:
|
||||
continue
|
||||
release_recoverable = (
|
||||
_is_release_task(task)
|
||||
and task.get('download_id')
|
||||
and task.get('status') in ['failed', 'not_found']
|
||||
)
|
||||
if task['status'] not in ['downloading', 'queued'] and not release_recoverable:
|
||||
continue
|
||||
|
||||
# Check for timeouts and errors - retries handled directly in _should_retry_task
|
||||
# If _should_retry_task returns True, it means retries were exhausted
|
||||
retry_exhausted = self._should_retry_task(task_id, task, live_transfers_lookup, current_time, deferred_ops)
|
||||
retry_exhausted = False
|
||||
if not release_recoverable:
|
||||
retry_exhausted = self._should_retry_task(task_id, task, live_transfers_lookup, current_time, deferred_ops)
|
||||
# Collect exhausted tasks to handle outside lock (prevents deadlock)
|
||||
if retry_exhausted:
|
||||
exhausted_tasks.append((batch_id, task_id))
|
||||
|
||||
# ENHANCED: Check for successful completions (especially YouTube)
|
||||
task_filename = task.get('filename') or task.get('track_info', {}).get('filename')
|
||||
task_username = task.get('username') or task.get('track_info', {}).get('username')
|
||||
# ENHANCED: Check for successful completions (especially YouTube).
|
||||
# Release-style sources can report a completed audio file
|
||||
# name that differs from the original indexer URL/title
|
||||
# stored on the task, so prefer the stable download_id.
|
||||
live_info = _lookup_live_info(task, live_transfers_lookup)
|
||||
|
||||
if task_filename and task_username:
|
||||
lookup_key = _make_context_key(task_username, task_filename)
|
||||
live_info = live_transfers_lookup.get(lookup_key)
|
||||
|
||||
if live_info:
|
||||
state = live_info.get('state', '')
|
||||
# Trigger post-processing if download is completed successfully
|
||||
# slskd uses compound states like 'Completed, Succeeded' - use substring matching
|
||||
# Must exclude error states first (matching _build_batch_status_data's prioritized checking)
|
||||
has_error = ('Errored' in state or 'Failed' in state or 'Rejected' in state or 'TimedOut' in state)
|
||||
has_completion = ('Completed' in state or 'Succeeded' in state)
|
||||
# Verify bytes actually transferred before trusting state string.
|
||||
# slskd can report "Completed" before the full file is flushed to disk,
|
||||
# or on connection drops that leave a partial file.
|
||||
if has_completion and not has_error:
|
||||
expected_size = live_info.get('size', 0)
|
||||
transferred = live_info.get('bytesTransferred', 0)
|
||||
if expected_size > 0 and transferred < expected_size:
|
||||
if not task.get('_incomplete_warned'):
|
||||
logger.debug(f"Monitor: {task_id} state={state} but bytes incomplete ({transferred}/{expected_size}) — waiting")
|
||||
task['_incomplete_warned'] = True
|
||||
continue
|
||||
if has_completion and not has_error and task['status'] == 'downloading':
|
||||
task.pop('_incomplete_warned', None)
|
||||
# CRITICAL FIX: Transition to 'post_processing' HERE so downloads
|
||||
# don't depend on browser polling to trigger post-processing.
|
||||
# Previously, post-processing was only submitted by _build_batch_status_data
|
||||
# (called from browser-polled endpoints), meaning closing the browser
|
||||
# left tasks stuck in 'downloading' forever.
|
||||
task['status'] = 'post_processing'
|
||||
task['status_change_time'] = current_time
|
||||
logger.info(f"Monitor detected completed download for {task_id} ({state}) - submitting post-processing")
|
||||
# Collect for handling outside the lock to prevent deadlock.
|
||||
# _on_download_completed acquires tasks_lock which is non-reentrant.
|
||||
completed_tasks.append((batch_id, task_id))
|
||||
if live_info:
|
||||
state = live_info.get('state', '')
|
||||
# Trigger post-processing if download is completed successfully
|
||||
# slskd uses compound states like 'Completed, Succeeded' - use substring matching
|
||||
# Must exclude error states first (matching _build_batch_status_data's prioritized checking)
|
||||
has_error = ('Errored' in state or 'Failed' in state or 'Rejected' in state or 'TimedOut' in state)
|
||||
has_completion = ('Completed' in state or 'Succeeded' in state)
|
||||
# Verify bytes actually transferred before trusting state string.
|
||||
# slskd can report "Completed" before the full file is flushed to disk,
|
||||
# or on connection drops that leave a partial file.
|
||||
if has_completion and not has_error:
|
||||
expected_size = live_info.get('size', 0)
|
||||
transferred = live_info.get('bytesTransferred', 0)
|
||||
if expected_size > 0 and transferred < expected_size:
|
||||
if not task.get('_incomplete_warned'):
|
||||
logger.debug("Monitor: %s state=%s but bytes incomplete (%s/%s) - waiting", task_id, state, transferred, expected_size)
|
||||
task['_incomplete_warned'] = True
|
||||
continue
|
||||
if has_completion and not has_error and (
|
||||
task['status'] == 'downloading' or release_recoverable
|
||||
):
|
||||
task.pop('_incomplete_warned', None)
|
||||
# CRITICAL FIX: Transition to 'post_processing' HERE so downloads
|
||||
# don't depend on browser polling to trigger post-processing.
|
||||
# Previously, post-processing was only submitted by _build_batch_status_data
|
||||
# (called from browser-polled endpoints), meaning closing the browser
|
||||
# left tasks stuck in 'downloading' forever.
|
||||
task['status'] = 'post_processing'
|
||||
task['status_change_time'] = current_time
|
||||
logger.info(f"Monitor detected completed download for {task_id} ({state}) - submitting post-processing")
|
||||
# Collect for handling outside the lock to prevent deadlock.
|
||||
# _on_download_completed acquires tasks_lock which is non-reentrant.
|
||||
completed_tasks.append((batch_id, task_id))
|
||||
|
||||
# ---- All work below runs WITHOUT tasks_lock held ----
|
||||
if globals().get('IS_SHUTTING_DOWN', False) or not self.monitoring:
|
||||
|
|
@ -308,7 +343,7 @@ class WebUIDownloadMonitor:
|
|||
for download in all_downloads:
|
||||
key = _make_context_key(download.username, download.filename)
|
||||
# Convert DownloadStatus to transfer dict format for monitor compatibility
|
||||
live_transfers[key] = {
|
||||
transfer_row = {
|
||||
'id': download.id,
|
||||
'filename': download.filename,
|
||||
'username': download.username,
|
||||
|
|
@ -318,6 +353,10 @@ class WebUIDownloadMonitor:
|
|||
'bytesTransferred': download.transferred,
|
||||
'averageSpeed': download.speed,
|
||||
}
|
||||
live_transfers[key] = transfer_row
|
||||
id_key = _download_id_key(download.id)
|
||||
if id_key:
|
||||
live_transfers[id_key] = transfer_row
|
||||
except Exception as yt_error:
|
||||
logger.error(f"Monitor: Could not fetch streaming source downloads: {yt_error}")
|
||||
|
||||
|
|
@ -352,7 +391,7 @@ class WebUIDownloadMonitor:
|
|||
return False
|
||||
|
||||
lookup_key = _make_context_key(task_username, task_filename)
|
||||
live_info = live_transfers_lookup.get(lookup_key)
|
||||
live_info = _lookup_live_info(task, live_transfers_lookup)
|
||||
|
||||
if not live_info:
|
||||
# User-initiated manual pick — skip auto-retry. The status
|
||||
|
|
|
|||
|
|
@ -19,9 +19,11 @@ from __future__ import annotations
|
|||
|
||||
import logging
|
||||
import os
|
||||
import shutil
|
||||
import time
|
||||
import traceback
|
||||
from dataclasses import dataclass
|
||||
from difflib import SequenceMatcher
|
||||
from pathlib import Path
|
||||
from typing import Any, Callable, Optional
|
||||
|
||||
|
|
@ -34,7 +36,7 @@ from core.imports.context import (
|
|||
get_import_original_search,
|
||||
normalize_import_context,
|
||||
)
|
||||
from core.imports.filename import extract_track_number_from_filename
|
||||
from core.imports.filename import extract_track_number_from_filename, parse_filename_metadata
|
||||
from core.metadata import enrichment as metadata_enrichment
|
||||
from core.runtime_state import (
|
||||
download_tasks,
|
||||
|
|
@ -46,6 +48,85 @@ from core.runtime_state import (
|
|||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
_AUDIO_EXTENSIONS = {
|
||||
'.flac', '.ape', '.wav', '.alac', '.dsf', '.dff', '.aiff', '.aif',
|
||||
'.opus', '.ogg', '.m4a', '.aac', '.mp3', '.wma',
|
||||
}
|
||||
|
||||
|
||||
def _is_audio_file(path: str) -> bool:
|
||||
return Path(str(path or '')).suffix.lower() in _AUDIO_EXTENSIONS
|
||||
|
||||
|
||||
def _reject_non_audio_found_file(found_file: Optional[str], file_location: Optional[str]) -> tuple[Optional[str], Optional[str]]:
|
||||
if found_file and not _is_audio_file(found_file):
|
||||
logger.warning(
|
||||
"[Post-Processing] Ignoring non-audio candidate found during file search: %s",
|
||||
found_file,
|
||||
)
|
||||
return None, None
|
||||
return found_file, file_location
|
||||
|
||||
|
||||
def _normalize_match_text(value: str) -> str:
|
||||
return ''.join(ch.lower() for ch in str(value or '') if ch.isalnum())
|
||||
|
||||
|
||||
def _release_audio_match_score(path: str, expected_title: str, expected_artist: str) -> float:
|
||||
parsed = parse_filename_metadata(path)
|
||||
parsed_title = parsed.get('title') or Path(path).stem
|
||||
parsed_artist = parsed.get('artist') or ''
|
||||
expected_title_norm = _normalize_match_text(expected_title)
|
||||
parsed_title_norm = _normalize_match_text(parsed_title)
|
||||
if expected_title_norm and (
|
||||
expected_title_norm in parsed_title_norm or parsed_title_norm in expected_title_norm
|
||||
):
|
||||
title_score = 1.0
|
||||
else:
|
||||
title_score = SequenceMatcher(
|
||||
None,
|
||||
expected_title_norm,
|
||||
parsed_title_norm,
|
||||
).ratio()
|
||||
if expected_artist and parsed_artist:
|
||||
artist_score = SequenceMatcher(
|
||||
None,
|
||||
_normalize_match_text(expected_artist),
|
||||
_normalize_match_text(parsed_artist),
|
||||
).ratio()
|
||||
return (title_score * 0.75) + (artist_score * 0.25)
|
||||
return title_score
|
||||
|
||||
|
||||
def _track_title_from_task(track_info: Any, context: Optional[dict]) -> str:
|
||||
if isinstance(track_info, dict):
|
||||
title = track_info.get('name') or track_info.get('title')
|
||||
if title:
|
||||
return str(title)
|
||||
return get_import_clean_title(context or {}, default='')
|
||||
|
||||
|
||||
def _copy_release_audio_to_transfer(source_path: str, transfer_dir: str) -> Optional[str]:
|
||||
try:
|
||||
src = Path(source_path)
|
||||
if not src.exists() or not src.is_file():
|
||||
return None
|
||||
dest_dir = Path(transfer_dir)
|
||||
dest_dir.mkdir(parents=True, exist_ok=True)
|
||||
dest = dest_dir / src.name
|
||||
if dest.exists():
|
||||
stem, suffix = src.stem, src.suffix
|
||||
counter = 1
|
||||
while dest.exists():
|
||||
dest = dest_dir / f"{stem}_release_{counter}{suffix}"
|
||||
counter += 1
|
||||
shutil.copy2(src, dest)
|
||||
return str(dest)
|
||||
except Exception as exc:
|
||||
logger.warning("[Post-Processing] Could not copy release audio to transfer: %s", exc)
|
||||
return None
|
||||
|
||||
|
||||
@dataclass
|
||||
class PostProcessDeps:
|
||||
"""Bundle of dependencies the post-processing worker needs.
|
||||
|
|
@ -190,14 +271,71 @@ def run_post_processing_worker(task_id: str, batch_id: str, deps: PostProcessDep
|
|||
|
||||
# CRITICAL FIX: For YouTube downloads, the filename in task is 'id||title' (metadata),
|
||||
# but the actual file on disk is 'Title.mp3'. We must ask the client for the real path.
|
||||
# Torrent/usenet also use opaque "url||display" filenames. Their completed
|
||||
# job may contain a full release, so choose the best matching audio file
|
||||
# and copy it to transfer before importing instead of moving the client's
|
||||
# original completed download.
|
||||
if (task.get('username') == 'youtube' or '||' in str(task_filename)) and not found_file:
|
||||
logger.info(f"[Post-Processing] Detected YouTube download task: {task_id}")
|
||||
logger.info(f"[Post-Processing] Detected engine-backed download task: {task_id}")
|
||||
try:
|
||||
# Query the download orchestrator for the status which contains the real file path
|
||||
# CRITICAL FIX: Use the actual download_id designated by the client, not the internal task_id
|
||||
actual_download_id = task.get('download_id') or task_id
|
||||
status = deps.run_async(deps.download_orchestrator.get_download_status(actual_download_id))
|
||||
if status and status.file_path:
|
||||
if status and task.get('username') in ('torrent', 'usenet'):
|
||||
audio_files = list(getattr(status, 'audio_files', None) or [])
|
||||
if not audio_files and getattr(status, 'file_path', None):
|
||||
audio_files = [status.file_path]
|
||||
expected_title = _track_title_from_task(track_info, context)
|
||||
expected_artist = ''
|
||||
if isinstance(track_info, dict):
|
||||
artists = track_info.get('artists', [])
|
||||
if isinstance(artists, list) and artists:
|
||||
first_artist = artists[0]
|
||||
expected_artist = first_artist.get('name', '') if isinstance(first_artist, dict) else str(first_artist)
|
||||
elif isinstance(artists, str):
|
||||
expected_artist = artists
|
||||
if not expected_artist and context:
|
||||
artist_ctx = get_import_context_artist(context)
|
||||
expected_artist = artist_ctx.get('name', '') if isinstance(artist_ctx, dict) else ''
|
||||
|
||||
scored_files = [
|
||||
(_release_audio_match_score(path, expected_title, expected_artist), path)
|
||||
for path in audio_files
|
||||
if _is_audio_file(path)
|
||||
]
|
||||
scored_files.sort(reverse=True)
|
||||
if scored_files:
|
||||
best_score, best_path = scored_files[0]
|
||||
logger.info(
|
||||
"[Post-Processing] Best %s release file for '%s': %s (score %.2f)",
|
||||
task.get('username'), expected_title, best_path, best_score,
|
||||
)
|
||||
if best_score >= 0.80:
|
||||
copied_path = _copy_release_audio_to_transfer(best_path, transfer_dir)
|
||||
if copied_path:
|
||||
found_file = copied_path
|
||||
file_location = 'download'
|
||||
logger.info(
|
||||
"[Post-Processing] Copied matched %s release file to transfer: %s",
|
||||
task.get('username'), copied_path,
|
||||
)
|
||||
else:
|
||||
logger.warning(
|
||||
"[Post-Processing] No %s release file met match threshold for '%s' (best %.2f)",
|
||||
task.get('username'), expected_title, best_score,
|
||||
)
|
||||
if not found_file:
|
||||
with tasks_lock:
|
||||
if task_id in download_tasks:
|
||||
download_tasks[task_id]['status'] = 'failed'
|
||||
download_tasks[task_id]['error_message'] = (
|
||||
f"No matching audio file found in {task.get('username')} release"
|
||||
)
|
||||
deps.on_download_completed(batch_id, task_id, False)
|
||||
return
|
||||
|
||||
if status and status.file_path and not found_file and task.get('username') not in ('torrent', 'usenet'):
|
||||
real_path = status.file_path
|
||||
if os.path.exists(real_path):
|
||||
# Determine if it's in download or transfer directory
|
||||
|
|
@ -227,11 +365,12 @@ def run_post_processing_worker(task_id: str, batch_id: str, deps: PostProcessDep
|
|||
|
||||
logger.info(f"[Post-Processing] Resolved actual YouTube filename: {found_file} (Location: {file_location})")
|
||||
else:
|
||||
logger.warning(f"[Post-Processing] YouTube status reported path but file missing: {real_path}")
|
||||
logger.warning(f"[Post-Processing] Engine status reported path but file missing: {real_path}")
|
||||
else:
|
||||
logger.warning(f"[Post-Processing] YouTube status returned no file_path for task {task_id}")
|
||||
if not found_file:
|
||||
logger.warning(f"[Post-Processing] Engine status returned no file_path for task {task_id}")
|
||||
except Exception as e:
|
||||
logger.error(f"[Post-Processing] Failed to retrieve YouTube task status: {e}")
|
||||
logger.error(f"[Post-Processing] Failed to retrieve engine task status: {e}")
|
||||
|
||||
_file_search_max_retries = 5
|
||||
for retry_count in range(_file_search_max_retries):
|
||||
|
|
@ -257,6 +396,7 @@ def run_post_processing_worker(task_id: str, batch_id: str, deps: PostProcessDep
|
|||
# Strategy 1: Try with original filename in both downloads and transfer
|
||||
logger.info("[Post-Processing] Strategy 1: Searching with original filename...")
|
||||
found_file, file_location = deps.find_completed_file(download_dir, task_filename, transfer_dir)
|
||||
found_file, file_location = _reject_non_audio_found_file(found_file, file_location)
|
||||
|
||||
if found_file:
|
||||
logger.info(f"[Post-Processing] Strategy 1 SUCCESS: Found file with original filename in {file_location}: {found_file}")
|
||||
|
|
@ -269,7 +409,11 @@ def run_post_processing_worker(task_id: str, batch_id: str, deps: PostProcessDep
|
|||
found_result = deps.find_completed_file(transfer_dir, expected_final_filename)
|
||||
if found_result and found_result[0]:
|
||||
found_file, file_location = found_result[0], 'transfer'
|
||||
logger.info(f"[Post-Processing] Strategy 2 SUCCESS: Found file with expected final filename: {found_file}")
|
||||
found_file, file_location = _reject_non_audio_found_file(found_file, file_location)
|
||||
if found_file:
|
||||
logger.info(f"[Post-Processing] Strategy 2 SUCCESS: Found file with expected final filename: {found_file}")
|
||||
else:
|
||||
logger.error("[Post-Processing] Strategy 2 FAILED: Expected final filename resolved to a non-audio file")
|
||||
else:
|
||||
logger.error("[Post-Processing] Strategy 2 FAILED: Expected final filename not found in transfer folder")
|
||||
elif not expected_final_filename:
|
||||
|
|
|
|||
|
|
@ -34,9 +34,12 @@ from __future__ import annotations
|
|||
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
from dataclasses import dataclass
|
||||
from typing import Any, Callable
|
||||
|
||||
from core.imports.filename import extract_track_number_from_filename
|
||||
|
||||
# `shutil` and `SequenceMatcher` are imported inline inside try_staging_match()
|
||||
# to keep the lift byte-identical with the original web_server.py function body.
|
||||
|
||||
|
|
@ -50,6 +53,56 @@ from core.runtime_state import (
|
|||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _coerce_positive_int(value: Any, default: int = 0) -> int:
|
||||
try:
|
||||
coerced = int(str(value).split('/')[0])
|
||||
except (TypeError, ValueError):
|
||||
return default
|
||||
return coerced if coerced > 0 else default
|
||||
|
||||
|
||||
def _staging_title_variants(title: Any, normalize: Callable[[str], str]) -> list[str]:
|
||||
"""Return conservative title variants for release-file matching.
|
||||
|
||||
Torrent / usenet release files often encode featured artists in the
|
||||
filename/title while streaming metadata keeps them in the artist credit.
|
||||
Strip only feature/bonus noise here; keep version words like remix,
|
||||
extended, live, acoustic, etc. so distinct recordings do not collapse.
|
||||
"""
|
||||
raw = str(title or '').strip()
|
||||
if not raw:
|
||||
return []
|
||||
|
||||
compacted_separators = re.sub(r'[_]+', ' ', raw)
|
||||
compacted_separators = re.sub(r'\s+', ' ', compacted_separators).strip()
|
||||
|
||||
without_feat = re.sub(
|
||||
r'\s*[\(\[]\s*(?:feat\.?|ft\.?|featuring)\s+[^)\]]*[\)\]]',
|
||||
'',
|
||||
compacted_separators,
|
||||
flags=re.IGNORECASE,
|
||||
)
|
||||
without_feat = re.sub(
|
||||
r'\s+(?:feat\.?|ft\.?|featuring)\s+.*$',
|
||||
'',
|
||||
without_feat,
|
||||
flags=re.IGNORECASE,
|
||||
)
|
||||
without_bonus = re.sub(
|
||||
r'\s*[\(\[]\s*bonus\s+track\s*[\)\]]',
|
||||
'',
|
||||
without_feat,
|
||||
flags=re.IGNORECASE,
|
||||
)
|
||||
|
||||
variants: list[str] = []
|
||||
for candidate in (raw, compacted_separators, without_feat, without_bonus):
|
||||
normalized = normalize(candidate)
|
||||
if normalized and normalized not in variants:
|
||||
variants.append(normalized)
|
||||
return variants
|
||||
|
||||
|
||||
@dataclass
|
||||
class StagingDeps:
|
||||
"""Bundle of cross-cutting deps the staging-match helper needs."""
|
||||
|
|
@ -58,6 +111,12 @@ class StagingDeps:
|
|||
get_staging_file_cache: Callable[[str], list]
|
||||
docker_resolve_path: Callable[[str], str]
|
||||
post_process_matched_download_with_verification: Callable
|
||||
# Optional batch-field accessor. Returns ``download_batches[batch_id].get(field)``
|
||||
# when the runtime state is available, ``None`` otherwise. Injected so
|
||||
# this module doesn't have to import from runtime_state directly —
|
||||
# keeps the dep surface explicit and the function unit-testable
|
||||
# without a live batch dict.
|
||||
get_batch_field: Callable[[str, str], Any] = None # type: ignore[assignment]
|
||||
|
||||
|
||||
def try_staging_match(task_id, batch_id, track, deps: StagingDeps):
|
||||
|
|
@ -80,19 +139,24 @@ def try_staging_match(task_id, batch_id, track, deps: StagingDeps):
|
|||
normalize = deps.matching_engine.normalize_string
|
||||
norm_title = normalize(track_title)
|
||||
norm_artist = normalize(track_artist)
|
||||
title_variants = _staging_title_variants(track_title, normalize) or [norm_title]
|
||||
|
||||
best_match = None
|
||||
best_score = 0.0
|
||||
|
||||
for sf in staging_files:
|
||||
sf_norm_title = normalize(sf['title'])
|
||||
sf_title_variants = _staging_title_variants(sf['title'], normalize)
|
||||
sf_norm_artist = normalize(sf['artist'])
|
||||
|
||||
if not sf_norm_title:
|
||||
if not sf_title_variants:
|
||||
continue
|
||||
|
||||
# Title similarity (primary)
|
||||
title_sim = SequenceMatcher(None, norm_title, sf_norm_title).ratio()
|
||||
title_sim = max(
|
||||
SequenceMatcher(None, expected, candidate).ratio()
|
||||
for expected in title_variants
|
||||
for candidate in sf_title_variants
|
||||
)
|
||||
if title_sim < 0.80:
|
||||
continue
|
||||
|
||||
|
|
@ -141,20 +205,51 @@ def try_staging_match(task_id, batch_id, track, deps: StagingDeps):
|
|||
shutil.copy2(best_match['full_path'], dest_path)
|
||||
logger.info(f"[Staging] Copied to transfer: {dest_path}")
|
||||
|
||||
# Mark task as completed with staging context
|
||||
# Mark task as completed with staging context.
|
||||
# If the batch was populated by the torrent / usenet album-bundle
|
||||
# flow, prefer that provenance label over generic 'staging' so the
|
||||
# download history reflects the real source. The accessor is
|
||||
# injected via StagingDeps so this module doesn't reach into
|
||||
# runtime_state directly (see deps.get_batch_field docstring).
|
||||
_provenance_override = None
|
||||
if batch_id and deps.get_batch_field is not None:
|
||||
try:
|
||||
_provenance_override = deps.get_batch_field(batch_id, 'album_bundle_source')
|
||||
except Exception as _exc:
|
||||
logger.debug("get_batch_field failed: %s", _exc)
|
||||
_provenance_username = _provenance_override or 'staging'
|
||||
_private_album_bundle_staging = False
|
||||
if batch_id and deps.get_batch_field is not None:
|
||||
try:
|
||||
_private_album_bundle_staging = bool(
|
||||
deps.get_batch_field(batch_id, 'album_bundle_private_staging')
|
||||
)
|
||||
except Exception as _exc:
|
||||
logger.debug("get_batch_field failed: %s", _exc)
|
||||
with tasks_lock:
|
||||
if task_id in download_tasks:
|
||||
download_tasks[task_id]['status'] = 'post_processing'
|
||||
download_tasks[task_id]['filename'] = dest_path
|
||||
download_tasks[task_id]['username'] = 'staging'
|
||||
download_tasks[task_id]['username'] = _provenance_username
|
||||
download_tasks[task_id]['staging_match'] = True
|
||||
|
||||
if _private_album_bundle_staging:
|
||||
try:
|
||||
os.remove(best_match['full_path'])
|
||||
logger.debug("[Staging] Removed private album-bundle staging file: %s", best_match['full_path'])
|
||||
except FileNotFoundError:
|
||||
pass
|
||||
except Exception as _exc:
|
||||
logger.debug("[Staging] Could not remove private album-bundle staging file: %s", _exc)
|
||||
|
||||
# Run post-processing (tagging, AcoustID verification, path building)
|
||||
context_key = f"staging_{task_id}"
|
||||
with tasks_lock:
|
||||
track_info = download_tasks.get(task_id, {}).get('track_info', {})
|
||||
if not isinstance(track_info, dict):
|
||||
track_info = {}
|
||||
else:
|
||||
track_info = dict(track_info)
|
||||
|
||||
# Build spotify_artist / spotify_album context so post-processing can apply
|
||||
# the path template. Without these, _post_process_matched_download returns
|
||||
|
|
@ -223,20 +318,37 @@ def try_staging_match(task_id, batch_id, track, deps: StagingDeps):
|
|||
)
|
||||
has_clean_data = bool(track_title and track_artist and track_album_name)
|
||||
|
||||
track_number = (
|
||||
track_info.get('track_number', 0) or
|
||||
getattr(track, 'track_number', 0) or 0
|
||||
)
|
||||
disc_number = (
|
||||
track_info.get('disc_number', 1) or
|
||||
getattr(track, 'disc_number', 1) or 1
|
||||
file_track_number = (
|
||||
_coerce_positive_int(best_match.get('track_number'), 0) or
|
||||
extract_track_number_from_filename(best_match.get('full_path', ''))
|
||||
)
|
||||
file_disc_number = _coerce_positive_int(best_match.get('disc_number'), 1)
|
||||
if _private_album_bundle_staging:
|
||||
track_number = file_track_number
|
||||
disc_number = file_disc_number
|
||||
else:
|
||||
track_number = (
|
||||
_coerce_positive_int(track_info.get('track_number'), 0) or
|
||||
_coerce_positive_int(track_info.get('trackNumber'), 0) or
|
||||
_coerce_positive_int(getattr(track, 'track_number', 0), 0) or
|
||||
file_track_number
|
||||
)
|
||||
disc_number = (
|
||||
_coerce_positive_int(track_info.get('disc_number'), 0) or
|
||||
_coerce_positive_int(track_info.get('discNumber'), 0) or
|
||||
_coerce_positive_int(getattr(track, 'disc_number', 0), 0) or
|
||||
file_disc_number
|
||||
)
|
||||
track_info['track_number'] = track_number
|
||||
track_info['disc_number'] = disc_number
|
||||
|
||||
context = {
|
||||
'track_info': track_info,
|
||||
'spotify_artist': spotify_artist_ctx,
|
||||
'spotify_album': spotify_album_ctx,
|
||||
'original_search_result': {
|
||||
'username': _provenance_username,
|
||||
'filename': best_match.get('full_path', ''),
|
||||
'title': track_title,
|
||||
'artist': track_artist,
|
||||
'spotify_clean_title': track_title,
|
||||
|
|
|
|||
|
|
@ -88,7 +88,9 @@ class StatusDeps:
|
|||
# slskd's live_transfers path and must NOT hit the engine fallback.
|
||||
_STREAMING_SOURCE_NAMES = frozenset((
|
||||
'youtube', 'tidal', 'qobuz', 'hifi', 'deezer_dl', 'lidarr', 'soundcloud', 'amazon',
|
||||
'torrent', 'usenet',
|
||||
))
|
||||
_RELEASE_SOURCE_NAMES = frozenset(('torrent', 'usenet'))
|
||||
|
||||
# Keep these in sync with the engine plugins' state strings.
|
||||
_ENGINE_FAILURE_STATES = ('Errored', 'Failed', 'Rejected', 'TimedOut', 'Aborted')
|
||||
|
|
@ -136,21 +138,12 @@ def _apply_engine_state_fallback(
|
|||
|
||||
Mutates ``task`` in place (status / error_message) the same way the
|
||||
Soulseek branch does, so the next status poll sees the new state.
|
||||
Submits post-processing on terminal success and fires
|
||||
``on_download_completed`` on terminal failure to free the worker
|
||||
slot.
|
||||
Submits post-processing on terminal success. Manual-pick failures
|
||||
are completed here; automatic failures keep the existing retry path
|
||||
in charge.
|
||||
"""
|
||||
if deps.download_orchestrator is None or deps.run_async is None:
|
||||
return
|
||||
if task.get('status') in ('completed', 'failed', 'cancelled', 'not_found', 'post_processing'):
|
||||
return
|
||||
# Scope this fallback to user-initiated manual picks. Auto attempts
|
||||
# already flow through the live_transfers_lookup IF branch (the engine
|
||||
# pre-populates non-Soulseek records via get_all_downloads), and on
|
||||
# failure the monitor's existing retry path picks the next candidate.
|
||||
# Marking auto attempts failed here would short-circuit that fallback.
|
||||
if not task.get('_user_manual_pick'):
|
||||
return
|
||||
download_id = task.get('download_id')
|
||||
if not download_id:
|
||||
return
|
||||
|
|
@ -158,6 +151,17 @@ def _apply_engine_state_fallback(
|
|||
username = task.get('username') or ti.get('username')
|
||||
if username not in _STREAMING_SOURCE_NAMES:
|
||||
return
|
||||
manual_pick = bool(task.get('_user_manual_pick'))
|
||||
if not manual_pick and username not in _RELEASE_SOURCE_NAMES:
|
||||
return
|
||||
release_source = username in _RELEASE_SOURCE_NAMES
|
||||
terminal_status = task.get('status') in ('completed', 'failed', 'cancelled', 'not_found', 'post_processing')
|
||||
if terminal_status and not (
|
||||
release_source
|
||||
and task.get('status') in ('failed', 'not_found')
|
||||
and download_id
|
||||
):
|
||||
return
|
||||
|
||||
try:
|
||||
record = deps.run_async(
|
||||
|
|
@ -189,6 +193,14 @@ def _apply_engine_state_fallback(
|
|||
return
|
||||
|
||||
if any(s in state_str for s in _ENGINE_FAILURE_STATES):
|
||||
if not manual_pick:
|
||||
task_status['status'] = task.get('status', 'downloading')
|
||||
task_status['progress'] = _engine_progress_pct(record)
|
||||
logger.info(
|
||||
"[Engine Fallback] Task %s engine reports '%s' for auto %s attempt; leaving retry handling to monitor",
|
||||
task_id, state_str, username,
|
||||
)
|
||||
return
|
||||
if task['status'] != 'failed':
|
||||
task['status'] = 'failed'
|
||||
err = getattr(record, 'error_message', None) or getattr(record, 'error', None) or ''
|
||||
|
|
@ -247,6 +259,9 @@ def build_batch_status_data(batch_id: str, batch: dict, live_transfers_lookup: d
|
|||
"playlist_id": batch.get('playlist_id'), # Include playlist_id for rehydration
|
||||
"playlist_name": batch.get('playlist_name'), # Include playlist_name for reference
|
||||
}
|
||||
album_bundle = _build_album_bundle_status(batch)
|
||||
if album_bundle:
|
||||
response_data['album_bundle'] = album_bundle
|
||||
|
||||
if response_data["phase"] == 'analysis':
|
||||
response_data['analysis_progress'] = {
|
||||
|
|
@ -318,6 +333,17 @@ def build_batch_status_data(batch_id: str, batch: dict, live_transfers_lookup: d
|
|||
_ti = task.get('track_info') if isinstance(task.get('track_info'), dict) else {}
|
||||
task_filename = task.get('filename') or _ti.get('filename')
|
||||
task_username = task.get('username') or _ti.get('username')
|
||||
if (
|
||||
task_username in _RELEASE_SOURCE_NAMES
|
||||
and task['status'] not in ['completed', 'cancelled', 'post_processing']
|
||||
and task.get('download_id')
|
||||
):
|
||||
_apply_engine_state_fallback(
|
||||
task_id, task, task_status, batch_id, deps,
|
||||
)
|
||||
batch_tasks.append(task_status)
|
||||
continue
|
||||
|
||||
if task_filename and task_username:
|
||||
lookup_key = deps.make_context_key(task_username, task_filename)
|
||||
|
||||
|
|
@ -610,7 +636,7 @@ def build_unified_downloads_response(limit: int, deps: StatusDeps) -> dict:
|
|||
for bid, batch in download_batches.items():
|
||||
queue = batch.get('queue', [])
|
||||
statuses = [download_tasks[tid]['status'] for tid in queue if tid in download_tasks]
|
||||
batch_summaries.append({
|
||||
summary = {
|
||||
'batch_id': bid,
|
||||
'playlist_id': batch.get('playlist_id', ''),
|
||||
'batch_name': batch.get('playlist_name') or batch.get('album_name') or '',
|
||||
|
|
@ -621,7 +647,11 @@ def build_unified_downloads_response(limit: int, deps: StatusDeps) -> dict:
|
|||
'failed': sum(1 for s in statuses if s in ('failed', 'not_found', 'cancelled')),
|
||||
'active': sum(1 for s in statuses if s in ('downloading', 'searching', 'post_processing')),
|
||||
'queued': sum(1 for s in statuses if s in ('queued', 'pending')),
|
||||
})
|
||||
}
|
||||
album_bundle = _build_album_bundle_status(batch)
|
||||
if album_bundle:
|
||||
summary['album_bundle'] = album_bundle
|
||||
batch_summaries.append(summary)
|
||||
|
||||
return {
|
||||
'success': True,
|
||||
|
|
@ -630,3 +660,38 @@ def build_unified_downloads_response(limit: int, deps: StatusDeps) -> dict:
|
|||
'batches': batch_summaries,
|
||||
'timestamp': time.time(),
|
||||
}
|
||||
|
||||
|
||||
def _build_album_bundle_status(batch: dict) -> dict:
|
||||
"""Return public batch-level status for torrent/Usenet album-bundle work."""
|
||||
state = batch.get('album_bundle_state')
|
||||
if not state:
|
||||
return {}
|
||||
|
||||
status = {
|
||||
'state': state,
|
||||
'source': batch.get('album_bundle_source'),
|
||||
'release': batch.get('album_bundle_release'),
|
||||
'progress': batch.get('album_bundle_progress'),
|
||||
'progress_percent': _album_bundle_progress_percent(
|
||||
batch.get('album_bundle_progress')
|
||||
),
|
||||
'speed': batch.get('album_bundle_speed'),
|
||||
'downloaded': batch.get('album_bundle_downloaded'),
|
||||
'size': batch.get('album_bundle_size'),
|
||||
'seeders': batch.get('album_bundle_seeders'),
|
||||
'grabs': batch.get('album_bundle_grabs'),
|
||||
'count': batch.get('album_bundle_count'),
|
||||
}
|
||||
return {key: value for key, value in status.items() if value is not None}
|
||||
|
||||
|
||||
def _album_bundle_progress_percent(value: Any) -> int:
|
||||
try:
|
||||
progress = float(value)
|
||||
except (TypeError, ValueError):
|
||||
return 0
|
||||
|
||||
if progress <= 1:
|
||||
progress *= 100
|
||||
return max(0, min(100, int(round(progress))))
|
||||
|
|
|
|||
|
|
@ -25,12 +25,41 @@ import traceback
|
|||
from dataclasses import dataclass
|
||||
from typing import Any, Callable, Optional
|
||||
|
||||
from core.runtime_state import download_tasks, tasks_lock
|
||||
from core.runtime_state import download_batches, download_tasks, tasks_lock
|
||||
from core.spotify_client import Track as SpotifyTrack
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _private_album_bundle_staging_miss_reason(batch_id: Optional[str], deps: Any) -> Optional[str]:
|
||||
"""Return a user-facing miss reason when per-track search should stop.
|
||||
|
||||
Torrent / usenet album batches first download one private staged release,
|
||||
then each track claims the matching staged file. If that claim fails after
|
||||
the release is already staged, falling through to the normal per-track
|
||||
search only retries release-level sources N times and can keep re-adding
|
||||
the same torrent. Treat the staged release as authoritative for this pass.
|
||||
"""
|
||||
if not batch_id:
|
||||
return None
|
||||
|
||||
batch = download_batches.get(batch_id)
|
||||
if not isinstance(batch, dict):
|
||||
return None
|
||||
|
||||
source = (batch.get('album_bundle_source') or '').lower()
|
||||
mode = (getattr(deps.download_orchestrator, 'mode', '') or '').lower()
|
||||
if (
|
||||
batch.get('album_bundle_private_staging')
|
||||
and batch.get('album_bundle_state') == 'staged'
|
||||
and source in ('torrent', 'usenet')
|
||||
and mode == source
|
||||
):
|
||||
return f'Track was not found in the staged {source} album release'
|
||||
|
||||
return None
|
||||
|
||||
|
||||
@dataclass
|
||||
class TaskWorkerDeps:
|
||||
"""Bundle of cross-cutting deps the per-task download worker needs."""
|
||||
|
|
@ -128,6 +157,21 @@ def download_track_worker(task_id: str, batch_id: Optional[str], deps: TaskWorke
|
|||
# === STAGING CHECK: Check staging folder for existing file before searching ===
|
||||
if deps.try_staging_match(task_id, batch_id, track):
|
||||
return
|
||||
staging_miss_reason = _private_album_bundle_staging_miss_reason(batch_id, deps)
|
||||
if staging_miss_reason:
|
||||
logger.warning(
|
||||
"[Modal Worker] %s for '%s'; skipping redundant per-track %s search",
|
||||
staging_miss_reason,
|
||||
track.name,
|
||||
getattr(deps.download_orchestrator, 'mode', 'release-source'),
|
||||
)
|
||||
with tasks_lock:
|
||||
if task_id in download_tasks:
|
||||
download_tasks[task_id]['status'] = 'not_found'
|
||||
download_tasks[task_id]['error_message'] = staging_miss_reason
|
||||
if batch_id:
|
||||
deps.on_download_completed(batch_id, task_id, False)
|
||||
return
|
||||
|
||||
# Initialize task state tracking (like GUI's parallel_search_tracking)
|
||||
with tasks_lock:
|
||||
|
|
@ -147,6 +191,27 @@ def download_track_worker(task_id: str, batch_id: Optional[str], deps: TaskWorke
|
|||
artist_name = track.artists[0] if track.artists else None
|
||||
track_name = track.name
|
||||
|
||||
release_queries = []
|
||||
try:
|
||||
_download_mode = (getattr(deps.download_orchestrator, 'mode', '') or '').lower()
|
||||
_track_album = (getattr(track, 'album', '') or '').strip()
|
||||
_track_title = (getattr(track, 'name', '') or '').strip()
|
||||
_track_artists = list(getattr(track, 'artists', []) or [])
|
||||
_first_artist = _track_artists[0] if _track_artists else ''
|
||||
_primary_artist = (
|
||||
(_first_artist.get('name', '') if isinstance(_first_artist, dict) else str(_first_artist))
|
||||
or ''
|
||||
).strip()
|
||||
if (
|
||||
_download_mode in ('torrent', 'usenet')
|
||||
and _primary_artist
|
||||
and _track_album
|
||||
and _track_album.lower() not in ('unknown album', _track_title.lower())
|
||||
):
|
||||
release_queries.append(f"{_primary_artist} {_track_album}".strip())
|
||||
except Exception as _release_query_exc:
|
||||
logger.debug("[Modal Worker] release query hint failed: %s", _release_query_exc)
|
||||
|
||||
# Start with matching engine queries
|
||||
search_queries = deps.matching_engine.generate_download_queries(track)
|
||||
|
||||
|
|
@ -175,8 +240,14 @@ def download_track_worker(task_id: str, batch_id: Optional[str], deps: TaskWorke
|
|||
if cleaned_name and cleaned_name.lower() != track_name.lower():
|
||||
legacy_queries.append(cleaned_name.strip())
|
||||
|
||||
# Combine enhanced queries with legacy fallbacks
|
||||
all_queries = search_queries + legacy_queries
|
||||
# Combine enhanced queries with legacy fallbacks.
|
||||
#
|
||||
# Torrent / usenet can use full album releases as a fallback for
|
||||
# single-track requests, but trying the album release first makes
|
||||
# playlist batches download whole albums before checking whether a
|
||||
# track-shaped release exists. Keep release queries last so singles
|
||||
# stay light when the indexer has a direct result.
|
||||
all_queries = search_queries + legacy_queries + release_queries
|
||||
|
||||
# Remove duplicates while preserving order
|
||||
unique_queries = []
|
||||
|
|
@ -209,8 +280,30 @@ def download_track_worker(task_id: str, batch_id: Optional[str], deps: TaskWorke
|
|||
logger.debug(f"About to call soulseek search for task {task_id}")
|
||||
|
||||
try:
|
||||
# Hybrid + album-context batches must skip torrent / usenet during
|
||||
# the per-track loop — they're release-level sources, can't match
|
||||
# individual tracks meaningfully, and album-bundle handling only
|
||||
# fires in single-source mode (see core/downloads/master.py). The
|
||||
# exclusion lets the hybrid chain fall through to per-track-
|
||||
# compatible sources (soulseek / streaming) instead of attempting
|
||||
# N redundant Prowlarr searches that all download the same album
|
||||
# torrent and rely on the auto-import sweep to clean up.
|
||||
_exclude_for_hybrid_album = None
|
||||
try:
|
||||
_batch_is_album = False
|
||||
if batch_id:
|
||||
from core.runtime_state import download_batches as _db
|
||||
_b = _db.get(batch_id)
|
||||
if isinstance(_b, dict):
|
||||
_batch_is_album = bool(_b.get('is_album_download'))
|
||||
if _batch_is_album and getattr(deps.download_orchestrator, 'mode', '') == 'hybrid':
|
||||
_exclude_for_hybrid_album = ['torrent', 'usenet']
|
||||
except Exception as _exc_filter_err:
|
||||
logger.debug("[Modal Worker] album-source-exclusion check failed: %s", _exc_filter_err)
|
||||
# Perform search with timeout
|
||||
tracks_result, _ = deps.run_async(deps.download_orchestrator.search(query, timeout=30))
|
||||
tracks_result, _ = deps.run_async(deps.download_orchestrator.search(
|
||||
query, timeout=30, exclude_sources=_exclude_for_hybrid_album,
|
||||
))
|
||||
logger.debug(f"Search completed for task {task_id}, got {len(tracks_result) if tracks_result else 0} results")
|
||||
|
||||
# CRITICAL: Check cancellation immediately after search returns
|
||||
|
|
|
|||
|
|
@ -25,6 +25,20 @@ def init(matching_engine_obj, download_orchestrator_obj):
|
|||
download_orchestrator = download_orchestrator_obj
|
||||
|
||||
|
||||
def _torrent_usenet_artist_is_fallback(result):
|
||||
"""True when a release result has no parsed artist, only indexer filler."""
|
||||
if getattr(result, 'username', None) not in ('torrent', 'usenet'):
|
||||
return False
|
||||
artist = (getattr(result, 'artist', None) or '').strip()
|
||||
if not artist:
|
||||
return True
|
||||
metadata = getattr(result, '_source_metadata', None) or {}
|
||||
indexer = str(metadata.get('indexer') or '').strip()
|
||||
if artist.lower() in ('torrent', 'usenet'):
|
||||
return True
|
||||
return bool(indexer and artist.lower() == indexer.lower())
|
||||
|
||||
|
||||
def filter_soundcloud_previews(results, expected_track):
|
||||
"""Drop SoundCloud preview snippets so they never reach the cache,
|
||||
the modal, or the auto-download attempt.
|
||||
|
|
@ -97,8 +111,12 @@ def get_valid_candidates(results, spotify_track, query):
|
|||
return []
|
||||
|
||||
# Streaming sources (YouTube, Tidal, Qobuz, HiFi, Deezer, SoundCloud) return structured API results
|
||||
# with proper artist/title metadata — score using the same matching engine as Soulseek
|
||||
_streaming_sources = ("youtube", "tidal", "qobuz", "hifi", "deezer_dl", "soundcloud", "amazon")
|
||||
# with proper artist/title metadata — score using the same matching engine as Soulseek.
|
||||
# Torrent / usenet results also belong here: their filename field is a download URL, not
|
||||
# a slskd-style ``Artist/Album/Track.flac`` path, so the Soulseek matcher would extract
|
||||
# garbage segments from it. Routing them through the streaming path means score_track_match
|
||||
# reads ``r.title`` and ``r.artist`` directly (which the torrent/usenet projections pre-fill).
|
||||
_streaming_sources = ("youtube", "tidal", "qobuz", "hifi", "deezer_dl", "soundcloud", "amazon", "torrent", "usenet")
|
||||
if results[0].username in _streaming_sources:
|
||||
source_label = results[0].username.replace('_dl', '').title()
|
||||
expected_artists = spotify_track.artists if spotify_track else []
|
||||
|
|
@ -132,16 +150,55 @@ def get_valid_candidates(results, spotify_track, query):
|
|||
)
|
||||
continue
|
||||
|
||||
# Score using matching engine's generic scorer (same weights as Soulseek)
|
||||
# Score using matching engine's generic scorer (same weights as Soulseek).
|
||||
# Torrent/usenet release projections sometimes only have the indexer name
|
||||
# in the artist field when a title did not parse as "Artist - Release".
|
||||
# Treat that as unknown artist, not as a real mismatch.
|
||||
has_only_fallback_artist = _torrent_usenet_artist_is_fallback(r)
|
||||
candidate_artists = [] if has_only_fallback_artist else ([r.artist] if r.artist else [])
|
||||
confidence, match_type = matching_engine.score_track_match(
|
||||
source_title=expected_title,
|
||||
source_artists=expected_artists,
|
||||
source_duration_ms=expected_duration,
|
||||
candidate_title=r.title or '',
|
||||
candidate_artists=[r.artist] if r.artist else [],
|
||||
candidate_artists=candidate_artists,
|
||||
candidate_duration_ms=r.duration or 0,
|
||||
)
|
||||
|
||||
# Album-name fallback for torrent / usenet per-track results.
|
||||
#
|
||||
# When this fallback runs: hybrid mode + non-album batch (single
|
||||
# track wishlist / playlist of singles). Album-context batches
|
||||
# never reach here — the album-bundle gate in
|
||||
# core/downloads/album_bundle_dispatch.py engages the bulk-
|
||||
# download flow in single-source mode, and the hybrid chain
|
||||
# filter in core/downloads/task_worker.py strips torrent /
|
||||
# usenet from album batches in hybrid mode. What's left is the
|
||||
# single-track-in-hybrid case where a user is searching for one
|
||||
# track and the only torrent / usenet result is the album that
|
||||
# contains it.
|
||||
#
|
||||
# Without this fallback, "Luther (with SZA)" against a
|
||||
# candidate titled "GNX (2024) [FLAC]" scores ~0 on track-title
|
||||
# alone — even though the album torrent does in fact contain
|
||||
# the wanted track. Scoring the candidate title against the
|
||||
# wanted track's ALBUM name and taking the max gives album-
|
||||
# level releases a fair shot. The Auto-Import sweep then picks
|
||||
# the right file out of the downloaded album folder.
|
||||
expected_album = getattr(spotify_track, 'album', None) if spotify_track else None
|
||||
if r.username in ('torrent', 'usenet') and expected_album:
|
||||
album_conf, _ = matching_engine.score_track_match(
|
||||
source_title=expected_album,
|
||||
source_artists=expected_artists,
|
||||
source_duration_ms=0, # albums don't have one duration
|
||||
candidate_title=r.title or '',
|
||||
candidate_artists=candidate_artists,
|
||||
candidate_duration_ms=0,
|
||||
)
|
||||
if album_conf > confidence:
|
||||
confidence = album_conf
|
||||
match_type = 'album_release'
|
||||
|
||||
# Version detection penalty — reject live/remix/acoustic when expecting original
|
||||
r_title_lower = (r.title or '').lower()
|
||||
is_wrong_version = False
|
||||
|
|
@ -162,8 +219,10 @@ def get_valid_candidates(results, spotify_track, query):
|
|||
|
||||
# Artist gate — streaming APIs (Tidal/Qobuz/HiFi/Deezer) have reliable metadata,
|
||||
# so "My Will" by "B. Starr" should never match expected "B小町".
|
||||
# Skip for YouTube — artist is parsed from video titles and often unreliable.
|
||||
if r.username != 'youtube':
|
||||
# YouTube stays excluded because video-title parsing is unreliable.
|
||||
# Torrent/usenet must also pass this gate so title-only matches
|
||||
# from the wrong artist do not get downloaded.
|
||||
if r.username != 'youtube' and not has_only_fallback_artist:
|
||||
from difflib import SequenceMatcher
|
||||
import re as _re
|
||||
_cand_artist_raw = r.artist or ''
|
||||
|
|
@ -195,6 +254,16 @@ def get_valid_candidates(results, spotify_track, query):
|
|||
# so falling to SequenceMatcher means the strings are genuinely
|
||||
# different. 0.5 gives a safer buffer without blocking real
|
||||
# matches that would have scored above 0.85 anyway.
|
||||
if r.username in ('torrent', 'usenet') and _best_artist < 0.5:
|
||||
logger.info(
|
||||
"[%s] Rejecting candidate due to artist mismatch: "
|
||||
"expected=%s candidate=%r title=%r",
|
||||
source_label,
|
||||
list(expected_artists),
|
||||
_cand_artist_raw,
|
||||
r.title or '',
|
||||
)
|
||||
continue
|
||||
if _best_artist < 0.5 and confidence < 0.85:
|
||||
continue
|
||||
|
||||
|
|
|
|||
|
|
@ -23,6 +23,8 @@ import re
|
|||
import uuid
|
||||
import time
|
||||
import shutil
|
||||
import json
|
||||
import base64
|
||||
import subprocess
|
||||
import threading
|
||||
from typing import List, Optional, Dict, Any, Tuple
|
||||
|
|
@ -75,6 +77,13 @@ HLS_QUALITY_MAP = {
|
|||
|
||||
HLS_MAP_TAG_RE = re.compile(r'#EXT-X-MAP:.*URI="([^"]+)"')
|
||||
|
||||
TRACK_ENDPOINT_QUALITY_MAP = {
|
||||
'hires': 'HI_RES_LOSSLESS',
|
||||
'lossless': 'LOSSLESS',
|
||||
'high': 'HIGH',
|
||||
'low': 'LOW',
|
||||
}
|
||||
|
||||
# Default public hifi-api instances (ordered by preference)
|
||||
DEFAULT_INSTANCES = [
|
||||
'https://triton.squid.wtf',
|
||||
|
|
@ -244,6 +253,122 @@ class HiFiClient(DownloadSourcePlugin):
|
|||
return data.get('version') or data.get('data', {}).get('version')
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def _extract_manifest_uri(data: Any) -> Optional[str]:
|
||||
try:
|
||||
inner = data.get('data', data) if isinstance(data, dict) else data
|
||||
attrs = inner.get('data', {}).get('attributes', {})
|
||||
return attrs.get('uri')
|
||||
except (AttributeError, KeyError):
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def _extract_track_manifest_urls(data: Any) -> List[str]:
|
||||
try:
|
||||
inner = data.get('data', data) if isinstance(data, dict) else data
|
||||
manifest_b64 = inner.get('manifest')
|
||||
if not manifest_b64:
|
||||
return []
|
||||
manifest = json.loads(base64.b64decode(manifest_b64))
|
||||
if manifest.get('encryptionType') not in (None, 'NONE'):
|
||||
return []
|
||||
urls = manifest.get('urls') or []
|
||||
return [url for url in urls if isinstance(url, str) and url]
|
||||
except Exception as e:
|
||||
logger.debug("Failed to extract legacy HiFi track manifest URLs: %s", e)
|
||||
return []
|
||||
|
||||
@staticmethod
|
||||
def _extension_from_track_manifest(data: Any, fallback: str) -> str:
|
||||
try:
|
||||
inner = data.get('data', data) if isinstance(data, dict) else data
|
||||
manifest = json.loads(base64.b64decode(inner.get('manifest') or ''))
|
||||
mime = (manifest.get('mimeType') or '').lower()
|
||||
codecs = (manifest.get('codecs') or '').lower()
|
||||
if 'flac' in mime or 'flac' in codecs:
|
||||
return 'flac'
|
||||
if 'mp4' in mime or 'aac' in codecs:
|
||||
return 'm4a'
|
||||
except Exception as e:
|
||||
logger.debug("Failed to infer legacy HiFi track manifest extension: %s", e)
|
||||
return fallback
|
||||
|
||||
def check_instance_capabilities(self, url: str, timeout: int = 5) -> Dict[str, Any]:
|
||||
"""Probe one public HiFi instance using the endpoints SoulSync needs."""
|
||||
entry = {
|
||||
'url': url,
|
||||
'status': 'unknown',
|
||||
'version': None,
|
||||
'can_search': False,
|
||||
'can_download': False,
|
||||
}
|
||||
try:
|
||||
root = self.session.get(
|
||||
f'{url}/',
|
||||
timeout=timeout,
|
||||
headers={'Accept': 'application/json'},
|
||||
)
|
||||
if not root.ok:
|
||||
entry['status'] = f'error (HTTP {root.status_code})'
|
||||
return entry
|
||||
|
||||
data = root.json()
|
||||
entry['version'] = data.get('version') or data.get('data', {}).get('version')
|
||||
entry['status'] = 'online'
|
||||
|
||||
search = self.session.get(
|
||||
f'{url}/search/',
|
||||
params={'s': 'test', 'limit': 1},
|
||||
timeout=timeout,
|
||||
)
|
||||
entry['can_search'] = search.ok
|
||||
|
||||
manifest = self.session.get(
|
||||
f'{url}/trackManifests/',
|
||||
params={
|
||||
'id': '1550546',
|
||||
'formats': 'FLAC',
|
||||
'usage': 'DOWNLOAD',
|
||||
'manifestType': 'HLS',
|
||||
'adaptive': 'true',
|
||||
'uriScheme': 'HTTPS',
|
||||
},
|
||||
timeout=timeout,
|
||||
)
|
||||
entry['can_download'] = (
|
||||
manifest.ok
|
||||
and bool(self._extract_manifest_uri(manifest.json()))
|
||||
)
|
||||
if not manifest.ok:
|
||||
entry['download_error'] = f'HTTP {manifest.status_code}'
|
||||
elif not entry['can_download']:
|
||||
legacy = self.session.get(
|
||||
f'{url}/track/',
|
||||
params={'id': '1550546', 'quality': 'LOSSLESS'},
|
||||
timeout=timeout,
|
||||
)
|
||||
entry['can_download'] = (
|
||||
legacy.ok
|
||||
and bool(self._extract_track_manifest_urls(legacy.json()))
|
||||
)
|
||||
if not legacy.ok:
|
||||
entry['download_error'] = f'HTTP {legacy.status_code}'
|
||||
elif not entry['can_download']:
|
||||
entry['download_error'] = 'No playable manifest URL'
|
||||
else:
|
||||
entry['download_probe'] = 'track'
|
||||
else:
|
||||
entry['download_probe'] = 'trackManifests'
|
||||
except http_requests.exceptions.SSLError:
|
||||
entry['status'] = 'ssl_error'
|
||||
except http_requests.exceptions.ConnectTimeout:
|
||||
entry['status'] = 'timeout'
|
||||
except http_requests.exceptions.ConnectionError:
|
||||
entry['status'] = 'offline'
|
||||
except Exception as e:
|
||||
entry['status'] = f'error ({type(e).__name__})'
|
||||
return entry
|
||||
|
||||
def search_tracks(self, title: str = None, artist: str = None,
|
||||
album: str = None, limit: int = 20) -> List[Dict]:
|
||||
params = {'limit': limit}
|
||||
|
|
@ -433,15 +558,12 @@ class HiFiClient(DownloadSourcePlugin):
|
|||
|
||||
data = self._api_get('/trackManifests/', params=params, timeout=20)
|
||||
if not data:
|
||||
return None
|
||||
return self._get_legacy_track_manifest(track_id, quality)
|
||||
|
||||
try:
|
||||
inner = data.get('data', data) if isinstance(data, dict) else data
|
||||
attrs = inner.get('data', {}).get('attributes', {})
|
||||
uri = attrs.get('uri')
|
||||
except (AttributeError, KeyError) as e:
|
||||
logger.warning(f"Failed to extract playlist URI from manifest response: {e}")
|
||||
return None
|
||||
uri = self._extract_manifest_uri(data)
|
||||
if uri is None:
|
||||
logger.warning("Failed to extract playlist URI from manifest response")
|
||||
return self._get_legacy_track_manifest(track_id, quality)
|
||||
|
||||
if not uri:
|
||||
logger.warning(f"No playlist URI in manifest for track {track_id}")
|
||||
|
|
@ -488,6 +610,28 @@ class HiFiClient(DownloadSourcePlugin):
|
|||
'quality': quality,
|
||||
}
|
||||
|
||||
def _get_legacy_track_manifest(self, track_id: int, quality: str = 'lossless') -> Optional[Dict]:
|
||||
q_info = HLS_QUALITY_MAP.get(quality, HLS_QUALITY_MAP['lossless'])
|
||||
api_quality = TRACK_ENDPOINT_QUALITY_MAP.get(quality, 'LOSSLESS')
|
||||
data = self._api_get('/track/', params={'id': track_id, 'quality': api_quality}, timeout=20)
|
||||
if not data:
|
||||
return None
|
||||
|
||||
direct_urls = self._extract_track_manifest_urls(data)
|
||||
if not direct_urls:
|
||||
logger.warning(f"No playable URL in legacy HiFi manifest for track {track_id}")
|
||||
return None
|
||||
|
||||
extension = self._extension_from_track_manifest(data, q_info['extension'])
|
||||
logger.info(f"HiFi legacy track manifest for track {track_id}: "
|
||||
f"{len(direct_urls)} direct URL(s) ({quality})")
|
||||
return {
|
||||
'direct_urls': direct_urls,
|
||||
'extension': extension,
|
||||
'codec': q_info['codec'],
|
||||
'quality': quality,
|
||||
}
|
||||
|
||||
def _demux_flac(self, input_path: Path, output_path: Path) -> None:
|
||||
ffmpeg = shutil.which('ffmpeg')
|
||||
if not ffmpeg:
|
||||
|
|
@ -619,7 +763,13 @@ class HiFiClient(DownloadSourcePlugin):
|
|||
return None
|
||||
|
||||
manifest_info = self._get_hls_manifest(track_id, quality=q_key)
|
||||
if not manifest_info or not manifest_info.get('segment_uris'):
|
||||
if (
|
||||
not manifest_info
|
||||
or (
|
||||
not manifest_info.get('segment_uris')
|
||||
and not manifest_info.get('direct_urls')
|
||||
)
|
||||
):
|
||||
logger.warning(f"No HLS manifest at quality {q_key}, trying next")
|
||||
continue
|
||||
|
||||
|
|
@ -628,16 +778,17 @@ class HiFiClient(DownloadSourcePlugin):
|
|||
out_filename = f"{safe_name}.{extension}"
|
||||
out_path = self.download_path / out_filename
|
||||
|
||||
is_flac = q_key in ('hires', 'lossless')
|
||||
is_direct = bool(manifest_info.get('direct_urls'))
|
||||
is_flac = q_key in ('hires', 'lossless') and not is_direct
|
||||
intermediate_path = out_path.with_suffix('.m4a') if is_flac else out_path
|
||||
|
||||
try:
|
||||
init_uri = manifest_info.get('init_uri')
|
||||
segment_uris = manifest_info['segment_uris']
|
||||
segment_uris = manifest_info.get('segment_uris') or manifest_info.get('direct_urls') or []
|
||||
total_segments = len(segment_uris) + (1 if init_uri else 0)
|
||||
|
||||
logger.info(f"Downloading from HiFi ({q_key}): {out_filename} "
|
||||
f"({total_segments} segments)")
|
||||
f"({total_segments} {'URL(s)' if is_direct else 'segments'})")
|
||||
|
||||
downloaded = 0
|
||||
speed_start = time.time()
|
||||
|
|
|
|||
|
|
@ -198,6 +198,10 @@ def record_library_history_download(context: Dict[str, Any]) -> None:
|
|||
"deezer_dl": "Deezer",
|
||||
"lidarr": "Lidarr",
|
||||
"soundcloud": "SoundCloud",
|
||||
"amazon": "Amazon",
|
||||
"staging": "Staging",
|
||||
"torrent": "Torrent",
|
||||
"usenet": "Usenet",
|
||||
# Auto-import isn't a download source, but flows through the
|
||||
# same post-process pipeline (file lands → record provenance
|
||||
# + history → write to library DB). Tagging it as "Auto-Import"
|
||||
|
|
@ -276,6 +280,7 @@ def record_download_provenance(context: Dict[str, Any]) -> None:
|
|||
"deezer_dl": "deezer",
|
||||
"lidarr": "lidarr",
|
||||
"soundcloud": "soundcloud",
|
||||
"amazon": "amazon",
|
||||
# Auto-import: surfaced in provenance so the redownload modal
|
||||
# can tell the user "this came from staging on <date>" instead
|
||||
# of falsely listing soulseek as the source. The underlying
|
||||
|
|
@ -283,6 +288,16 @@ def record_download_provenance(context: Dict[str, Any]) -> None:
|
|||
# separately via the source-aware ID columns on the tracks
|
||||
# row itself.
|
||||
"auto_import": "auto_import",
|
||||
# Generic staging-match (user dropped files manually OR a
|
||||
# source we don't have a more specific label for). Better
|
||||
# than defaulting to 'soulseek' which would falsely tag the
|
||||
# provenance.
|
||||
"staging": "staging",
|
||||
# Torrent / usenet album-bundle flow — the staging matcher
|
||||
# overrides 'staging' with the bundle source so the history
|
||||
# shows where the files actually came from.
|
||||
"torrent": "torrent",
|
||||
"usenet": "usenet",
|
||||
}.get(username, "soulseek")
|
||||
|
||||
ti = context.get("track_info") or context.get("search_result") or {}
|
||||
|
|
|
|||
|
|
@ -8,6 +8,7 @@ Transparent to callers: check cache before API call, store after success.
|
|||
import json
|
||||
import logging
|
||||
import threading
|
||||
import time
|
||||
from datetime import datetime
|
||||
from typing import Optional, Dict, List, Tuple
|
||||
|
||||
|
|
@ -49,6 +50,33 @@ class MetadataCache:
|
|||
from database.music_database import get_database
|
||||
return get_database()
|
||||
|
||||
@staticmethod
|
||||
def _is_transient_sqlite_io_error(exc: Exception) -> bool:
|
||||
return 'disk i/o error' in str(exc).lower()
|
||||
|
||||
def _run_maintenance_write(self, label: str, operation, default: int = 0) -> int:
|
||||
"""Run a maintenance write with one retry for transient SQLite I/O."""
|
||||
for attempt in range(2):
|
||||
try:
|
||||
db = self._get_db()
|
||||
conn = db._get_connection()
|
||||
try:
|
||||
return operation(conn)
|
||||
finally:
|
||||
conn.close()
|
||||
except Exception as e:
|
||||
if self._is_transient_sqlite_io_error(e) and attempt == 0:
|
||||
logger.warning(
|
||||
"%s hit transient SQLite disk I/O error; retrying once: %s",
|
||||
label,
|
||||
e,
|
||||
)
|
||||
time.sleep(0.25)
|
||||
continue
|
||||
logger.error("%s error: %s", label, e)
|
||||
return default
|
||||
return default
|
||||
|
||||
# ─── Entity Methods ───────────────────────────────────────────────
|
||||
|
||||
def get_entity(self, source: str, entity_type: str, entity_id: str) -> Optional[dict]:
|
||||
|
|
@ -566,137 +594,113 @@ class MetadataCache:
|
|||
|
||||
def evict_expired(self) -> int:
|
||||
"""Delete entries that have exceeded their TTL. Returns count of evicted entries."""
|
||||
try:
|
||||
db = self._get_db()
|
||||
conn = db._get_connection()
|
||||
try:
|
||||
cursor = conn.cursor()
|
||||
def _operation(conn):
|
||||
cursor = conn.cursor()
|
||||
|
||||
# Entities
|
||||
cursor.execute("""
|
||||
DELETE FROM metadata_cache_entities
|
||||
WHERE julianday('now') - julianday(updated_at) > ttl_days
|
||||
""")
|
||||
entity_count = cursor.rowcount
|
||||
# Entities
|
||||
cursor.execute("""
|
||||
DELETE FROM metadata_cache_entities
|
||||
WHERE julianday('now') - julianday(updated_at) > ttl_days
|
||||
""")
|
||||
entity_count = cursor.rowcount
|
||||
|
||||
# Searches
|
||||
cursor.execute("""
|
||||
DELETE FROM metadata_cache_searches
|
||||
WHERE julianday('now') - julianday(created_at) > ttl_days
|
||||
""")
|
||||
search_count = cursor.rowcount
|
||||
# Searches
|
||||
cursor.execute("""
|
||||
DELETE FROM metadata_cache_searches
|
||||
WHERE julianday('now') - julianday(created_at) > ttl_days
|
||||
""")
|
||||
search_count = cursor.rowcount
|
||||
|
||||
conn.commit()
|
||||
total = entity_count + search_count
|
||||
if total > 0:
|
||||
logger.info(f"Evicted {total} expired cache entries ({entity_count} entities, {search_count} searches)")
|
||||
return total
|
||||
finally:
|
||||
conn.close()
|
||||
except Exception as e:
|
||||
logger.error(f"Cache eviction error: {e}")
|
||||
return 0
|
||||
conn.commit()
|
||||
total = entity_count + search_count
|
||||
if total > 0:
|
||||
logger.info(f"Evicted {total} expired cache entries ({entity_count} entities, {search_count} searches)")
|
||||
return total
|
||||
|
||||
return self._run_maintenance_write("Cache eviction", _operation)
|
||||
|
||||
def clean_junk_entities(self) -> int:
|
||||
"""Delete cached entities with empty/placeholder names."""
|
||||
try:
|
||||
db = self._get_db()
|
||||
conn = db._get_connection()
|
||||
try:
|
||||
cursor = conn.cursor()
|
||||
junk_names = "', '".join(self._JUNK_NAMES - {''}) # exclude empty, handled separately
|
||||
cursor.execute(f"""
|
||||
DELETE FROM metadata_cache_entities
|
||||
WHERE (name IS NULL
|
||||
OR TRIM(name) = ''
|
||||
OR LOWER(TRIM(name)) IN ('{junk_names}'))
|
||||
AND entity_id NOT LIKE '%\\_features' ESCAPE '\\'
|
||||
AND entity_id NOT LIKE '%\\_tracks' ESCAPE '\\'
|
||||
""")
|
||||
count = cursor.rowcount
|
||||
conn.commit()
|
||||
if count > 0:
|
||||
logger.info(f"Cleaned {count} junk entities from cache")
|
||||
return count
|
||||
finally:
|
||||
conn.close()
|
||||
except Exception as e:
|
||||
logger.error(f"Junk cleanup error: {e}")
|
||||
return 0
|
||||
def _operation(conn):
|
||||
cursor = conn.cursor()
|
||||
junk_names = "', '".join(self._JUNK_NAMES - {''}) # exclude empty, handled separately
|
||||
cursor.execute(f"""
|
||||
DELETE FROM metadata_cache_entities
|
||||
WHERE (name IS NULL
|
||||
OR TRIM(name) = ''
|
||||
OR LOWER(TRIM(name)) IN ('{junk_names}'))
|
||||
AND entity_id NOT LIKE '%\\_features' ESCAPE '\\'
|
||||
AND entity_id NOT LIKE '%\\_tracks' ESCAPE '\\'
|
||||
""")
|
||||
count = cursor.rowcount
|
||||
conn.commit()
|
||||
if count > 0:
|
||||
logger.info(f"Cleaned {count} junk entities from cache")
|
||||
return count
|
||||
|
||||
return self._run_maintenance_write("Junk cleanup", _operation)
|
||||
|
||||
def clean_orphaned_searches(self) -> int:
|
||||
"""Delete search results where <50% of referenced entities still exist."""
|
||||
try:
|
||||
db = self._get_db()
|
||||
conn = db._get_connection()
|
||||
try:
|
||||
cursor = conn.cursor()
|
||||
cursor.execute("SELECT id, source, search_type, result_ids FROM metadata_cache_searches")
|
||||
rows = cursor.fetchall()
|
||||
def _operation(conn):
|
||||
cursor = conn.cursor()
|
||||
cursor.execute("SELECT id, source, search_type, result_ids FROM metadata_cache_searches")
|
||||
rows = cursor.fetchall()
|
||||
|
||||
dead_ids = []
|
||||
for row in rows:
|
||||
try:
|
||||
result_ids = json.loads(row['result_ids'] or '[]')
|
||||
except (json.JSONDecodeError, TypeError):
|
||||
dead_ids.append(row['id'])
|
||||
continue
|
||||
dead_ids = []
|
||||
for row in rows:
|
||||
try:
|
||||
result_ids = json.loads(row['result_ids'] or '[]')
|
||||
except (json.JSONDecodeError, TypeError):
|
||||
dead_ids.append(row['id'])
|
||||
continue
|
||||
|
||||
if not result_ids:
|
||||
dead_ids.append(row['id'])
|
||||
continue
|
||||
if not result_ids:
|
||||
dead_ids.append(row['id'])
|
||||
continue
|
||||
|
||||
# Check how many referenced entities still exist
|
||||
placeholders = ','.join('?' * len(result_ids))
|
||||
cursor.execute(f"""
|
||||
SELECT COUNT(*) FROM metadata_cache_entities
|
||||
WHERE source = ? AND entity_type = ? AND entity_id IN ({placeholders})
|
||||
""", [row['source'], row['search_type']] + list(result_ids))
|
||||
found = cursor.fetchone()[0]
|
||||
# Check how many referenced entities still exist
|
||||
placeholders = ','.join('?' * len(result_ids))
|
||||
cursor.execute(f"""
|
||||
SELECT COUNT(*) FROM metadata_cache_entities
|
||||
WHERE source = ? AND entity_type = ? AND entity_id IN ({placeholders})
|
||||
""", [row['source'], row['search_type']] + list(result_ids))
|
||||
found = cursor.fetchone()[0]
|
||||
|
||||
if found < len(result_ids) * 0.5:
|
||||
dead_ids.append(row['id'])
|
||||
if found < len(result_ids) * 0.5:
|
||||
dead_ids.append(row['id'])
|
||||
|
||||
if dead_ids:
|
||||
# Delete in chunks to stay under SQLite variable limit
|
||||
for i in range(0, len(dead_ids), 400):
|
||||
chunk = dead_ids[i:i + 400]
|
||||
placeholders = ','.join('?' * len(chunk))
|
||||
cursor.execute(f"DELETE FROM metadata_cache_searches WHERE id IN ({placeholders})", chunk)
|
||||
conn.commit()
|
||||
if dead_ids:
|
||||
# Delete in chunks to stay under SQLite variable limit
|
||||
for i in range(0, len(dead_ids), 400):
|
||||
chunk = dead_ids[i:i + 400]
|
||||
placeholders = ','.join('?' * len(chunk))
|
||||
cursor.execute(f"DELETE FROM metadata_cache_searches WHERE id IN ({placeholders})", chunk)
|
||||
conn.commit()
|
||||
|
||||
count = len(dead_ids)
|
||||
if count > 0:
|
||||
logger.info(f"Cleaned {count} orphaned search results from cache")
|
||||
return count
|
||||
finally:
|
||||
conn.close()
|
||||
except Exception as e:
|
||||
logger.error(f"Orphan search cleanup error: {e}")
|
||||
return 0
|
||||
count = len(dead_ids)
|
||||
if count > 0:
|
||||
logger.info(f"Cleaned {count} orphaned search results from cache")
|
||||
return count
|
||||
|
||||
return self._run_maintenance_write("Orphan search cleanup", _operation)
|
||||
|
||||
def clean_stale_musicbrainz_nulls(self, max_age_days: int = 30) -> int:
|
||||
"""Delete MusicBrainz cache entries where lookup found nothing (null MBID) and age > max_age_days."""
|
||||
try:
|
||||
db = self._get_db()
|
||||
conn = db._get_connection()
|
||||
try:
|
||||
cursor = conn.cursor()
|
||||
cursor.execute("""
|
||||
DELETE FROM musicbrainz_cache
|
||||
WHERE musicbrainz_id IS NULL
|
||||
AND julianday('now') - julianday(last_updated) > ?
|
||||
""", (max_age_days,))
|
||||
count = cursor.rowcount
|
||||
conn.commit()
|
||||
if count > 0:
|
||||
logger.info(f"Cleaned {count} stale MusicBrainz null entries (>{max_age_days} days)")
|
||||
return count
|
||||
finally:
|
||||
conn.close()
|
||||
except Exception as e:
|
||||
logger.error(f"MusicBrainz null cleanup error: {e}")
|
||||
return 0
|
||||
def _operation(conn):
|
||||
cursor = conn.cursor()
|
||||
cursor.execute("""
|
||||
DELETE FROM musicbrainz_cache
|
||||
WHERE musicbrainz_id IS NULL
|
||||
AND julianday('now') - julianday(last_updated) > ?
|
||||
""", (max_age_days,))
|
||||
count = cursor.rowcount
|
||||
conn.commit()
|
||||
if count > 0:
|
||||
logger.info(f"Cleaned {count} stale MusicBrainz null entries (>{max_age_days} days)")
|
||||
return count
|
||||
|
||||
return self._run_maintenance_write("MusicBrainz null cleanup", _operation)
|
||||
|
||||
def get_health_stats(self) -> dict:
|
||||
"""Return cache health statistics for the repair dashboard.
|
||||
|
|
|
|||
|
|
@ -35,6 +35,64 @@ logger = get_logger("repair_worker")
|
|||
AUDIO_EXTENSIONS = {'.mp3', '.flac', '.ogg', '.opus', '.m4a', '.aac', '.wav', '.wma', '.aiff', '.aif'}
|
||||
|
||||
|
||||
def _album_fill_artist_names_match(expected_artist: str, candidate_artist: str) -> bool:
|
||||
"""Strict artist gate for Album Completeness auto-fill.
|
||||
|
||||
Auto-fill moves/copies files into an existing album, so artist identity
|
||||
must outrank album/title similarity. Use the alias-aware matcher when it
|
||||
is available, then fall back to conservative normalized similarity.
|
||||
"""
|
||||
expected = (expected_artist or '').strip()
|
||||
candidate = (candidate_artist or '').strip()
|
||||
if not expected or not candidate:
|
||||
return False
|
||||
|
||||
try:
|
||||
from core.matching.artist_aliases import artist_names_match
|
||||
matched, score = artist_names_match(expected, candidate, threshold=0.82)
|
||||
if matched:
|
||||
return True
|
||||
if score < 0.82:
|
||||
return False
|
||||
except Exception as alias_err:
|
||||
logger.debug("artist_names_match unavailable, using fallback: %s", alias_err)
|
||||
|
||||
try:
|
||||
from core.matching_engine import MusicMatchingEngine
|
||||
engine = MusicMatchingEngine()
|
||||
expected_norm = engine.clean_artist(expected)
|
||||
candidate_norm = engine.clean_artist(candidate)
|
||||
except Exception:
|
||||
expected_norm = expected.lower()
|
||||
candidate_norm = candidate.lower()
|
||||
|
||||
if not expected_norm or not candidate_norm:
|
||||
return False
|
||||
return expected_norm == candidate_norm or SequenceMatcher(None, expected_norm, candidate_norm).ratio() >= 0.82
|
||||
|
||||
|
||||
def _album_fill_target_artist_allows_track(album_artist: str, track_artists: List[str]) -> bool:
|
||||
"""Return whether a source track can be auto-filled into an album artist.
|
||||
|
||||
Compilation-style album artists are allowed to contain varied track
|
||||
artists. Normal albums require at least one source track artist to match
|
||||
the target album artist before any local candidate is considered.
|
||||
"""
|
||||
album_artist = (album_artist or '').strip()
|
||||
if not album_artist:
|
||||
return False
|
||||
|
||||
normalized_album_artist = album_artist.lower().strip()
|
||||
if normalized_album_artist in {'various artists', 'various', 'soundtrack'}:
|
||||
return True
|
||||
|
||||
source_artists = [str(a).strip() for a in (track_artists or []) if str(a or '').strip()]
|
||||
if not source_artists:
|
||||
return True
|
||||
|
||||
return any(_album_fill_artist_names_match(album_artist, artist) for artist in source_artists)
|
||||
|
||||
|
||||
def _split_acoustid_credit(credit: str) -> List[str]:
|
||||
"""Split an AcoustID artist credit into individual contributor names.
|
||||
|
||||
|
|
@ -2097,6 +2155,21 @@ class RepairWorker:
|
|||
track_details.append({'track': track_name, 'status': 'skipped', 'reason': 'no track name'})
|
||||
continue
|
||||
|
||||
if not _album_fill_target_artist_allows_track(artist_name, track_artists):
|
||||
skipped_count += 1
|
||||
logger.warning(
|
||||
"Album auto-fill skipped '%s': source artist(s) %s do not match target album artist '%s'",
|
||||
track_name, track_artists, artist_name,
|
||||
)
|
||||
track_details.append({
|
||||
'track': track_name,
|
||||
'status': 'skipped',
|
||||
'reason': 'source artist does not match target album artist',
|
||||
'source_artists': track_artists,
|
||||
'target_artist': artist_name,
|
||||
})
|
||||
continue
|
||||
|
||||
# Search library for this track
|
||||
candidates = self.db.search_tracks(title=track_name, artist=artist_search, limit=20)
|
||||
|
||||
|
|
@ -2117,8 +2190,22 @@ class RepairWorker:
|
|||
|
||||
# Artist match (more lenient)
|
||||
cand_artist = getattr(cand, 'artist_name', '') or ''
|
||||
artist_sim = SequenceMatcher(None, artist_search.lower(), cand_artist.lower()).ratio()
|
||||
if artist_sim < 0.50:
|
||||
candidate_artist_fields = [
|
||||
cand_artist,
|
||||
getattr(cand, 'track_artist', '') or '',
|
||||
]
|
||||
expected_artist_names = track_artists or [artist_name]
|
||||
if not any(
|
||||
_album_fill_artist_names_match(expected, candidate)
|
||||
for expected in expected_artist_names
|
||||
for candidate in candidate_artist_fields
|
||||
):
|
||||
logger.debug(
|
||||
"Album auto-fill rejected candidate '%s' by '%s' for expected artist(s) %s",
|
||||
getattr(cand, 'title', ''),
|
||||
cand_artist,
|
||||
expected_artist_names,
|
||||
)
|
||||
continue
|
||||
|
||||
# Quality gate
|
||||
|
|
|
|||
|
|
@ -164,28 +164,60 @@ class QBittorrentAdapter:
|
|||
category: str,
|
||||
save_path: Optional[str],
|
||||
) -> Optional[str]:
|
||||
data = {'urls': url_or_magnet, 'category': category or self._category}
|
||||
cat = category or self._category
|
||||
# Snapshot the current set of torrent hashes BEFORE adding —
|
||||
# qBittorrent's /add endpoint returns 200 "Ok." regardless of
|
||||
# whether the URL was actually accepted or registered, and
|
||||
# category-filtered lookups race the add (qBit hasn't
|
||||
# categorised the new torrent yet on the first poll). Diffing
|
||||
# before / after is the only reliable way to recover the hash.
|
||||
before = self._all_hashes()
|
||||
if before is None:
|
||||
return None
|
||||
data = {'urls': url_or_magnet, 'category': cat}
|
||||
if save_path or self._save_path:
|
||||
data['savepath'] = save_path or self._save_path
|
||||
resp = self._call('POST', '/api/v2/torrents/add', data=data)
|
||||
if not resp or not resp.ok:
|
||||
logger.warning("qBittorrent /torrents/add returned HTTP %s body=%r",
|
||||
resp.status_code if resp else 'no-response',
|
||||
(resp.text[:200] if resp else ''))
|
||||
return None
|
||||
# qBittorrent's /add endpoint returns 200 "Ok." on success but
|
||||
# NOT the resulting info-hash. We have to look it up via
|
||||
# /torrents/info filtered by category — the just-added torrent
|
||||
# will be the newest entry in that category.
|
||||
return self._lookup_latest_hash(category or self._category)
|
||||
if resp.text and resp.text.strip() and resp.text.strip() != 'Ok.':
|
||||
logger.warning("qBittorrent /torrents/add unexpected body: %r", resp.text[:200])
|
||||
return None
|
||||
new_hash = self._poll_for_new_hash(before)
|
||||
if not new_hash:
|
||||
logger.error("qBittorrent accepted the request but no new torrent appeared — "
|
||||
"URL may have been rejected (bad magnet, unreachable HTTPS, "
|
||||
"duplicate hash, etc.)")
|
||||
return new_hash
|
||||
|
||||
def _lookup_latest_hash(self, category: str) -> Optional[str]:
|
||||
resp = self._call('GET', '/api/v2/torrents/info', params={'category': category, 'sort': 'added_on', 'reverse': 'true', 'limit': 1})
|
||||
def _all_hashes(self) -> Optional[set]:
|
||||
"""Return the set of every torrent hash qBit currently tracks,
|
||||
or None on lookup failure."""
|
||||
resp = self._call('GET', '/api/v2/torrents/info')
|
||||
if not resp or not resp.ok:
|
||||
return None
|
||||
try:
|
||||
items = resp.json()
|
||||
if items:
|
||||
return items[0].get('hash')
|
||||
return {item.get('hash') for item in resp.json() if item.get('hash')}
|
||||
except Exception as e:
|
||||
logger.error("qBittorrent /torrents/info parse failed: %s", e)
|
||||
return None
|
||||
|
||||
def _poll_for_new_hash(self, before: set) -> Optional[str]:
|
||||
"""Poll up to ~5s for a new torrent to appear (qBit takes a
|
||||
moment to fetch the .torrent file from the URL and register
|
||||
it). Returns the new hash, or None if nothing showed up."""
|
||||
import time as _time
|
||||
for _ in range(10):
|
||||
_time.sleep(0.5)
|
||||
current = self._all_hashes()
|
||||
if current is None:
|
||||
continue
|
||||
new = current - before
|
||||
if new:
|
||||
return next(iter(new))
|
||||
return None
|
||||
|
||||
async def add_torrent_file(
|
||||
|
|
@ -205,14 +237,18 @@ class QBittorrentAdapter:
|
|||
category: str,
|
||||
save_path: Optional[str],
|
||||
) -> Optional[str]:
|
||||
data = {'category': category or self._category}
|
||||
cat = category or self._category
|
||||
before = self._all_hashes()
|
||||
if before is None:
|
||||
return None
|
||||
data = {'category': cat}
|
||||
if save_path or self._save_path:
|
||||
data['savepath'] = save_path or self._save_path
|
||||
files = {'torrents': ('soulsync.torrent', file_bytes, 'application/x-bittorrent')}
|
||||
resp = self._call('POST', '/api/v2/torrents/add', data=data, files=files)
|
||||
if not resp or not resp.ok:
|
||||
return None
|
||||
return self._lookup_latest_hash(category or self._category)
|
||||
return self._poll_for_new_hash(before)
|
||||
|
||||
async def get_status(self, torrent_id: str) -> Optional[TorrentStatus]:
|
||||
loop = asyncio.get_event_loop()
|
||||
|
|
|
|||
|
|
@ -64,6 +64,17 @@ def set_album_api_track_count(cursor, album_id, count):
|
|||
(count, album_id),
|
||||
)
|
||||
except Exception as e:
|
||||
if "api_track_count" in str(e) and "no such column" in str(e).lower():
|
||||
try:
|
||||
cursor.execute("ALTER TABLE albums ADD COLUMN api_track_count INTEGER DEFAULT NULL")
|
||||
cursor.execute(
|
||||
"UPDATE albums SET api_track_count = ? WHERE id = ?",
|
||||
(count, album_id),
|
||||
)
|
||||
logger.info("Repaired missing api_track_count column while caching album track count")
|
||||
return
|
||||
except Exception as repair_error:
|
||||
e = repair_error
|
||||
logger.warning(
|
||||
"Failed to cache api_track_count for album %s: %s", album_id, e
|
||||
)
|
||||
|
|
|
|||
|
|
@ -766,6 +766,8 @@ class MusicDatabase:
|
|||
except Exception as ps_err:
|
||||
logger.error(f"Personalized-playlist schema init failed: {ps_err}")
|
||||
|
||||
self._ensure_core_media_schema_columns(cursor)
|
||||
|
||||
conn.commit()
|
||||
logger.info("Database initialized successfully")
|
||||
|
||||
|
|
@ -775,6 +777,29 @@ class MusicDatabase:
|
|||
|
||||
self._init_manual_library_match_table()
|
||||
|
||||
def _ensure_core_media_schema_columns(self, cursor):
|
||||
"""Repair required media-library columns that older migrations may miss.
|
||||
|
||||
A few legacy migrations rebuild artists/albums/tracks in place. Newer
|
||||
installs get these columns from CREATE TABLE, but upgraded databases can
|
||||
occasionally miss one if a previous migration path failed or was marked
|
||||
complete before the column existed.
|
||||
"""
|
||||
try:
|
||||
cursor.execute("PRAGMA table_info(tracks)")
|
||||
track_cols = {c[1] for c in cursor.fetchall()}
|
||||
if track_cols and 'file_size' not in track_cols:
|
||||
cursor.execute("ALTER TABLE tracks ADD COLUMN file_size INTEGER")
|
||||
logger.info("Repaired missing file_size column on tracks table")
|
||||
|
||||
cursor.execute("PRAGMA table_info(albums)")
|
||||
album_cols = {c[1] for c in cursor.fetchall()}
|
||||
if album_cols and 'api_track_count' not in album_cols:
|
||||
cursor.execute("ALTER TABLE albums ADD COLUMN api_track_count INTEGER DEFAULT NULL")
|
||||
logger.info("Repaired missing api_track_count column on albums table")
|
||||
except Exception as e:
|
||||
logger.error("Error repairing core media schema columns: %s", e)
|
||||
|
||||
def _add_mirrored_playlist_explored_column(self, cursor):
|
||||
"""Add explored_at column to mirrored_playlists to persist explore badge."""
|
||||
try:
|
||||
|
|
@ -4566,45 +4591,72 @@ class MusicDatabase:
|
|||
|
||||
# VACUUM to actually shrink the database file and reclaim disk space
|
||||
logger.info("Vacuuming database to reclaim disk space...")
|
||||
cursor.execute("VACUUM")
|
||||
self._vacuum_best_effort(cursor)
|
||||
|
||||
logger.info("All database data cleared and file compacted")
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error clearing database: {e}")
|
||||
raise
|
||||
|
||||
def _vacuum_best_effort(self, cursor):
|
||||
"""Run VACUUM without making the caller fail if compaction hiccups."""
|
||||
try:
|
||||
cursor.execute("VACUUM")
|
||||
except Exception as e:
|
||||
logger.warning(
|
||||
"Database VACUUM failed after data was already cleared; continuing without compaction: %s",
|
||||
e,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _is_transient_sqlite_io_error(exc: Exception) -> bool:
|
||||
return "disk i/o error" in str(exc).lower()
|
||||
|
||||
def clear_server_data(self, server_source: str):
|
||||
"""Clear data for specific server only (server-aware full refresh)"""
|
||||
try:
|
||||
with self._get_connection() as conn:
|
||||
cursor = conn.cursor()
|
||||
|
||||
# Delete only data from the specified server
|
||||
# Order matters: tracks -> albums -> artists (foreign key constraints)
|
||||
cursor.execute("DELETE FROM tracks WHERE server_source = ?", (server_source,))
|
||||
tracks_deleted = cursor.rowcount
|
||||
|
||||
cursor.execute("DELETE FROM albums WHERE server_source = ?", (server_source,))
|
||||
albums_deleted = cursor.rowcount
|
||||
|
||||
cursor.execute("DELETE FROM artists WHERE server_source = ?", (server_source,))
|
||||
artists_deleted = cursor.rowcount
|
||||
|
||||
conn.commit()
|
||||
|
||||
# Only VACUUM if we deleted a significant amount of data
|
||||
if tracks_deleted > 1000 or albums_deleted > 100:
|
||||
logger.info("Vacuuming database to reclaim disk space...")
|
||||
cursor.execute("VACUUM")
|
||||
|
||||
logger.info(f"Cleared {server_source} data: {artists_deleted} artists, {albums_deleted} albums, {tracks_deleted} tracks")
|
||||
|
||||
# Note: Watchlist and wishlist are preserved as they are server-agnostic
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error clearing {server_source} database data: {e}")
|
||||
raise
|
||||
for attempt in range(2):
|
||||
try:
|
||||
with self._get_connection() as conn:
|
||||
cursor = conn.cursor()
|
||||
|
||||
# Delete only data from the specified server
|
||||
# Order matters: tracks -> albums -> artists (foreign key constraints)
|
||||
cursor.execute("DELETE FROM tracks WHERE server_source = ?", (server_source,))
|
||||
tracks_deleted = cursor.rowcount
|
||||
|
||||
cursor.execute("DELETE FROM albums WHERE server_source = ?", (server_source,))
|
||||
albums_deleted = cursor.rowcount
|
||||
|
||||
cursor.execute("DELETE FROM artists WHERE server_source = ?", (server_source,))
|
||||
artists_deleted = cursor.rowcount
|
||||
|
||||
conn.commit()
|
||||
|
||||
# Only VACUUM if we deleted a significant amount of data
|
||||
if tracks_deleted > 1000 or albums_deleted > 100:
|
||||
logger.info("Vacuuming database to reclaim disk space...")
|
||||
self._vacuum_best_effort(cursor)
|
||||
|
||||
logger.info(
|
||||
f"Cleared {server_source} data: {artists_deleted} artists, "
|
||||
f"{albums_deleted} albums, {tracks_deleted} tracks"
|
||||
)
|
||||
|
||||
# Note: Watchlist and wishlist are preserved as they are server-agnostic
|
||||
return
|
||||
|
||||
except Exception as e:
|
||||
if self._is_transient_sqlite_io_error(e) and attempt == 0:
|
||||
logger.warning(
|
||||
"Transient disk I/O error clearing %s database data; retrying once: %s",
|
||||
server_source,
|
||||
e,
|
||||
)
|
||||
time.sleep(0.25)
|
||||
continue
|
||||
logger.error(f"Error clearing {server_source} database data: {e}")
|
||||
raise
|
||||
|
||||
def cleanup_orphaned_records(self) -> Dict[str, int]:
|
||||
"""Remove artists and albums that have no associated tracks"""
|
||||
|
|
@ -5499,6 +5551,25 @@ class MusicDatabase:
|
|||
return True
|
||||
|
||||
except Exception as e:
|
||||
error_text = str(e).lower()
|
||||
if (
|
||||
'file_size' in error_text
|
||||
and ('no such column' in error_text or 'no column named' in error_text)
|
||||
and retry_count < max_retries - 1
|
||||
):
|
||||
try:
|
||||
repair_conn = conn if 'conn' in locals() else self._get_connection()
|
||||
repair_cursor = repair_conn.cursor()
|
||||
self._ensure_core_media_schema_columns(repair_cursor)
|
||||
repair_conn.commit()
|
||||
if repair_conn is not conn:
|
||||
repair_conn.close()
|
||||
retry_count += 1
|
||||
logger.info("Repaired missing file_size column while importing media track; retrying")
|
||||
continue
|
||||
except Exception as schema_error:
|
||||
logger.error("Failed to repair tracks.file_size during track import: %s", schema_error)
|
||||
|
||||
retry_count += 1
|
||||
if "database is locked" in str(e).lower() and retry_count < max_retries:
|
||||
logger.warning(f"Database locked on track '{getattr(track_obj, 'title', 'Unknown')}', retrying {retry_count}/{max_retries}...")
|
||||
|
|
|
|||
|
|
@ -48,6 +48,26 @@ services:
|
|||
# - /path/to/music-library:/host/music:rw
|
||||
# Optional: Mount your music library for Plex/Jellyfin/Navidrome access
|
||||
# - /path/to/your/music:/music:ro
|
||||
#
|
||||
# ─── Torrent / Usenet download sources (Prowlarr integration) ────────
|
||||
# If you use the Torrent Only or Usenet Only download sources, SoulSync
|
||||
# needs to read the files your torrent / usenet client downloads.
|
||||
#
|
||||
# EASIEST SETUP: point qBittorrent, SABnzbd / NZBGet, AND slskd at the
|
||||
# SAME download folder. The ./downloads mount above is reused; nothing
|
||||
# new to add here. Just configure each client's download path to
|
||||
# /downloads (inside its own container) — files land in the same place
|
||||
# SoulSync already reads from.
|
||||
#
|
||||
# SEPARATE FOLDERS (if you prefer): uncomment + edit these to point at
|
||||
# the client's download folder. The in-container path must MATCH the
|
||||
# save_path the client reports.
|
||||
# - /path/to/qbittorrent/downloads:/downloads/torrents:rw
|
||||
# - /path/to/sabnzbd/complete:/downloads/usenet:rw
|
||||
# Then set the torrent client's save path to /downloads/torrents and
|
||||
# SAB / NZBGet's complete folder to /downloads/usenet (or whatever
|
||||
# in-container path you chose). Skip these if your downloader runs
|
||||
# outside Docker and writes to a path already mounted above.
|
||||
extra_hosts:
|
||||
# Allow container to reach host services
|
||||
- "host.docker.internal:host-gateway"
|
||||
|
|
|
|||
76
pr_description.md
Normal file
76
pr_description.md
Normal file
|
|
@ -0,0 +1,76 @@
|
|||
## Summary
|
||||
|
||||
Adds torrent and usenet as release-oriented download sources backed by Prowlarr and the configured downloader clients. These sources can download full releases, stage the resulting audio files, and let SoulSync match/import the requested tracks through the existing post-processing pipeline.
|
||||
|
||||
This PR also improves the torrent/usenet user experience with clearer release-download progress, source-aware service labels, library history provenance, and safer staged-release matching for album files that include featured-artist or bonus-track filename noise.
|
||||
|
||||
## Scope
|
||||
|
||||
- Adds torrent and usenet plugin support for release downloads.
|
||||
- Adds album-bundle staging for single-source torrent/usenet album downloads.
|
||||
- Keeps hybrid album downloads on per-track-capable sources by excluding torrent/usenet from hybrid album per-track searches.
|
||||
- Allows torrent/usenet in hybrid for non-album track downloads, such as playlist singles and wishlist tracks.
|
||||
- Selects the requested audio file from completed release folders instead of importing the first audio file blindly.
|
||||
- Reports live album-bundle progress to the Downloads page and download modal.
|
||||
- Records torrent/usenet provenance in library history.
|
||||
- Adds UI polish for release-first download states.
|
||||
|
||||
## Behavior Gates
|
||||
|
||||
- Users who do not select torrent or usenet as a download source should not hit the new download paths.
|
||||
- Single-source `torrent` / `usenet` album downloads use the release staging flow.
|
||||
- Hybrid album downloads skip torrent/usenet and continue through the existing per-track source chain.
|
||||
- Hybrid non-album downloads may use torrent/usenet when they are included in the hybrid order.
|
||||
|
||||
## Notable Implementation Details
|
||||
|
||||
- Torrent/usenet release downloads use private per-batch staging folders under the configured album-bundle staging root.
|
||||
- Post-processing receives the real source label (`torrent` / `usenet`) so library history no longer falls back to `Soulseek`.
|
||||
- Staging matching strips only conservative noise like `(feat. Artist)` and `(Bonus Track)` while preserving meaningful version text like `remix`, `extended`, `live`, and `acoustic`.
|
||||
- When a staged release does not contain a requested track, the task is marked not found instead of repeatedly searching/downloading the same release.
|
||||
- Completed release downloads expose all discovered audio files so post-processing can choose the best matching track.
|
||||
|
||||
## Testing
|
||||
|
||||
Manually tested:
|
||||
|
||||
- Torrent-only album download for `good kid, m.A.A.d city (Deluxe)`.
|
||||
- Torrent playlist/single-track flow.
|
||||
- Album-bundle progress in the download modal and Downloads page.
|
||||
- Torrent source labeling in library history for new imports.
|
||||
- Staging match behavior for featured-artist filenames and bonus-track labels.
|
||||
- Quarantine behavior for wrong-version matches.
|
||||
|
||||
Automated tests run during development:
|
||||
|
||||
```bash
|
||||
.venv/bin/python -m pytest \
|
||||
tests/downloads/test_downloads_status.py \
|
||||
tests/test_album_bundle_dispatch.py \
|
||||
tests/downloads/test_downloads_staging.py \
|
||||
tests/test_torrent_usenet_plugins.py
|
||||
```
|
||||
|
||||
```bash
|
||||
.venv/bin/python -m pytest \
|
||||
tests/downloads/test_downloads_validation.py \
|
||||
tests/test_manual_pick_no_auto_retry.py \
|
||||
tests/downloads/test_downloads_post_processing.py \
|
||||
tests/downloads/test_downloads_task_worker.py \
|
||||
tests/imports/test_import_side_effects.py
|
||||
```
|
||||
|
||||
Focused checks also passed for:
|
||||
|
||||
- staged release feature suffix matching
|
||||
- bonus-track title matching
|
||||
- wrong-version separation
|
||||
- private torrent album staging miss handling
|
||||
- torrent/usenet history source labels
|
||||
|
||||
## Reviewer Notes
|
||||
|
||||
- This is intentionally gated behind torrent/usenet source selection, but it is still a new release-oriented download path and should be considered beta/experimental for first release.
|
||||
- Remote downloader setups need SoulSync to be able to read the downloader save path. Local/all-in-one setups should be the easiest path.
|
||||
- Existing library history rows that were previously recorded as `Soulseek` are not backfilled by this PR.
|
||||
- Release matching is naturally fuzzier than track-native sources, so reviewer focus should stay on false positives, version handling, and staged-file selection.
|
||||
|
|
@ -329,6 +329,48 @@ def test_batch_completion_emits_batch_complete_when_all_done():
|
|||
assert 'b1' in monitor.stopped
|
||||
|
||||
|
||||
def test_batch_completion_cleans_private_album_bundle_staging(tmp_path):
|
||||
staging_dir = tmp_path / 'b1'
|
||||
staging_dir.mkdir()
|
||||
(staging_dir / 'leftover.flac').write_bytes(b'audio')
|
||||
|
||||
download_tasks['t1'] = {'status': 'completed', 'track_info': {'name': 'X'}}
|
||||
download_batches['b1'] = {
|
||||
'queue': ['t1'], 'queue_index': 1, 'active_count': 1,
|
||||
'max_concurrent': 1, 'permanently_failed_tracks': [],
|
||||
'cancelled_tracks': set(),
|
||||
'album_bundle_private_staging': True,
|
||||
'album_bundle_source': 'torrent',
|
||||
'album_bundle_staging_path': str(staging_dir),
|
||||
}
|
||||
deps, _ = _build_deps()
|
||||
|
||||
lc.on_download_completed('b1', 't1', True, deps)
|
||||
|
||||
assert not staging_dir.exists()
|
||||
|
||||
|
||||
def test_batch_completion_keeps_unexpected_staging_path(tmp_path):
|
||||
staging_dir = tmp_path / 'shared-staging'
|
||||
staging_dir.mkdir()
|
||||
(staging_dir / 'leftover.flac').write_bytes(b'audio')
|
||||
|
||||
download_tasks['t1'] = {'status': 'completed', 'track_info': {'name': 'X'}}
|
||||
download_batches['b1'] = {
|
||||
'queue': ['t1'], 'queue_index': 1, 'active_count': 1,
|
||||
'max_concurrent': 1, 'permanently_failed_tracks': [],
|
||||
'cancelled_tracks': set(),
|
||||
'album_bundle_private_staging': True,
|
||||
'album_bundle_source': 'torrent',
|
||||
'album_bundle_staging_path': str(staging_dir),
|
||||
}
|
||||
deps, _ = _build_deps()
|
||||
|
||||
lc.on_download_completed('b1', 't1', True, deps)
|
||||
|
||||
assert staging_dir.exists()
|
||||
|
||||
|
||||
def test_batch_completion_skips_emit_when_zero_successful():
|
||||
"""Don't emit batch_complete if nothing actually downloaded."""
|
||||
download_tasks['t1'] = {'status': 'failed', 'track_info': {'name': 'X'}, 'track_index': 0}
|
||||
|
|
|
|||
|
|
@ -232,6 +232,28 @@ def test_file_found_in_downloads_with_context_runs_post_process_with_verificatio
|
|||
assert any(c[0] == 'post_process' for c in rec.calls)
|
||||
|
||||
|
||||
def test_file_search_ignores_non_audio_candidates(monkeypatch):
|
||||
download_tasks['t1'] = {
|
||||
'status': 'post_processing',
|
||||
'filename': 'Artist - Album.cue',
|
||||
'username': 'torrent',
|
||||
'track_info': {'name': 'Money'},
|
||||
}
|
||||
matched_downloads_context['torrent::Artist - Album.cue'] = {
|
||||
'original_search_result': {'title': 'Money', 'track_number': 1},
|
||||
}
|
||||
monkeypatch.setattr(pp.time, 'sleep', lambda s: None)
|
||||
deps, rec = _build_deps(
|
||||
find_completed_file=lambda *a, **kw: ('/downloads/Artist - Album.cue', 'download'),
|
||||
)
|
||||
|
||||
pp.run_post_processing_worker('t1', 'b1', deps)
|
||||
|
||||
assert download_tasks['t1']['status'] == 'failed'
|
||||
assert not any(c[0] == 'post_process' for c in rec.calls)
|
||||
assert ('on_complete', ('b1', 't1', False), {}) in rec.calls
|
||||
|
||||
|
||||
def test_file_found_in_downloads_no_context_marks_completed_directly():
|
||||
"""No matched context for the file → just mark completed since file exists."""
|
||||
download_tasks['t1'] = {
|
||||
|
|
@ -323,6 +345,139 @@ def test_youtube_task_uses_get_download_status_to_resolve_path(monkeypatch):
|
|||
assert any(c[0] == 'mark_completed' for c in rec.calls)
|
||||
|
||||
|
||||
def test_torrent_release_copies_best_matching_audio_to_transfer(tmp_path):
|
||||
release_dir = tmp_path / 'release'
|
||||
release_dir.mkdir()
|
||||
wrong = release_dir / '01 - Intro.flac'
|
||||
right = release_dir / '02 - Money.flac'
|
||||
wrong.write_bytes(b'wrong')
|
||||
right.write_bytes(b'right')
|
||||
transfer_dir = tmp_path / 'transfer'
|
||||
|
||||
filename = 'magnet:?xt=abc||Artist - Album'
|
||||
download_tasks['t1'] = {
|
||||
'status': 'post_processing',
|
||||
'filename': filename,
|
||||
'username': 'torrent',
|
||||
'download_id': 'dl-torrent-1',
|
||||
'track_info': {'name': 'Money', 'artists': [{'name': 'Artist'}]},
|
||||
}
|
||||
matched_downloads_context[f'torrent::{filename}'] = {
|
||||
'original_search_result': {'title': 'Money', 'track_number': 2},
|
||||
}
|
||||
|
||||
class _FakeStatus:
|
||||
file_path = str(wrong)
|
||||
audio_files = [str(wrong), str(right)]
|
||||
|
||||
class _FakeTorrentClient:
|
||||
def get_download_status(self, dl_id):
|
||||
assert dl_id == 'dl-torrent-1'
|
||||
return _FakeStatus()
|
||||
|
||||
deps, rec = _build_deps(
|
||||
config=_FakeConfig({'soulseek.transfer_path': str(transfer_dir)}),
|
||||
download_orchestrator=_FakeTorrentClient(),
|
||||
run_async=lambda coro: coro,
|
||||
)
|
||||
|
||||
pp.run_post_processing_worker('t1', 'b1', deps)
|
||||
|
||||
copied = transfer_dir / '02 - Money.flac'
|
||||
assert copied.exists()
|
||||
assert right.exists()
|
||||
assert any(c[0] == 'post_process' and c[1][2] == str(copied) for c in rec.calls)
|
||||
|
||||
|
||||
def test_torrent_release_prefers_task_title_over_release_context(tmp_path):
|
||||
release_dir = tmp_path / 'release'
|
||||
release_dir.mkdir()
|
||||
wrong = release_dir / '09.Harry Styles - Pop.flac'
|
||||
right = release_dir / '10.Harry Styles - American Girls.flac'
|
||||
wrong.write_bytes(b'wrong')
|
||||
right.write_bytes(b'right')
|
||||
transfer_dir = tmp_path / 'transfer'
|
||||
|
||||
filename = 'http://prowlarr/download?id=1||Harry Styles - Kiss All The Time'
|
||||
download_tasks['t1'] = {
|
||||
'status': 'post_processing',
|
||||
'filename': filename,
|
||||
'username': 'torrent',
|
||||
'download_id': 'dl-torrent-1',
|
||||
'track_info': {'name': 'American Girls', 'artists': [{'name': 'Harry Styles'}]},
|
||||
}
|
||||
matched_downloads_context[f'torrent::{filename}'] = {
|
||||
'original_search_result': {'title': 'Pop', 'clean_title': 'Pop', 'track_number': 9},
|
||||
}
|
||||
|
||||
class _FakeStatus:
|
||||
file_path = str(wrong)
|
||||
audio_files = [str(wrong), str(right)]
|
||||
|
||||
class _FakeTorrentClient:
|
||||
def get_download_status(self, dl_id):
|
||||
assert dl_id == 'dl-torrent-1'
|
||||
return _FakeStatus()
|
||||
|
||||
deps, rec = _build_deps(
|
||||
config=_FakeConfig({'soulseek.transfer_path': str(transfer_dir)}),
|
||||
download_orchestrator=_FakeTorrentClient(),
|
||||
run_async=lambda coro: coro,
|
||||
)
|
||||
|
||||
pp.run_post_processing_worker('t1', 'b1', deps)
|
||||
|
||||
copied = transfer_dir / '10.Harry Styles - American Girls.flac'
|
||||
assert copied.exists()
|
||||
assert any(c[0] == 'post_process' and c[1][2] == str(copied) for c in rec.calls)
|
||||
|
||||
|
||||
def test_torrent_release_without_matching_file_does_not_fallback_to_generic_search(tmp_path):
|
||||
release_dir = tmp_path / 'release'
|
||||
release_dir.mkdir()
|
||||
wrong = release_dir / '09.Harry Styles - Pop.flac'
|
||||
wrong.write_bytes(b'wrong')
|
||||
transfer_dir = tmp_path / 'transfer'
|
||||
|
||||
filename = 'http://prowlarr/download?id=1||Harry Styles - Kiss All The Time'
|
||||
download_tasks['t1'] = {
|
||||
'status': 'post_processing',
|
||||
'filename': filename,
|
||||
'username': 'torrent',
|
||||
'download_id': 'dl-torrent-1',
|
||||
'track_info': {'name': 'American Girls', 'artists': [{'name': 'Harry Styles'}]},
|
||||
}
|
||||
matched_downloads_context[f'torrent::{filename}'] = {
|
||||
'original_search_result': {'title': 'Pop', 'clean_title': 'Pop', 'track_number': 9},
|
||||
}
|
||||
|
||||
class _FakeStatus:
|
||||
file_path = str(wrong)
|
||||
audio_files = [str(wrong)]
|
||||
|
||||
class _FakeTorrentClient:
|
||||
def get_download_status(self, dl_id):
|
||||
assert dl_id == 'dl-torrent-1'
|
||||
return _FakeStatus()
|
||||
|
||||
def _unexpected_search(*args, **kwargs):
|
||||
raise AssertionError("torrent releases should not fall back to generic file search")
|
||||
|
||||
deps, rec = _build_deps(
|
||||
config=_FakeConfig({'soulseek.transfer_path': str(transfer_dir)}),
|
||||
download_orchestrator=_FakeTorrentClient(),
|
||||
run_async=lambda coro: coro,
|
||||
find_completed_file=_unexpected_search,
|
||||
)
|
||||
|
||||
pp.run_post_processing_worker('t1', 'b1', deps)
|
||||
|
||||
assert download_tasks['t1']['status'] == 'failed'
|
||||
assert 'No matching audio file' in download_tasks['t1']['error_message']
|
||||
assert any(c[0] == 'on_complete' and c[1] == ('b1', 't1', False) for c in rec.calls)
|
||||
assert not list(transfer_dir.glob('*'))
|
||||
|
||||
|
||||
def test_fuzzy_context_matching_when_exact_key_missing(monkeypatch):
|
||||
"""When exact key isn't in matched_downloads_context, worker tries fuzzy match
|
||||
constrained to same Soulseek username."""
|
||||
|
|
|
|||
|
|
@ -60,6 +60,7 @@ def _build_deps(
|
|||
transfer_path,
|
||||
staging_files=None,
|
||||
post_process_calls=None,
|
||||
get_batch_field=None,
|
||||
):
|
||||
post_process_calls = post_process_calls if post_process_calls is not None else []
|
||||
deps = ds.StagingDeps(
|
||||
|
|
@ -68,6 +69,7 @@ def _build_deps(
|
|||
get_staging_file_cache=lambda batch_id: staging_files or [],
|
||||
docker_resolve_path=lambda p: p, # passthrough
|
||||
post_process_matched_download_with_verification=lambda *a, **kw: post_process_calls.append((a, kw)),
|
||||
get_batch_field=get_batch_field,
|
||||
)
|
||||
deps._post_process_calls = post_process_calls
|
||||
return deps
|
||||
|
|
@ -157,6 +159,56 @@ def test_exact_match_copies_to_transfer_and_marks_post_processing(tmp_path):
|
|||
assert context_key == 'staging_t4'
|
||||
|
||||
|
||||
def test_private_album_bundle_staging_source_is_removed_after_claim(tmp_path):
|
||||
src_file = tmp_path / 'private' / 'Hello.flac'
|
||||
src_file.parent.mkdir()
|
||||
src_file.write_bytes(b'fake audio')
|
||||
|
||||
transfer_dir = tmp_path / 'transfer'
|
||||
|
||||
def get_batch_field(_batch_id, field):
|
||||
if field == 'album_bundle_source':
|
||||
return 'torrent'
|
||||
if field == 'album_bundle_private_staging':
|
||||
return True
|
||||
return None
|
||||
|
||||
deps = _build_deps(
|
||||
transfer_path=str(transfer_dir),
|
||||
staging_files=[
|
||||
{'full_path': str(src_file), 'title': 'Hello', 'artist': 'Artist One'},
|
||||
],
|
||||
get_batch_field=get_batch_field,
|
||||
)
|
||||
_seed_task('t_private')
|
||||
|
||||
result = ds.try_staging_match('t_private', 'b_private', _Track(), deps)
|
||||
|
||||
assert result is True
|
||||
assert (transfer_dir / 'Hello.flac').exists()
|
||||
assert not src_file.exists()
|
||||
assert download_tasks['t_private']['username'] == 'torrent'
|
||||
|
||||
|
||||
def test_public_staging_source_is_kept_after_match(tmp_path):
|
||||
src_file = tmp_path / 'staging' / 'Hello.flac'
|
||||
src_file.parent.mkdir()
|
||||
src_file.write_bytes(b'fake audio')
|
||||
|
||||
deps = _build_deps(
|
||||
transfer_path=str(tmp_path / 'transfer'),
|
||||
staging_files=[
|
||||
{'full_path': str(src_file), 'title': 'Hello', 'artist': 'Artist One'},
|
||||
],
|
||||
)
|
||||
_seed_task('t_public')
|
||||
|
||||
result = ds.try_staging_match('t_public', 'b_public', _Track(), deps)
|
||||
|
||||
assert result is True
|
||||
assert src_file.exists()
|
||||
|
||||
|
||||
def test_existing_file_in_transfer_gets_staging_suffix(tmp_path):
|
||||
"""If destination already exists, suffix '_staging' added to avoid overwrite."""
|
||||
src_file = tmp_path / 'staging' / 'Hello.flac'
|
||||
|
|
@ -226,6 +278,180 @@ def test_explicit_album_context_uses_real_data(tmp_path):
|
|||
assert ctx['staging_source'] is True
|
||||
|
||||
|
||||
def test_staging_context_falls_back_to_matched_file_track_number(tmp_path):
|
||||
"""Album-bundle staging can recover numbering from the selected audio file."""
|
||||
src_file = tmp_path / 'staging' / '03 - Backseat Freestyle.flac'
|
||||
src_file.parent.mkdir()
|
||||
src_file.touch()
|
||||
|
||||
deps = _build_deps(
|
||||
transfer_path=str(tmp_path / 'transfer'),
|
||||
staging_files=[
|
||||
{
|
||||
'full_path': str(src_file),
|
||||
'title': 'Backseat Freestyle',
|
||||
'artist': 'Kendrick Lamar',
|
||||
'track_number': 3,
|
||||
'disc_number': 1,
|
||||
},
|
||||
],
|
||||
)
|
||||
_seed_task('t6b', track_info={
|
||||
'_is_explicit_album_download': True,
|
||||
'_explicit_album_context': {'id': 'alb', 'name': 'good kid, m.A.A.d city (Deluxe)'},
|
||||
'_explicit_artist_context': {'id': 'art', 'name': 'Kendrick Lamar'},
|
||||
})
|
||||
|
||||
ds.try_staging_match(
|
||||
't6b', 'b1',
|
||||
_Track(name='Backseat Freestyle', artists=['Kendrick Lamar']),
|
||||
deps,
|
||||
)
|
||||
|
||||
ctx = matched_downloads_context['staging_t6b']
|
||||
assert ctx['original_search_result']['track_number'] == 3
|
||||
assert ctx['original_search_result']['disc_number'] == 1
|
||||
|
||||
|
||||
def test_private_album_bundle_staging_overrides_default_track_info_number(tmp_path):
|
||||
"""Private release staging trusts the selected file number over weak task defaults."""
|
||||
src_file = tmp_path / 'staging' / '04-kendrick_lamar-the_art_of_peer_pressure.flac'
|
||||
src_file.parent.mkdir()
|
||||
src_file.touch()
|
||||
|
||||
def get_batch_field(_batch_id, field):
|
||||
if field == 'album_bundle_source':
|
||||
return 'torrent'
|
||||
if field == 'album_bundle_private_staging':
|
||||
return True
|
||||
return None
|
||||
|
||||
deps = _build_deps(
|
||||
transfer_path=str(tmp_path / 'transfer'),
|
||||
staging_files=[
|
||||
{
|
||||
'full_path': str(src_file),
|
||||
'title': 'The Art of Peer Pressure',
|
||||
'artist': 'Kendrick Lamar',
|
||||
},
|
||||
],
|
||||
get_batch_field=get_batch_field,
|
||||
)
|
||||
_seed_task('t6c', track_info={
|
||||
'_is_explicit_album_download': True,
|
||||
'_explicit_album_context': {'id': 'alb', 'name': 'good kid, m.A.A.d city (Deluxe)'},
|
||||
'_explicit_artist_context': {'id': 'art', 'name': 'Kendrick Lamar'},
|
||||
'track_number': 1,
|
||||
})
|
||||
|
||||
ds.try_staging_match(
|
||||
't6c', 'b1',
|
||||
_Track(name='The Art of Peer Pressure', artists=['Kendrick Lamar']),
|
||||
deps,
|
||||
)
|
||||
|
||||
ctx = matched_downloads_context['staging_t6c']
|
||||
assert ctx['track_info']['track_number'] == 4
|
||||
assert ctx['original_search_result']['track_number'] == 4
|
||||
assert ctx['original_search_result']['username'] == 'torrent'
|
||||
assert ctx['original_search_result']['filename'] == str(src_file)
|
||||
|
||||
|
||||
def test_staging_title_match_accepts_feature_suffix_from_release_file(tmp_path):
|
||||
"""Album releases can include featured artists in filenames."""
|
||||
src_file = tmp_path / 'staging' / '05-kendrick_lamar-money_trees_(feat._jay_rock).flac'
|
||||
src_file.parent.mkdir()
|
||||
src_file.touch()
|
||||
|
||||
deps = _build_deps(
|
||||
transfer_path=str(tmp_path / 'transfer'),
|
||||
staging_files=[
|
||||
{
|
||||
'full_path': str(src_file),
|
||||
'title': 'money_trees_(feat._jay_rock)',
|
||||
'artist': 'Kendrick Lamar',
|
||||
'track_number': 5,
|
||||
},
|
||||
],
|
||||
)
|
||||
_seed_task('t_feature', track_info={
|
||||
'_is_explicit_album_download': True,
|
||||
'_explicit_album_context': {'id': 'alb', 'name': 'good kid, m.A.A.d city (Deluxe)'},
|
||||
'_explicit_artist_context': {'id': 'art', 'name': 'Kendrick Lamar'},
|
||||
})
|
||||
|
||||
result = ds.try_staging_match(
|
||||
't_feature', 'b1',
|
||||
_Track(name='Money Trees', artists=['Kendrick Lamar']),
|
||||
deps,
|
||||
)
|
||||
|
||||
assert result is True
|
||||
assert matched_downloads_context['staging_t_feature']['track_info']['track_number'] == 5
|
||||
|
||||
|
||||
def test_staging_title_match_accepts_bonus_track_against_release_file(tmp_path):
|
||||
"""Expected bonus labels should not block matching the actual release file."""
|
||||
src_file = tmp_path / 'staging' / '13-kendrick_lamar-the_recipe_(feat._dr._dre).flac'
|
||||
src_file.parent.mkdir()
|
||||
src_file.touch()
|
||||
|
||||
deps = _build_deps(
|
||||
transfer_path=str(tmp_path / 'transfer'),
|
||||
staging_files=[
|
||||
{
|
||||
'full_path': str(src_file),
|
||||
'title': 'the_recipe_(feat._dr._dre)',
|
||||
'artist': 'Kendrick Lamar',
|
||||
'track_number': 13,
|
||||
},
|
||||
],
|
||||
)
|
||||
_seed_task('t_bonus', track_info={
|
||||
'_is_explicit_album_download': True,
|
||||
'_explicit_album_context': {'id': 'alb', 'name': 'good kid, m.A.A.d city (Deluxe)'},
|
||||
'_explicit_artist_context': {'id': 'art', 'name': 'Kendrick Lamar'},
|
||||
})
|
||||
|
||||
result = ds.try_staging_match(
|
||||
't_bonus', 'b1',
|
||||
_Track(name='The Recipe (Bonus Track)', artists=['Kendrick Lamar']),
|
||||
deps,
|
||||
)
|
||||
|
||||
assert result is True
|
||||
assert matched_downloads_context['staging_t_bonus']['track_info']['track_number'] == 13
|
||||
|
||||
|
||||
def test_staging_title_match_keeps_wrong_versions_separate(tmp_path):
|
||||
"""Do not strip remix/extended wording when matching staged release files."""
|
||||
src_file = tmp_path / 'staging' / '17-kendrick_lamar-swimming_pools_(drank)_(black_hippy_remix).flac'
|
||||
src_file.parent.mkdir()
|
||||
src_file.touch()
|
||||
|
||||
deps = _build_deps(
|
||||
transfer_path=str(tmp_path / 'transfer'),
|
||||
staging_files=[
|
||||
{
|
||||
'full_path': str(src_file),
|
||||
'title': 'swimming_pools_(drank)_(black_hippy_remix)',
|
||||
'artist': 'Kendrick Lamar',
|
||||
'track_number': 17,
|
||||
},
|
||||
],
|
||||
)
|
||||
_seed_task('t_wrong_version')
|
||||
|
||||
result = ds.try_staging_match(
|
||||
't_wrong_version', 'b1',
|
||||
_Track(name='Swimming Pools (Drank) (Extended Version)', artists=['Kendrick Lamar']),
|
||||
deps,
|
||||
)
|
||||
|
||||
assert result is False
|
||||
assert 'staging_t_wrong_version' not in matched_downloads_context
|
||||
|
||||
|
||||
def test_fallback_context_synthesizes_from_track(tmp_path):
|
||||
"""Without explicit context, synthesizes spotify_artist/album from the track."""
|
||||
src_file = tmp_path / 'staging' / 'Hello.flac'
|
||||
|
|
|
|||
|
|
@ -40,6 +40,8 @@ def _build_deps(
|
|||
make_key=None,
|
||||
submit_pp=None,
|
||||
cached_transfers=None,
|
||||
download_orchestrator=None,
|
||||
run_async=None,
|
||||
):
|
||||
submitted = []
|
||||
|
||||
|
|
@ -53,6 +55,8 @@ def _build_deps(
|
|||
make_context_key=make_key or (lambda u, f: f"{u}::{f}"),
|
||||
submit_post_processing=submit_pp or _default_submit,
|
||||
get_cached_transfer_data=cached_transfers or (lambda: {}),
|
||||
download_orchestrator=download_orchestrator,
|
||||
run_async=run_async,
|
||||
)
|
||||
return deps, submitted
|
||||
|
||||
|
|
@ -85,6 +89,38 @@ def test_analysis_phase_includes_analysis_progress_and_results():
|
|||
assert out['analysis_results'] == [{'track_index': 0, 'found': True}]
|
||||
|
||||
|
||||
def test_album_downloading_phase_exposes_bundle_progress_without_task_safety_valve():
|
||||
deps, _ = _build_deps(config=_FakeConfig({'soulseek.download_timeout': 1}))
|
||||
download_tasks['t1'] = {
|
||||
'track_index': 0,
|
||||
'status': 'queued',
|
||||
'track_info': {},
|
||||
'status_change_time': 0,
|
||||
}
|
||||
batch = {
|
||||
'phase': 'album_downloading',
|
||||
'queue': ['t1'],
|
||||
'album_bundle_state': 'downloading',
|
||||
'album_bundle_source': 'torrent',
|
||||
'album_bundle_release': 'GNX [FLAC]',
|
||||
'album_bundle_progress': 0.42,
|
||||
'album_bundle_speed': 2048,
|
||||
'album_bundle_size': 4096,
|
||||
}
|
||||
out = st.build_batch_status_data('b1', batch, {}, deps)
|
||||
assert out['album_bundle'] == {
|
||||
'state': 'downloading',
|
||||
'source': 'torrent',
|
||||
'release': 'GNX [FLAC]',
|
||||
'progress': 0.42,
|
||||
'progress_percent': 42,
|
||||
'speed': 2048,
|
||||
'size': 4096,
|
||||
}
|
||||
assert 'tasks' not in out
|
||||
assert download_tasks['t1']['status'] == 'queued'
|
||||
|
||||
|
||||
def test_complete_phase_includes_wishlist_summary_when_present():
|
||||
deps, _ = _build_deps()
|
||||
batch = {
|
||||
|
|
@ -254,6 +290,128 @@ def test_post_processing_status_progress_is_95():
|
|||
assert out['tasks'][0]['progress'] == 95
|
||||
|
||||
|
||||
def test_auto_torrent_without_live_entry_uses_engine_success_fallback():
|
||||
class _Record:
|
||||
state = 'Completed, Succeeded'
|
||||
progress = 100
|
||||
|
||||
class _Orchestrator:
|
||||
def get_download_status(self, download_id):
|
||||
assert download_id == 'dl1'
|
||||
return _Record()
|
||||
|
||||
deps, submitted = _build_deps(
|
||||
download_orchestrator=_Orchestrator(),
|
||||
run_async=lambda value: value,
|
||||
)
|
||||
download_tasks['t1'] = {
|
||||
'track_index': 0,
|
||||
'status': 'downloading',
|
||||
'track_info': {},
|
||||
'filename': 'song.flac',
|
||||
'username': 'torrent',
|
||||
'download_id': 'dl1',
|
||||
}
|
||||
batch = {'phase': 'downloading', 'queue': ['t1']}
|
||||
out = st.build_batch_status_data('b1', batch, {}, deps)
|
||||
assert out['tasks'][0]['status'] == 'post_processing'
|
||||
assert download_tasks['t1']['status'] == 'post_processing'
|
||||
assert submitted == [('t1', 'b1')]
|
||||
|
||||
|
||||
def test_auto_torrent_prefers_engine_success_over_live_entry():
|
||||
class _Record:
|
||||
state = 'Completed, Succeeded'
|
||||
progress = 100
|
||||
|
||||
class _Orchestrator:
|
||||
def get_download_status(self, download_id):
|
||||
assert download_id == 'dl1'
|
||||
return _Record()
|
||||
|
||||
deps, submitted = _build_deps(
|
||||
download_orchestrator=_Orchestrator(),
|
||||
run_async=lambda value: value,
|
||||
)
|
||||
download_tasks['t1'] = {
|
||||
'track_index': 0,
|
||||
'status': 'downloading',
|
||||
'track_info': {},
|
||||
'filename': 'song.flac',
|
||||
'username': 'torrent',
|
||||
'download_id': 'dl1',
|
||||
}
|
||||
live = {'torrent::song.flac': {
|
||||
'state': 'InProgress, Downloading',
|
||||
'percentComplete': 100,
|
||||
}}
|
||||
batch = {'phase': 'downloading', 'queue': ['t1']}
|
||||
out = st.build_batch_status_data('b1', batch, live, deps)
|
||||
assert out['tasks'][0]['status'] == 'post_processing'
|
||||
assert download_tasks['t1']['status'] == 'post_processing'
|
||||
assert submitted == [('t1', 'b1')]
|
||||
|
||||
|
||||
def test_auto_torrent_engine_failure_does_not_bypass_monitor_retry():
|
||||
class _Record:
|
||||
state = 'Completed, Errored'
|
||||
progress = 100
|
||||
error = 'client failed'
|
||||
|
||||
class _Orchestrator:
|
||||
def get_download_status(self, download_id):
|
||||
assert download_id == 'dl1'
|
||||
return _Record()
|
||||
|
||||
deps, submitted = _build_deps(
|
||||
download_orchestrator=_Orchestrator(),
|
||||
run_async=lambda value: value,
|
||||
)
|
||||
download_tasks['t1'] = {
|
||||
'track_index': 0,
|
||||
'status': 'downloading',
|
||||
'track_info': {},
|
||||
'filename': 'song.flac',
|
||||
'username': 'torrent',
|
||||
'download_id': 'dl1',
|
||||
}
|
||||
batch = {'phase': 'downloading', 'queue': ['t1']}
|
||||
out = st.build_batch_status_data('b1', batch, {}, deps)
|
||||
assert out['tasks'][0]['status'] == 'downloading'
|
||||
assert download_tasks['t1']['status'] == 'downloading'
|
||||
assert submitted == []
|
||||
|
||||
|
||||
def test_auto_torrent_engine_success_recovers_premature_failed_task():
|
||||
class _Record:
|
||||
state = 'Completed, Succeeded'
|
||||
progress = 100
|
||||
|
||||
class _Orchestrator:
|
||||
def get_download_status(self, download_id):
|
||||
assert download_id == 'dl1'
|
||||
return _Record()
|
||||
|
||||
deps, submitted = _build_deps(
|
||||
download_orchestrator=_Orchestrator(),
|
||||
run_async=lambda value: value,
|
||||
)
|
||||
download_tasks['t1'] = {
|
||||
'track_index': 0,
|
||||
'status': 'failed',
|
||||
'track_info': {},
|
||||
'filename': 'release-url||Release',
|
||||
'username': 'torrent',
|
||||
'download_id': 'dl1',
|
||||
'error_message': 'premature failure',
|
||||
}
|
||||
batch = {'phase': 'downloading', 'queue': ['t1']}
|
||||
out = st.build_batch_status_data('b1', batch, {}, deps)
|
||||
assert out['tasks'][0]['status'] == 'post_processing'
|
||||
assert download_tasks['t1']['status'] == 'post_processing'
|
||||
assert submitted == [('t1', 'b1')]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Safety valve (stuck task handling)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
|
@ -340,6 +498,26 @@ def test_batched_status_no_filter_returns_all_batches():
|
|||
assert set(out['batches'].keys()) == {'b1', 'b2'}
|
||||
|
||||
|
||||
def test_unified_downloads_response_includes_album_bundle_summary():
|
||||
deps, _ = _build_deps()
|
||||
download_batches['b1'] = {
|
||||
'phase': 'album_downloading',
|
||||
'playlist_id': 'pl1',
|
||||
'playlist_name': 'GNX',
|
||||
'queue': [],
|
||||
'album_bundle_state': 'downloading',
|
||||
'album_bundle_source': 'torrent',
|
||||
'album_bundle_progress': 75,
|
||||
}
|
||||
out = st.build_unified_downloads_response(300, deps)
|
||||
assert out['batches'][0]['album_bundle'] == {
|
||||
'state': 'downloading',
|
||||
'source': 'torrent',
|
||||
'progress': 75,
|
||||
'progress_percent': 75,
|
||||
}
|
||||
|
||||
|
||||
def test_batched_status_metadata_present():
|
||||
deps, _ = _build_deps()
|
||||
download_batches['b1'] = {'phase': 'unknown'}
|
||||
|
|
|
|||
|
|
@ -5,14 +5,16 @@ from __future__ import annotations
|
|||
import pytest
|
||||
|
||||
from core.downloads import task_worker as tw
|
||||
from core.runtime_state import download_tasks
|
||||
from core.runtime_state import download_batches, download_tasks
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def reset_state():
|
||||
download_tasks.clear()
|
||||
download_batches.clear()
|
||||
yield
|
||||
download_tasks.clear()
|
||||
download_batches.clear()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
|
|
@ -37,7 +39,7 @@ class _FakeClient:
|
|||
orchestrator); plain attrs (hybrid_order, hybrid_primary, etc.)
|
||||
are set as attributes so getattr() lookups still resolve them."""
|
||||
_CLIENT_NAMES = {'soulseek', 'youtube', 'tidal', 'qobuz', 'hifi',
|
||||
'deezer_dl', 'lidarr', 'soundcloud'}
|
||||
'deezer_dl', 'lidarr', 'soundcloud', 'torrent', 'usenet'}
|
||||
|
||||
def __init__(self, results=None, mode='soulseek', subclients=None):
|
||||
self._results = results if results is not None else []
|
||||
|
|
@ -54,7 +56,7 @@ class _FakeClient:
|
|||
def client(self, name):
|
||||
return self._client_map.get(name)
|
||||
|
||||
async def search(self, query, timeout=30):
|
||||
async def search(self, query, timeout=30, exclude_sources=None):
|
||||
self.search_calls.append((query, timeout))
|
||||
return (self._results, None)
|
||||
|
||||
|
|
@ -194,6 +196,33 @@ def test_staging_match_hit_returns_immediately():
|
|||
assert rec.calls == []
|
||||
|
||||
|
||||
def test_private_torrent_album_staging_miss_skips_per_track_search():
|
||||
_seed_task(track_info={
|
||||
'id': 'sp-1', 'name': 'Money Trees', 'artists': ['Kendrick Lamar'],
|
||||
'album': 'good kid, m.A.A.d city (Deluxe)', 'duration_ms': 387000,
|
||||
})
|
||||
download_batches['b1'] = {
|
||||
'album_bundle_private_staging': True,
|
||||
'album_bundle_state': 'staged',
|
||||
'album_bundle_source': 'torrent',
|
||||
}
|
||||
client = _FakeClient(results=['should-not-search'], mode='torrent')
|
||||
rec = _Recorder()
|
||||
deps, _ = _build_deps(
|
||||
soulseek=client,
|
||||
matching=_FakeMatchEngine(queries=['Kendrick Lamar Money Trees']),
|
||||
try_staging_match=lambda *a, **kw: False,
|
||||
on_download_completed=rec('done'),
|
||||
)
|
||||
|
||||
tw.download_track_worker('t1', 'b1', deps)
|
||||
|
||||
assert client.search_calls == []
|
||||
assert download_tasks['t1']['status'] == 'not_found'
|
||||
assert 'staged torrent album release' in download_tasks['t1']['error_message']
|
||||
assert ('done', ('b1', 't1', False), {}) in rec.calls
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Search loop happy path
|
||||
# ---------------------------------------------------------------------------
|
||||
|
|
@ -219,6 +248,25 @@ def test_first_query_success_returns_after_storing_source():
|
|||
assert download_tasks['t1']['status'] == 'searching'
|
||||
|
||||
|
||||
def test_torrent_mode_uses_album_release_after_track_queries():
|
||||
_seed_task(track_info={
|
||||
'id': 'sp-1', 'name': 'Money', 'artists': ['Pink Floyd'],
|
||||
'album': 'The Dark Side of the Moon', 'duration_ms': 383000,
|
||||
})
|
||||
client = _FakeClient(results=[], mode='torrent')
|
||||
rec = _Recorder()
|
||||
deps, _ = _build_deps(
|
||||
soulseek=client,
|
||||
matching=_FakeMatchEngine(queries=['Pink Floyd Money']),
|
||||
on_download_completed=rec('done'),
|
||||
)
|
||||
|
||||
tw.download_track_worker('t1', 'b1', deps)
|
||||
|
||||
assert client.search_calls[0][0] == 'Pink Floyd Money'
|
||||
assert client.search_calls[-1][0] == 'Pink Floyd The Dark Side of the Moon'
|
||||
|
||||
|
||||
def test_no_results_marks_not_found_and_calls_completion():
|
||||
_seed_task()
|
||||
rec = _Recorder()
|
||||
|
|
@ -271,7 +319,7 @@ def test_cancellation_mid_query_returns_without_completion():
|
|||
_seed_task()
|
||||
rec = _Recorder()
|
||||
|
||||
def _cancel_during_search(query, timeout=30):
|
||||
def _cancel_during_search(query, timeout=30, exclude_sources=None):
|
||||
download_tasks['t1']['status'] = 'cancelled'
|
||||
|
||||
async def _empty():
|
||||
|
|
|
|||
|
|
@ -123,3 +123,47 @@ def test_keeps_tidal_candidate_inside_integrity_duration_tolerance(monkeypatch):
|
|||
result = get_valid_candidates([tidal], expected, 'Artist Song')
|
||||
|
||||
assert result == [tidal]
|
||||
|
||||
|
||||
def test_rejects_torrent_title_match_from_wrong_artist(monkeypatch):
|
||||
monkeypatch.setattr(validation, 'matching_engine', _MatchingEngine())
|
||||
expected = _Track(duration_ms=180_000, name='The Man I Need', artists=('Olivia Dean',))
|
||||
wrong_artist = _Candidate(
|
||||
username='torrent',
|
||||
duration=None,
|
||||
title='The Man I Need',
|
||||
artist='Tinkabelle',
|
||||
)
|
||||
|
||||
assert get_valid_candidates([wrong_artist], expected, 'Olivia Dean The Man I Need') == []
|
||||
|
||||
|
||||
def test_keeps_torrent_title_match_from_expected_artist(monkeypatch):
|
||||
monkeypatch.setattr(validation, 'matching_engine', _MatchingEngine())
|
||||
expected = _Track(duration_ms=180_000, name='The Man I Need', artists=('Olivia Dean',))
|
||||
correct_artist = _Candidate(
|
||||
username='torrent',
|
||||
duration=None,
|
||||
title='The Man I Need',
|
||||
artist='Olivia Dean',
|
||||
)
|
||||
|
||||
result = get_valid_candidates([correct_artist], expected, 'Olivia Dean The Man I Need')
|
||||
|
||||
assert result == [correct_artist]
|
||||
|
||||
|
||||
def test_keeps_torrent_title_match_when_artist_is_indexer_fallback(monkeypatch):
|
||||
monkeypatch.setattr(validation, 'matching_engine', _MatchingEngine())
|
||||
expected = _Track(duration_ms=180_000, name='The Man I Need', artists=('Olivia Dean',))
|
||||
candidate = _Candidate(
|
||||
username='torrent',
|
||||
duration=None,
|
||||
title='The Man I Need',
|
||||
artist='Indexer',
|
||||
)
|
||||
candidate._source_metadata = {'indexer': 'Indexer'}
|
||||
|
||||
result = get_valid_candidates([candidate], expected, 'Olivia Dean The Man I Need')
|
||||
|
||||
assert result == [candidate]
|
||||
|
|
|
|||
|
|
@ -102,3 +102,153 @@ def test_cancel_download_marks_cancelled(hifi_client_with_engine):
|
|||
ok = _run_async(client.cancel_download('dl-1', None, remove=False))
|
||||
assert ok is True
|
||||
assert engine.get_record('hifi', 'dl-1')['state'] == 'Cancelled'
|
||||
|
||||
|
||||
def test_instance_capability_probe_uses_track_manifests_not_legacy_track():
|
||||
class _Response:
|
||||
def __init__(self, *, ok=True, status_code=200, payload=None):
|
||||
self.ok = ok
|
||||
self.status_code = status_code
|
||||
self._payload = payload or {}
|
||||
|
||||
def json(self):
|
||||
return self._payload
|
||||
|
||||
class _Session:
|
||||
def __init__(self):
|
||||
self.calls = []
|
||||
|
||||
def get(self, url, **kwargs):
|
||||
self.calls.append((url, kwargs))
|
||||
if url.endswith('/search/'):
|
||||
return _Response(payload={'items': []})
|
||||
if url.endswith('/trackManifests/'):
|
||||
return _Response(payload={
|
||||
'data': {
|
||||
'data': {
|
||||
'attributes': {
|
||||
'uri': 'https://cdn.example/playlist.m3u8',
|
||||
},
|
||||
},
|
||||
},
|
||||
})
|
||||
if url.endswith('/'):
|
||||
return _Response(payload={'version': 'test'})
|
||||
return _Response(ok=False, status_code=404)
|
||||
|
||||
client = HiFiClient.__new__(HiFiClient)
|
||||
client.session = _Session()
|
||||
|
||||
result = client.check_instance_capabilities('https://hifi.example')
|
||||
|
||||
called_urls = [url for url, _ in client.session.calls]
|
||||
assert result['can_search'] is True
|
||||
assert result['can_download'] is True
|
||||
assert any(url.endswith('/trackManifests/') for url in called_urls)
|
||||
assert not any(url.endswith('/track') for url in called_urls)
|
||||
|
||||
|
||||
def test_instance_capability_probe_reports_manifest_without_uri_as_limited():
|
||||
class _Response:
|
||||
ok = True
|
||||
status_code = 200
|
||||
|
||||
def __init__(self, payload):
|
||||
self._payload = payload
|
||||
|
||||
def json(self):
|
||||
return self._payload
|
||||
|
||||
class _Session:
|
||||
def get(self, url, **kwargs):
|
||||
if url.endswith('/search/'):
|
||||
return _Response({'items': []})
|
||||
if url.endswith('/trackManifests/'):
|
||||
return _Response({'data': {'data': {'attributes': {}}}})
|
||||
if url.endswith('/'):
|
||||
return _Response({'version': 'test'})
|
||||
return _Response({'data': {'data': {'attributes': {}}}})
|
||||
|
||||
client = HiFiClient.__new__(HiFiClient)
|
||||
client.session = _Session()
|
||||
|
||||
result = client.check_instance_capabilities('https://hifi.example')
|
||||
|
||||
assert result['can_search'] is True
|
||||
assert result['can_download'] is False
|
||||
assert result['download_error'] == 'No playable manifest URL'
|
||||
|
||||
|
||||
def test_instance_capability_probe_accepts_legacy_track_manifest():
|
||||
import base64
|
||||
import json
|
||||
|
||||
class _Response:
|
||||
def __init__(self, payload, *, ok=True, status_code=200):
|
||||
self._payload = payload
|
||||
self.ok = ok
|
||||
self.status_code = status_code
|
||||
|
||||
def json(self):
|
||||
return self._payload
|
||||
|
||||
class _Session:
|
||||
def __init__(self):
|
||||
self.calls = []
|
||||
|
||||
def get(self, url, **kwargs):
|
||||
self.calls.append((url, kwargs))
|
||||
if url.endswith('/search/'):
|
||||
return _Response({'items': []})
|
||||
if url.endswith('/trackManifests/'):
|
||||
return _Response({'data': {'data': {'attributes': {}}}})
|
||||
if url.endswith('/track/'):
|
||||
manifest = base64.b64encode(json.dumps({
|
||||
'mimeType': 'audio/flac',
|
||||
'codecs': 'flac',
|
||||
'encryptionType': 'NONE',
|
||||
'urls': ['https://cdn.example/track.flac'],
|
||||
}).encode()).decode()
|
||||
return _Response({'data': {'manifest': manifest}})
|
||||
if url.endswith('/'):
|
||||
return _Response({'version': 'test'})
|
||||
return _Response({}, ok=False, status_code=404)
|
||||
|
||||
client = HiFiClient.__new__(HiFiClient)
|
||||
client.session = _Session()
|
||||
|
||||
result = client.check_instance_capabilities('https://hifi.example')
|
||||
|
||||
assert result['can_search'] is True
|
||||
assert result['can_download'] is True
|
||||
assert result['download_probe'] == 'track'
|
||||
assert any(url.endswith('/track/') for url, _ in client.session.calls)
|
||||
|
||||
|
||||
def test_get_hls_manifest_falls_back_to_legacy_track_endpoint():
|
||||
import base64
|
||||
import json
|
||||
|
||||
client = HiFiClient.__new__(HiFiClient)
|
||||
calls = []
|
||||
|
||||
def _fake_api_get(path, params=None, timeout=15):
|
||||
calls.append((path, params))
|
||||
if path == '/trackManifests/':
|
||||
return None
|
||||
manifest = base64.b64encode(json.dumps({
|
||||
'mimeType': 'audio/flac',
|
||||
'codecs': 'flac',
|
||||
'encryptionType': 'NONE',
|
||||
'urls': ['https://cdn.example/track.flac'],
|
||||
}).encode()).decode()
|
||||
return {'data': {'manifest': manifest}}
|
||||
|
||||
client._api_get = _fake_api_get
|
||||
|
||||
result = client._get_hls_manifest(123, quality='lossless')
|
||||
|
||||
assert result['direct_urls'] == ['https://cdn.example/track.flac']
|
||||
assert result['extension'] == 'flac'
|
||||
assert calls[0][0] == '/trackManifests/'
|
||||
assert calls[1][0] == '/track/'
|
||||
|
|
|
|||
|
|
@ -2,6 +2,8 @@ import os
|
|||
import sqlite3
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
|
||||
from core.imports import side_effects
|
||||
|
||||
|
||||
|
|
@ -364,6 +366,42 @@ def test_library_history_labels_auto_import(monkeypatch):
|
|||
assert captured["title"] == "Auto-Imported Track"
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("username", "expected"),
|
||||
[
|
||||
("torrent", "Torrent"),
|
||||
("usenet", "Usenet"),
|
||||
("staging", "Staging"),
|
||||
],
|
||||
)
|
||||
def test_library_history_labels_release_and_staging_sources(monkeypatch, username, expected):
|
||||
"""Release/staging imports should not fall through to the Soulseek label."""
|
||||
captured = {}
|
||||
|
||||
class _DBStub:
|
||||
def add_library_history_entry(self, **kwargs):
|
||||
captured.update(kwargs)
|
||||
|
||||
monkeypatch.setattr(side_effects, "get_database", lambda: _DBStub())
|
||||
|
||||
context = {
|
||||
"track_info": {
|
||||
"name": "Imported Track",
|
||||
"artists": [{"name": "Some Artist"}],
|
||||
"album": "Some Album",
|
||||
},
|
||||
"original_search_result": {
|
||||
"username": username,
|
||||
"filename": "source-file.flac",
|
||||
},
|
||||
"_final_processed_path": "/library/some-album/01.flac",
|
||||
}
|
||||
|
||||
side_effects.record_library_history_download(context)
|
||||
|
||||
assert captured["download_source"] == expected
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Album duration parity — must equal sum of all track durations, not whatever
|
||||
# the first imported track happened to be.
|
||||
|
|
|
|||
44
tests/metadata/test_cache_maintenance_retry.py
Normal file
44
tests/metadata/test_cache_maintenance_retry.py
Normal file
|
|
@ -0,0 +1,44 @@
|
|||
import sqlite3
|
||||
from types import SimpleNamespace
|
||||
|
||||
import core.metadata.cache as cache_module
|
||||
from core.metadata.cache import MetadataCache
|
||||
|
||||
|
||||
def test_maintenance_write_retries_once_after_disk_io(monkeypatch):
|
||||
cache = MetadataCache()
|
||||
attempts = []
|
||||
|
||||
class _Conn:
|
||||
def close(self):
|
||||
pass
|
||||
|
||||
monkeypatch.setattr(cache, "_get_db", lambda: SimpleNamespace(_get_connection=lambda: _Conn()))
|
||||
monkeypatch.setattr(cache_module.time, "sleep", lambda _seconds: None)
|
||||
|
||||
def _operation(_conn):
|
||||
attempts.append(1)
|
||||
if len(attempts) == 1:
|
||||
raise sqlite3.OperationalError("disk I/O error")
|
||||
return 9
|
||||
|
||||
assert cache._run_maintenance_write("Cache eviction", _operation) == 9
|
||||
assert len(attempts) == 2
|
||||
|
||||
|
||||
def test_maintenance_write_does_not_retry_non_io_errors(monkeypatch):
|
||||
cache = MetadataCache()
|
||||
attempts = []
|
||||
|
||||
class _Conn:
|
||||
def close(self):
|
||||
pass
|
||||
|
||||
monkeypatch.setattr(cache, "_get_db", lambda: SimpleNamespace(_get_connection=lambda: _Conn()))
|
||||
|
||||
def _operation(_conn):
|
||||
attempts.append(1)
|
||||
raise sqlite3.OperationalError("syntax error")
|
||||
|
||||
assert cache._run_maintenance_write("Cache eviction", _operation) == 0
|
||||
assert len(attempts) == 1
|
||||
301
tests/test_album_bundle.py
Normal file
301
tests/test_album_bundle.py
Normal file
|
|
@ -0,0 +1,301 @@
|
|||
"""Tests for ``core/download_plugins/album_bundle.py``.
|
||||
|
||||
The shared helpers used by both the torrent and usenet album-bundle
|
||||
flows. Pins the pick heuristic, the atomic-copy invariant
|
||||
(no partial files ever visible at the audio extension), the
|
||||
collision-suffix logic, and the config-driven poll cadence so a
|
||||
future tweak in either plugin can't break the contract.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import threading
|
||||
import time
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Optional
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
|
||||
from core.download_plugins.album_bundle import (
|
||||
ALBUM_PICK_MAX_BYTES,
|
||||
ALBUM_PICK_MIN_BYTES,
|
||||
DEFAULT_POLL_INTERVAL_SECONDS,
|
||||
DEFAULT_POLL_TIMEOUT_SECONDS,
|
||||
atomic_copy_to_staging,
|
||||
copy_audio_files_atomically,
|
||||
get_poll_interval,
|
||||
get_poll_timeout,
|
||||
pick_best_album_release,
|
||||
quality_score,
|
||||
unique_staging_path,
|
||||
)
|
||||
|
||||
|
||||
# Minimal release-result shim — duck-types the fields the picker reads.
|
||||
@dataclass
|
||||
class _Release:
|
||||
title: str
|
||||
size: int
|
||||
seeders: Optional[int] = None
|
||||
grabs: Optional[int] = None
|
||||
|
||||
|
||||
def _flac_quality_guess(title: str) -> str:
|
||||
"""Stand-in for the plugin's title→quality function."""
|
||||
t = (title or '').lower()
|
||||
if 'flac' in t:
|
||||
return 'flac'
|
||||
if 'aac' in t:
|
||||
return 'aac'
|
||||
if 'ogg' in t:
|
||||
return 'ogg'
|
||||
return 'mp3'
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# pick_best_album_release
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_picker_returns_none_for_empty_input() -> None:
|
||||
assert pick_best_album_release([], _flac_quality_guess) is None
|
||||
|
||||
|
||||
def test_picker_drops_singletons_when_albums_present() -> None:
|
||||
"""Single-track torrents under 40 MB shouldn't beat an album-sized
|
||||
candidate even if the single has thousands of seeders."""
|
||||
single = _Release(title='Track [MP3]', size=10_000_000, seeders=10_000)
|
||||
album = _Release(title='Album [MP3]', size=120_000_000, seeders=5)
|
||||
assert pick_best_album_release([single, album], _flac_quality_guess) is album
|
||||
|
||||
|
||||
def test_picker_prefers_flac_when_tied_on_seeders() -> None:
|
||||
flac = _Release(title='Album [FLAC]', size=400_000_000, seeders=50)
|
||||
mp3 = _Release(title='Album [MP3]', size=130_000_000, seeders=50)
|
||||
assert pick_best_album_release([flac, mp3], _flac_quality_guess) is flac
|
||||
|
||||
|
||||
def test_picker_uses_grabs_when_seeders_is_none() -> None:
|
||||
"""Usenet results have ``seeders=None`` — the picker should fall
|
||||
back to ``grabs`` so popularity still drives the ranking."""
|
||||
cold = _Release(title='Album A [MP3]', size=200_000_000, seeders=None, grabs=1)
|
||||
popular = _Release(title='Album B [MP3]', size=200_000_000, seeders=None, grabs=999)
|
||||
assert pick_best_album_release([cold, popular], _flac_quality_guess) is popular
|
||||
|
||||
|
||||
def test_picker_falls_back_when_all_below_floor() -> None:
|
||||
"""When every candidate is below the 40 MB album-size floor,
|
||||
return the most-seeded one rather than None — the user still
|
||||
wants a download attempt."""
|
||||
small_low = _Release(title='X', size=5_000_000, seeders=10)
|
||||
small_high = _Release(title='Y', size=8_000_000, seeders=200)
|
||||
assert pick_best_album_release([small_low, small_high], _flac_quality_guess) is small_high
|
||||
|
||||
|
||||
def test_picker_size_floor_matches_constant() -> None:
|
||||
"""If someone moves the constant the floor moves with it — pin
|
||||
the relationship to catch accidental literals creeping back in."""
|
||||
just_below = _Release(title='Below', size=ALBUM_PICK_MIN_BYTES - 1, seeders=999)
|
||||
just_above = _Release(title='Above', size=ALBUM_PICK_MIN_BYTES + 1, seeders=1)
|
||||
assert pick_best_album_release([just_below, just_above], _flac_quality_guess) is just_above
|
||||
|
||||
|
||||
def test_picker_rejects_oversized_box_sets() -> None:
|
||||
"""Anything past 3 GB drops out of the preferred pool — most likely
|
||||
a multi-disc box set with scans + bonus material, not what the
|
||||
user asked for."""
|
||||
sane = _Release(title='Album [FLAC]', size=400_000_000, seeders=10)
|
||||
box = _Release(title='Album Box [FLAC]', size=ALBUM_PICK_MAX_BYTES + 1_000_000, seeders=999)
|
||||
# Sane wins even with 100x fewer seeders, because box is outside
|
||||
# the preferred range.
|
||||
assert pick_best_album_release([sane, box], _flac_quality_guess) is sane
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# quality_score
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_quality_score_orders_formats() -> None:
|
||||
assert quality_score('Album [FLAC]', _flac_quality_guess) > quality_score('Album [MP3]', _flac_quality_guess)
|
||||
assert quality_score('Album [AAC]', _flac_quality_guess) > quality_score('Album [MP3]', _flac_quality_guess)
|
||||
assert quality_score('Bare title', _flac_quality_guess) == quality_score('Album [MP3]', _flac_quality_guess)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# unique_staging_path
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_unique_staging_path_returns_natural_when_clear(tmp_path: Path) -> None:
|
||||
src = tmp_path / 'src.flac'
|
||||
src.write_bytes(b'fLaC')
|
||||
staging = tmp_path / 'staging'
|
||||
staging.mkdir()
|
||||
assert unique_staging_path(staging, src) == staging / 'src.flac'
|
||||
|
||||
|
||||
def test_unique_staging_path_suffixes_on_collision(tmp_path: Path) -> None:
|
||||
src = tmp_path / 'src.flac'
|
||||
src.write_bytes(b'fLaC')
|
||||
staging = tmp_path / 'staging'
|
||||
staging.mkdir()
|
||||
(staging / 'src.flac').write_bytes(b'existing')
|
||||
assert unique_staging_path(staging, src) == staging / 'src_1.flac'
|
||||
|
||||
|
||||
def test_unique_staging_path_increments_suffix(tmp_path: Path) -> None:
|
||||
src = tmp_path / 'src.flac'
|
||||
src.write_bytes(b'fLaC')
|
||||
staging = tmp_path / 'staging'
|
||||
staging.mkdir()
|
||||
(staging / 'src.flac').write_bytes(b'1')
|
||||
(staging / 'src_1.flac').write_bytes(b'2')
|
||||
(staging / 'src_2.flac').write_bytes(b'3')
|
||||
assert unique_staging_path(staging, src) == staging / 'src_3.flac'
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# atomic_copy_to_staging
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_atomic_copy_lands_at_final_path(tmp_path: Path) -> None:
|
||||
src = tmp_path / 'src.flac'
|
||||
src.write_bytes(b'fLaC payload')
|
||||
dest = tmp_path / 'staging' / 'track.flac'
|
||||
dest.parent.mkdir()
|
||||
assert atomic_copy_to_staging(src, dest) is True
|
||||
assert dest.read_bytes() == b'fLaC payload'
|
||||
|
||||
|
||||
def test_atomic_copy_leaves_no_tmp_files_after_success(tmp_path: Path) -> None:
|
||||
"""The .tmp.<random> sidecar must be cleaned up by the rename —
|
||||
no orphan files left behind on a successful copy."""
|
||||
src = tmp_path / 'src.flac'
|
||||
src.write_bytes(b'data')
|
||||
dest = tmp_path / 'staging' / 'track.flac'
|
||||
dest.parent.mkdir()
|
||||
atomic_copy_to_staging(src, dest)
|
||||
tmp_files = list(dest.parent.glob('*.tmp.*'))
|
||||
assert tmp_files == []
|
||||
|
||||
|
||||
def test_atomic_copy_never_exposes_partial_to_extension_scanner(tmp_path: Path) -> None:
|
||||
"""Auto-Import filters by audio extension — the in-flight file
|
||||
must NEVER be visible at its final extension until the copy is
|
||||
complete. We probe this by scanning the staging dir in parallel
|
||||
with the copy and assert the audio file is either absent OR
|
||||
fully written.
|
||||
"""
|
||||
src = tmp_path / 'src.flac'
|
||||
src.write_bytes(b'x' * (2 * 1024 * 1024))
|
||||
dest = tmp_path / 'staging' / 'track.flac'
|
||||
dest.parent.mkdir()
|
||||
|
||||
stop = threading.Event()
|
||||
saw_partial = threading.Event()
|
||||
expected_size = src.stat().st_size
|
||||
|
||||
def _scan_loop():
|
||||
while not stop.is_set():
|
||||
try:
|
||||
files = [p for p in dest.parent.iterdir() if p.suffix == '.flac']
|
||||
except FileNotFoundError:
|
||||
continue
|
||||
for fp in files:
|
||||
size = fp.stat().st_size
|
||||
if 0 < size < expected_size:
|
||||
saw_partial.set()
|
||||
return
|
||||
|
||||
scanner = threading.Thread(target=_scan_loop, daemon=True)
|
||||
scanner.start()
|
||||
try:
|
||||
for i in range(5):
|
||||
target = dest.with_name(f'track_{i}.flac')
|
||||
atomic_copy_to_staging(src, target)
|
||||
# Give the scanner a moment to drain any final scan iteration.
|
||||
time.sleep(0.05)
|
||||
finally:
|
||||
stop.set()
|
||||
scanner.join(timeout=1.0)
|
||||
|
||||
assert not saw_partial.is_set(), \
|
||||
"Scanner observed a partial audio file — atomic copy contract broken"
|
||||
|
||||
|
||||
def test_copy_audio_files_atomically_skips_failures(tmp_path: Path) -> None:
|
||||
"""One file failing to copy shouldn't stop the rest from being
|
||||
staged — partial results are better than a complete bailout."""
|
||||
src_a = tmp_path / 'a.flac'
|
||||
src_a.write_bytes(b'a')
|
||||
src_missing = tmp_path / 'does-not-exist.flac' # never created
|
||||
src_c = tmp_path / 'c.flac'
|
||||
src_c.write_bytes(b'c')
|
||||
staging = tmp_path / 'staging'
|
||||
out = copy_audio_files_atomically([src_a, src_missing, src_c], staging)
|
||||
assert len(out) == 2
|
||||
landed = sorted(Path(p).name for p in out)
|
||||
assert landed == ['a.flac', 'c.flac']
|
||||
|
||||
|
||||
def test_copy_audio_files_atomically_creates_staging_dir(tmp_path: Path) -> None:
|
||||
src = tmp_path / 'a.flac'
|
||||
src.write_bytes(b'a')
|
||||
staging = tmp_path / 'nested' / 'staging' / 'dir'
|
||||
out = copy_audio_files_atomically([src], staging)
|
||||
assert len(out) == 1
|
||||
assert staging.exists()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Config-driven poll cadence
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_get_poll_interval_uses_default_when_unset() -> None:
|
||||
with patch('core.download_plugins.album_bundle.config_manager') as cm:
|
||||
cm.get.return_value = DEFAULT_POLL_INTERVAL_SECONDS
|
||||
assert get_poll_interval() == DEFAULT_POLL_INTERVAL_SECONDS
|
||||
|
||||
|
||||
def test_get_poll_interval_honours_override() -> None:
|
||||
with patch('core.download_plugins.album_bundle.config_manager') as cm:
|
||||
cm.get.return_value = 5
|
||||
assert get_poll_interval() == 5.0
|
||||
|
||||
|
||||
def test_get_poll_interval_falls_back_on_garbage() -> None:
|
||||
"""Non-numeric / non-positive values fall back to the default
|
||||
rather than crashing the poll loop."""
|
||||
with patch('core.download_plugins.album_bundle.config_manager') as cm:
|
||||
cm.get.return_value = 'not-a-number'
|
||||
assert get_poll_interval() == DEFAULT_POLL_INTERVAL_SECONDS
|
||||
cm.get.return_value = -1
|
||||
assert get_poll_interval() == DEFAULT_POLL_INTERVAL_SECONDS
|
||||
|
||||
|
||||
def test_get_poll_timeout_uses_default_when_unset() -> None:
|
||||
with patch('core.download_plugins.album_bundle.config_manager') as cm:
|
||||
cm.get.return_value = DEFAULT_POLL_TIMEOUT_SECONDS
|
||||
assert get_poll_timeout() == DEFAULT_POLL_TIMEOUT_SECONDS
|
||||
|
||||
|
||||
def test_get_poll_timeout_honours_override() -> None:
|
||||
"""Users with slow trackers / large box sets can extend the
|
||||
deadline without touching code."""
|
||||
with patch('core.download_plugins.album_bundle.config_manager') as cm:
|
||||
cm.get.return_value = 86_400 # 24h
|
||||
assert get_poll_timeout() == 86_400.0
|
||||
|
||||
|
||||
def test_get_poll_timeout_falls_back_on_garbage() -> None:
|
||||
with patch('core.download_plugins.album_bundle.config_manager') as cm:
|
||||
cm.get.return_value = ''
|
||||
assert get_poll_timeout() == DEFAULT_POLL_TIMEOUT_SECONDS
|
||||
cm.get.return_value = 0
|
||||
assert get_poll_timeout() == DEFAULT_POLL_TIMEOUT_SECONDS
|
||||
322
tests/test_album_bundle_dispatch.py
Normal file
322
tests/test_album_bundle_dispatch.py
Normal file
|
|
@ -0,0 +1,322 @@
|
|||
"""Tests for ``core/downloads/album_bundle_dispatch.py``.
|
||||
|
||||
Pins the gate predicate, the resolution + run flow, and the
|
||||
fail / fall-through return contract. Mocks the config, plugin
|
||||
resolver, and state access so the dispatcher is testable without
|
||||
standing up runtime_state or a real plugin.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
|
||||
from core.downloads.album_bundle_dispatch import (
|
||||
BatchStateAccess,
|
||||
is_eligible,
|
||||
try_dispatch,
|
||||
)
|
||||
|
||||
|
||||
class _FakeState:
|
||||
"""In-memory ``BatchStateAccess`` for tests — records every
|
||||
update so assertions can check the sequence of fields set."""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self.fields: dict = {}
|
||||
self.update_calls: list = []
|
||||
self.failed_with: str = ''
|
||||
|
||||
def update_fields(self, batch_id: str, fields: dict) -> None:
|
||||
self.update_calls.append((batch_id, dict(fields)))
|
||||
self.fields.update(fields)
|
||||
|
||||
def mark_failed(self, batch_id: str, error: str) -> None:
|
||||
self.failed_with = error
|
||||
self.fields['phase'] = 'failed'
|
||||
self.fields['error'] = error
|
||||
self.fields['album_bundle_state'] = 'failed'
|
||||
|
||||
|
||||
def _config(values: dict):
|
||||
"""Build a config_get callable from a flat dict."""
|
||||
def _get(key, default=None):
|
||||
return values.get(key, default)
|
||||
return _get
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# is_eligible pure predicate
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_is_eligible_requires_album_flag() -> None:
|
||||
assert is_eligible(mode='torrent', is_album=False,
|
||||
album_name='X', artist_name='Y') is False
|
||||
|
||||
|
||||
def test_is_eligible_requires_torrent_or_usenet_mode() -> None:
|
||||
for mode in ('soulseek', 'youtube', 'tidal', 'qobuz', 'hifi',
|
||||
'deezer_dl', 'amazon', 'lidarr', 'soundcloud', 'hybrid'):
|
||||
assert is_eligible(mode=mode, is_album=True,
|
||||
album_name='X', artist_name='Y') is False
|
||||
|
||||
|
||||
def test_is_eligible_accepts_torrent_and_usenet() -> None:
|
||||
assert is_eligible(mode='torrent', is_album=True,
|
||||
album_name='X', artist_name='Y') is True
|
||||
assert is_eligible(mode='usenet', is_album=True,
|
||||
album_name='X', artist_name='Y') is True
|
||||
|
||||
|
||||
def test_is_eligible_requires_non_empty_names() -> None:
|
||||
assert is_eligible(mode='torrent', is_album=True,
|
||||
album_name='', artist_name='Y') is False
|
||||
assert is_eligible(mode='torrent', is_album=True,
|
||||
album_name='X', artist_name='') is False
|
||||
assert is_eligible(mode='torrent', is_album=True,
|
||||
album_name=' ', artist_name='Y') is False
|
||||
|
||||
|
||||
def test_is_eligible_case_insensitive_mode() -> None:
|
||||
assert is_eligible(mode='TORRENT', is_album=True,
|
||||
album_name='X', artist_name='Y') is True
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# try_dispatch — gate evaluation
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_dispatch_returns_false_when_not_album() -> None:
|
||||
state = _FakeState()
|
||||
plugin = MagicMock()
|
||||
result = try_dispatch(
|
||||
batch_id='b1', is_album=False,
|
||||
album_context={'name': 'X'}, artist_context={'name': 'Y'},
|
||||
config_get=_config({'download_source.mode': 'torrent'}),
|
||||
plugin_resolver=lambda _name: plugin, state=state,
|
||||
)
|
||||
assert result is False
|
||||
assert state.update_calls == []
|
||||
plugin.download_album_to_staging.assert_not_called()
|
||||
|
||||
|
||||
def test_dispatch_returns_false_for_non_torrent_modes() -> None:
|
||||
state = _FakeState()
|
||||
plugin = MagicMock()
|
||||
result = try_dispatch(
|
||||
batch_id='b1', is_album=True,
|
||||
album_context={'name': 'X'}, artist_context={'name': 'Y'},
|
||||
config_get=_config({'download_source.mode': 'soulseek'}),
|
||||
plugin_resolver=lambda _name: plugin, state=state,
|
||||
)
|
||||
assert result is False
|
||||
assert state.update_calls == []
|
||||
|
||||
|
||||
def test_dispatch_returns_false_when_plugin_missing() -> None:
|
||||
"""No plugin available → fall through to per-track flow with a
|
||||
warning. The state SHOULD NOT have been touched."""
|
||||
state = _FakeState()
|
||||
result = try_dispatch(
|
||||
batch_id='b1', is_album=True,
|
||||
album_context={'name': 'X'}, artist_context={'name': 'Y'},
|
||||
config_get=_config({'download_source.mode': 'torrent'}),
|
||||
plugin_resolver=lambda _name: None, state=state,
|
||||
)
|
||||
assert result is False
|
||||
assert state.update_calls == []
|
||||
|
||||
|
||||
def test_dispatch_returns_false_when_plugin_lacks_method() -> None:
|
||||
state = _FakeState()
|
||||
# Plugin that doesn't implement download_album_to_staging.
|
||||
class _LegacyPlugin:
|
||||
pass
|
||||
result = try_dispatch(
|
||||
batch_id='b1', is_album=True,
|
||||
album_context={'name': 'X'}, artist_context={'name': 'Y'},
|
||||
config_get=_config({'download_source.mode': 'torrent'}),
|
||||
plugin_resolver=lambda _name: _LegacyPlugin(), state=state,
|
||||
)
|
||||
assert result is False
|
||||
assert state.update_calls == []
|
||||
|
||||
|
||||
def test_dispatch_returns_false_when_resolver_raises() -> None:
|
||||
"""Plugin resolution can fail (registry not initialised); we log
|
||||
and fall through rather than crashing the master worker."""
|
||||
state = _FakeState()
|
||||
def _boom(_name):
|
||||
raise RuntimeError("registry not initialised")
|
||||
result = try_dispatch(
|
||||
batch_id='b1', is_album=True,
|
||||
album_context={'name': 'X'}, artist_context={'name': 'Y'},
|
||||
config_get=_config({'download_source.mode': 'torrent'}),
|
||||
plugin_resolver=_boom, state=state,
|
||||
)
|
||||
assert result is False
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# try_dispatch — success / failure paths
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_dispatch_success_returns_false_so_per_track_can_run() -> None:
|
||||
"""Success → master worker should CONTINUE to per-track flow so
|
||||
each task can hit try_staging_match and find its file."""
|
||||
state = _FakeState()
|
||||
plugin = MagicMock()
|
||||
plugin.download_album_to_staging.return_value = {
|
||||
'success': True, 'files': ['/tmp/a.flac', '/tmp/b.flac'], 'error': None,
|
||||
}
|
||||
result = try_dispatch(
|
||||
batch_id='b1', is_album=True,
|
||||
album_context={'name': 'GNX'}, artist_context={'name': 'Kendrick Lamar'},
|
||||
config_get=_config({
|
||||
'download_source.mode': 'torrent',
|
||||
'import.staging_path': '/staging/path',
|
||||
}),
|
||||
plugin_resolver=lambda _name: plugin, state=state,
|
||||
)
|
||||
assert result is False
|
||||
# Plugin was called with the right args.
|
||||
args = plugin.download_album_to_staging.call_args
|
||||
assert args.args[0] == 'GNX'
|
||||
assert args.args[1] == 'Kendrick Lamar'
|
||||
assert args.args[2].replace('\\', '/').endswith('storage/album_bundle_staging/b1')
|
||||
# Phase transitioned through searching → analysis.
|
||||
assert state.fields['phase'] == 'analysis'
|
||||
assert state.fields['album_bundle_state'] == 'staged'
|
||||
assert state.fields['album_bundle_source'] == 'torrent'
|
||||
assert state.fields['album_bundle_private_staging'] is True
|
||||
assert state.fields['album_bundle_staging_path'].replace('\\', '/').endswith('storage/album_bundle_staging/b1')
|
||||
assert state.failed_with == ''
|
||||
|
||||
|
||||
def test_dispatch_uses_configured_private_album_bundle_staging_root() -> None:
|
||||
state = _FakeState()
|
||||
plugin = MagicMock()
|
||||
plugin.download_album_to_staging.return_value = {'success': True, 'files': ['/tmp/a.flac']}
|
||||
|
||||
try_dispatch(
|
||||
batch_id='batch:with/slash', is_album=True,
|
||||
album_context={'name': 'GNX'}, artist_context={'name': 'Kendrick Lamar'},
|
||||
config_get=_config({
|
||||
'download_source.mode': 'torrent',
|
||||
'download_source.album_bundle_staging_path': '/private/staging',
|
||||
}),
|
||||
plugin_resolver=lambda _name: plugin, state=state,
|
||||
)
|
||||
|
||||
staging_arg = plugin.download_album_to_staging.call_args.args[2].replace('\\', '/')
|
||||
assert staging_arg == '/private/staging/batch_with_slash'
|
||||
assert state.fields['album_bundle_staging_path'].replace('\\', '/') == staging_arg
|
||||
|
||||
|
||||
def test_dispatch_failure_returns_true_so_master_stops() -> None:
|
||||
state = _FakeState()
|
||||
plugin = MagicMock()
|
||||
plugin.download_album_to_staging.return_value = {
|
||||
'success': False, 'files': [], 'error': 'No torrent results found',
|
||||
}
|
||||
result = try_dispatch(
|
||||
batch_id='b1', is_album=True,
|
||||
album_context={'name': 'GNX'}, artist_context={'name': 'Kendrick Lamar'},
|
||||
config_get=_config({'download_source.mode': 'torrent'}),
|
||||
plugin_resolver=lambda _name: plugin, state=state,
|
||||
)
|
||||
assert result is True
|
||||
assert state.failed_with == 'No torrent results found'
|
||||
assert state.fields['phase'] == 'failed'
|
||||
|
||||
|
||||
def test_dispatch_plugin_exception_treated_as_failure() -> None:
|
||||
"""A bug / network error in the plugin must not propagate into
|
||||
the master worker — caught + treated as a normal failure so
|
||||
the batch reports the error cleanly."""
|
||||
state = _FakeState()
|
||||
plugin = MagicMock()
|
||||
plugin.download_album_to_staging.side_effect = RuntimeError("network down")
|
||||
result = try_dispatch(
|
||||
batch_id='b1', is_album=True,
|
||||
album_context={'name': 'GNX'}, artist_context={'name': 'Kendrick Lamar'},
|
||||
config_get=_config({'download_source.mode': 'torrent'}),
|
||||
plugin_resolver=lambda _name: plugin, state=state,
|
||||
)
|
||||
assert result is True
|
||||
assert 'network down' in state.failed_with
|
||||
|
||||
|
||||
def test_dispatch_strips_whitespace_from_names() -> None:
|
||||
"""Trailing whitespace in batch context shouldn't fail the
|
||||
eligibility predicate AND should be cleaned before passing to
|
||||
the plugin."""
|
||||
state = _FakeState()
|
||||
plugin = MagicMock()
|
||||
plugin.download_album_to_staging.return_value = {'success': True, 'files': ['/x']}
|
||||
try_dispatch(
|
||||
batch_id='b1', is_album=True,
|
||||
album_context={'name': ' GNX '}, artist_context={'name': ' Kendrick '},
|
||||
config_get=_config({'download_source.mode': 'torrent'}),
|
||||
plugin_resolver=lambda _name: plugin, state=state,
|
||||
)
|
||||
args = plugin.download_album_to_staging.call_args
|
||||
assert args.args[0] == 'GNX'
|
||||
assert args.args[1] == 'Kendrick'
|
||||
|
||||
|
||||
def test_dispatch_progress_callback_mirrors_payload_to_state() -> None:
|
||||
"""The progress callback the plugin gets must mirror its
|
||||
payload onto the batch state under ``album_bundle_*`` keys so
|
||||
the Downloads page can render progress while the torrent
|
||||
download runs."""
|
||||
state = _FakeState()
|
||||
captured_emit = {}
|
||||
|
||||
def _capture(album, artist, staging, emit):
|
||||
captured_emit['fn'] = emit
|
||||
emit({'state': 'searching', 'release': 'GNX [FLAC]'})
|
||||
emit({'state': 'downloading', 'progress': 0.42, 'speed': 1024 * 1024})
|
||||
emit({'state': 'staged', 'count': 12})
|
||||
return {'success': True, 'files': []}
|
||||
|
||||
plugin = MagicMock()
|
||||
plugin.download_album_to_staging.side_effect = _capture
|
||||
try_dispatch(
|
||||
batch_id='b1', is_album=True,
|
||||
album_context={'name': 'GNX'}, artist_context={'name': 'Kendrick Lamar'},
|
||||
config_get=_config({'download_source.mode': 'torrent'}),
|
||||
plugin_resolver=lambda _name: plugin, state=state,
|
||||
)
|
||||
# State should have seen each of the three lifecycle emissions.
|
||||
states_seen = [fields.get('album_bundle_state')
|
||||
for _, fields in state.update_calls
|
||||
if 'album_bundle_state' in fields]
|
||||
assert 'searching' in states_seen
|
||||
assert 'downloading' in states_seen
|
||||
assert 'staged' in states_seen
|
||||
# Numeric progress + release name made it through.
|
||||
assert state.fields['album_bundle_release'] == 'GNX [FLAC]'
|
||||
assert state.fields['album_bundle_progress'] == 0.42
|
||||
assert state.fields['album_bundle_count'] == 12
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Protocol conformance — runtime impl must satisfy the contract
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_runtime_state_impl_matches_protocol() -> None:
|
||||
"""Sanity check that the concrete BatchStateAccess impl in
|
||||
master.py implements both methods. We don't import master.py
|
||||
here (would pull in heavy deps); duck-check on the _FakeState
|
||||
instead since it's a sibling impl of the same Protocol."""
|
||||
state: BatchStateAccess = _FakeState()
|
||||
state.update_fields('b1', {'x': 1})
|
||||
state.mark_failed('b1', 'oops')
|
||||
assert state.fields['x'] == 1
|
||||
assert state.fields['error'] == 'oops'
|
||||
238
tests/test_archive_pipeline.py
Normal file
238
tests/test_archive_pipeline.py
Normal file
|
|
@ -0,0 +1,238 @@
|
|||
"""Tests for ``core/archive_pipeline.py``.
|
||||
|
||||
Covers the audio-file walker, archive detector, and zip / tar
|
||||
extraction (rar / 7z paths use optional deps so they're only
|
||||
exercised when the libs are present in the test environment).
|
||||
Path-traversal protection gets explicit coverage — a malicious
|
||||
archive must not escape the extraction directory.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import tarfile
|
||||
import zipfile
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from core.archive_pipeline import (
|
||||
AUDIO_EXTENSIONS,
|
||||
ARCHIVE_EXTENSIONS,
|
||||
collect_audio_after_extraction,
|
||||
extract_archive,
|
||||
extract_all_in_dir,
|
||||
find_archives_in_dir,
|
||||
is_archive,
|
||||
walk_audio_files,
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Constants
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_audio_extensions_cover_common_formats() -> None:
|
||||
for ext in ('.flac', '.mp3', '.m4a', '.ogg', '.opus', '.wav', '.aac', '.wma'):
|
||||
assert ext in AUDIO_EXTENSIONS
|
||||
|
||||
|
||||
def test_archive_extensions_cover_common_formats() -> None:
|
||||
for ext in ('.zip', '.rar', '.tar', '.7z'):
|
||||
assert ext in ARCHIVE_EXTENSIONS
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# is_archive
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_is_archive_detects_simple_extensions(tmp_path: Path) -> None:
|
||||
zip_path = tmp_path / 'x.zip'
|
||||
zip_path.write_bytes(b'PK\x03\x04') # minimal — is_archive doesn't validate content
|
||||
assert is_archive(zip_path) is True
|
||||
|
||||
|
||||
def test_is_archive_detects_compound_tar_extensions(tmp_path: Path) -> None:
|
||||
"""``.tar.gz`` etc. — Path.suffix only catches the last suffix,
|
||||
so the detector has to special-case compound extensions."""
|
||||
targz = tmp_path / 'x.tar.gz'
|
||||
targz.write_bytes(b'\x1f\x8b')
|
||||
assert is_archive(targz) is True
|
||||
|
||||
|
||||
def test_is_archive_returns_false_for_audio(tmp_path: Path) -> None:
|
||||
flac = tmp_path / 'song.flac'
|
||||
flac.write_bytes(b'fLaC')
|
||||
assert is_archive(flac) is False
|
||||
|
||||
|
||||
def test_is_archive_returns_false_for_missing_file(tmp_path: Path) -> None:
|
||||
assert is_archive(tmp_path / 'does-not-exist.zip') is False
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# walk_audio_files
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_walk_audio_files_finds_nested(tmp_path: Path) -> None:
|
||||
# Layout: root/album/disc1/track.flac + root/album/disc2/track.mp3
|
||||
(tmp_path / 'album' / 'disc1').mkdir(parents=True)
|
||||
(tmp_path / 'album' / 'disc2').mkdir(parents=True)
|
||||
(tmp_path / 'album' / 'disc1' / 'track1.flac').write_bytes(b'fLaC')
|
||||
(tmp_path / 'album' / 'disc2' / 'track1.mp3').write_bytes(b'ID3')
|
||||
(tmp_path / 'album' / 'cover.jpg').write_bytes(b'\xff\xd8')
|
||||
found = walk_audio_files(tmp_path)
|
||||
names = sorted(p.name for p in found)
|
||||
assert names == ['track1.flac', 'track1.mp3']
|
||||
|
||||
|
||||
def test_walk_audio_files_returns_empty_for_missing(tmp_path: Path) -> None:
|
||||
assert walk_audio_files(tmp_path / 'does-not-exist') == []
|
||||
|
||||
|
||||
def test_walk_audio_files_ignores_non_audio(tmp_path: Path) -> None:
|
||||
(tmp_path / 'readme.txt').write_text('hi')
|
||||
(tmp_path / 'cover.png').write_bytes(b'\x89PNG')
|
||||
assert walk_audio_files(tmp_path) == []
|
||||
|
||||
|
||||
def test_walk_audio_files_case_insensitive_extension(tmp_path: Path) -> None:
|
||||
"""Lots of torrents have uppercase extensions (.MP3, .FLAC) —
|
||||
the walker must catch those too."""
|
||||
(tmp_path / 'TRACK.MP3').write_bytes(b'ID3')
|
||||
(tmp_path / 'TRACK.FLAC').write_bytes(b'fLaC')
|
||||
found = walk_audio_files(tmp_path)
|
||||
assert len(found) == 2
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# find_archives_in_dir
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_find_archives_in_dir_only_top_level(tmp_path: Path) -> None:
|
||||
"""find_archives doesn't recurse — torrents put the archive at
|
||||
the top of the dir; deeper search risks extracting unrelated
|
||||
archives that ship inside a sample folder, etc."""
|
||||
(tmp_path / 'album.zip').write_bytes(b'PK\x03\x04')
|
||||
nested = tmp_path / 'subdir'
|
||||
nested.mkdir()
|
||||
(nested / 'nested.zip').write_bytes(b'PK\x03\x04')
|
||||
found = find_archives_in_dir(tmp_path)
|
||||
assert [p.name for p in found] == ['album.zip']
|
||||
|
||||
|
||||
def test_find_archives_in_dir_empty(tmp_path: Path) -> None:
|
||||
assert find_archives_in_dir(tmp_path) == []
|
||||
assert find_archives_in_dir(tmp_path / 'missing') == []
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# extract_archive — zip
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_extract_zip_writes_files(tmp_path: Path) -> None:
|
||||
zip_path = tmp_path / 'album.zip'
|
||||
with zipfile.ZipFile(zip_path, 'w') as zf:
|
||||
zf.writestr('track1.mp3', b'ID3 track1 data')
|
||||
zf.writestr('track2.flac', b'fLaC data')
|
||||
result = extract_archive(zip_path)
|
||||
assert result == zip_path.parent
|
||||
assert (tmp_path / 'track1.mp3').exists()
|
||||
assert (tmp_path / 'track2.flac').exists()
|
||||
|
||||
|
||||
def test_extract_zip_rejects_path_traversal(tmp_path: Path) -> None:
|
||||
"""A malicious archive trying to write ``../../etc/passwd`` must
|
||||
be refused without extracting anything."""
|
||||
zip_path = tmp_path / 'evil.zip'
|
||||
extract_dest = tmp_path / 'staging'
|
||||
extract_dest.mkdir()
|
||||
with zipfile.ZipFile(zip_path, 'w') as zf:
|
||||
zf.writestr('../escaped.txt', b'evil')
|
||||
zf.writestr('safe.mp3', b'ID3')
|
||||
extract_archive(zip_path, extract_to=extract_dest)
|
||||
# Neither file should have landed — extraction aborts on the first
|
||||
# traversal attempt.
|
||||
assert not (extract_dest / 'safe.mp3').exists()
|
||||
assert not (tmp_path / 'escaped.txt').exists()
|
||||
|
||||
|
||||
def test_extract_zip_returns_none_for_bad_zip(tmp_path: Path) -> None:
|
||||
bad = tmp_path / 'not-a-zip.zip'
|
||||
bad.write_bytes(b'this is not a zip')
|
||||
assert extract_archive(bad) is None
|
||||
|
||||
|
||||
def test_extract_archive_missing_file_returns_none(tmp_path: Path) -> None:
|
||||
assert extract_archive(tmp_path / 'does-not-exist.zip') is None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# extract_archive — tar
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_extract_tar_gz_writes_files(tmp_path: Path) -> None:
|
||||
payload = tmp_path / 'track.mp3'
|
||||
payload.write_bytes(b'ID3 track')
|
||||
tar_path = tmp_path / 'album.tar.gz'
|
||||
with tarfile.open(tar_path, 'w:gz') as tf:
|
||||
tf.add(payload, arcname='track.mp3')
|
||||
payload.unlink() # remove the source so we can verify the extract recreated it
|
||||
extract_archive(tar_path)
|
||||
assert (tmp_path / 'track.mp3').exists()
|
||||
|
||||
|
||||
def test_extract_tar_rejects_path_traversal(tmp_path: Path) -> None:
|
||||
extract_dest = tmp_path / 'staging'
|
||||
extract_dest.mkdir()
|
||||
tar_path = tmp_path / 'evil.tar'
|
||||
payload = tmp_path / 'src.txt'
|
||||
payload.write_bytes(b'evil')
|
||||
with tarfile.open(tar_path, 'w') as tf:
|
||||
info = tf.gettarinfo(str(payload), arcname='../escaped.txt')
|
||||
with payload.open('rb') as fh:
|
||||
tf.addfile(info, fh)
|
||||
extract_archive(tar_path, extract_to=extract_dest)
|
||||
assert not (tmp_path / 'escaped.txt').exists()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# extract_all_in_dir + collect_audio_after_extraction
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_extract_all_in_dir_handles_multiple_archives(tmp_path: Path) -> None:
|
||||
(tmp_path / 'one.zip') # placeholder
|
||||
z1 = tmp_path / 'one.zip'
|
||||
z2 = tmp_path / 'two.zip'
|
||||
with zipfile.ZipFile(z1, 'w') as zf:
|
||||
zf.writestr('a.mp3', b'a')
|
||||
with zipfile.ZipFile(z2, 'w') as zf:
|
||||
zf.writestr('b.mp3', b'b')
|
||||
out = extract_all_in_dir(tmp_path)
|
||||
assert len(out) == 2
|
||||
assert (tmp_path / 'a.mp3').exists()
|
||||
assert (tmp_path / 'b.mp3').exists()
|
||||
|
||||
|
||||
def test_collect_audio_after_extraction_combines_loose_and_extracted(tmp_path: Path) -> None:
|
||||
"""The typical mixed case: torrent dropped a .zip and also some
|
||||
loose .mp3 files alongside it. The collector returns BOTH."""
|
||||
(tmp_path / 'bonus.mp3').write_bytes(b'ID3')
|
||||
zip_path = tmp_path / 'main.zip'
|
||||
with zipfile.ZipFile(zip_path, 'w') as zf:
|
||||
zf.writestr('track1.flac', b'fLaC')
|
||||
zf.writestr('track2.flac', b'fLaC')
|
||||
found = collect_audio_after_extraction(tmp_path)
|
||||
names = sorted(p.name for p in found)
|
||||
assert names == ['bonus.mp3', 'track1.flac', 'track2.flac']
|
||||
|
||||
|
||||
def test_collect_audio_after_extraction_no_archives_no_audio(tmp_path: Path) -> None:
|
||||
assert collect_audio_after_extraction(tmp_path) == []
|
||||
98
tests/test_database_io_resilience.py
Normal file
98
tests/test_database_io_resilience.py
Normal file
|
|
@ -0,0 +1,98 @@
|
|||
import sqlite3
|
||||
|
||||
from database.music_database import MusicDatabase
|
||||
|
||||
|
||||
def test_clear_server_data_does_not_fail_when_vacuum_hits_disk_io():
|
||||
db = object.__new__(MusicDatabase)
|
||||
|
||||
class _Cursor:
|
||||
rowcount = 0
|
||||
|
||||
def __init__(self):
|
||||
self.calls = []
|
||||
|
||||
def execute(self, query, params=None):
|
||||
self.calls.append((query, params))
|
||||
if query == "VACUUM":
|
||||
raise sqlite3.OperationalError("disk I/O error")
|
||||
if "tracks" in query:
|
||||
self.rowcount = 1500
|
||||
elif "albums" in query:
|
||||
self.rowcount = 200
|
||||
elif "artists" in query:
|
||||
self.rowcount = 20
|
||||
|
||||
class _Conn:
|
||||
def __init__(self):
|
||||
self.cursor_obj = _Cursor()
|
||||
self.commits = 0
|
||||
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def __exit__(self, exc_type, exc, tb):
|
||||
return False
|
||||
|
||||
def cursor(self):
|
||||
return self.cursor_obj
|
||||
|
||||
def commit(self):
|
||||
self.commits += 1
|
||||
|
||||
conn = _Conn()
|
||||
db._get_connection = lambda: conn
|
||||
|
||||
db.clear_server_data("jellyfin")
|
||||
|
||||
assert conn.commits == 1
|
||||
assert any(call[0] == "VACUUM" for call in conn.cursor_obj.calls)
|
||||
|
||||
|
||||
def test_clear_server_data_retries_transient_disk_io_before_commit(monkeypatch):
|
||||
db = object.__new__(MusicDatabase)
|
||||
connections = []
|
||||
|
||||
class _Cursor:
|
||||
rowcount = 0
|
||||
|
||||
def __init__(self, fail_first_delete=False):
|
||||
self.fail_first_delete = fail_first_delete
|
||||
self.calls = []
|
||||
|
||||
def execute(self, query, params=None):
|
||||
self.calls.append((query, params))
|
||||
if self.fail_first_delete and "DELETE FROM tracks" in query:
|
||||
self.fail_first_delete = False
|
||||
raise sqlite3.OperationalError("disk I/O error")
|
||||
self.rowcount = 1
|
||||
|
||||
class _Conn:
|
||||
def __init__(self, fail_first_delete=False):
|
||||
self.cursor_obj = _Cursor(fail_first_delete=fail_first_delete)
|
||||
self.commits = 0
|
||||
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def __exit__(self, exc_type, exc, tb):
|
||||
return False
|
||||
|
||||
def cursor(self):
|
||||
return self.cursor_obj
|
||||
|
||||
def commit(self):
|
||||
self.commits += 1
|
||||
|
||||
def _connect():
|
||||
conn = _Conn(fail_first_delete=not connections)
|
||||
connections.append(conn)
|
||||
return conn
|
||||
|
||||
db._get_connection = _connect
|
||||
monkeypatch.setattr("database.music_database.time.sleep", lambda _seconds: None)
|
||||
|
||||
db.clear_server_data("jellyfin")
|
||||
|
||||
assert len(connections) == 2
|
||||
assert connections[1].commits == 1
|
||||
|
|
@ -85,6 +85,7 @@ def test_default_registry_registers_all_sources():
|
|||
expected = {
|
||||
'soulseek', 'youtube', 'tidal', 'qobuz',
|
||||
'hifi', 'deezer', 'lidarr', 'soundcloud', 'amazon',
|
||||
'torrent', 'usenet',
|
||||
}
|
||||
assert set(registry.names()) == expected
|
||||
|
||||
|
|
|
|||
|
|
@ -69,6 +69,53 @@ def test_file_size_column_exists_after_init(db: MusicDatabase) -> None:
|
|||
assert 'file_size' in cols
|
||||
|
||||
|
||||
def test_legacy_media_schema_repairs_required_refresh_columns(tmp_path: Path) -> None:
|
||||
"""Upgraded installs can have old library tables plus migration markers.
|
||||
Startup must repair the columns full refresh writes later."""
|
||||
db_path = tmp_path / 'legacy_missing_media_columns.db'
|
||||
conn = sqlite3.connect(str(db_path))
|
||||
cur = conn.cursor()
|
||||
cur.execute("CREATE TABLE metadata (key TEXT PRIMARY KEY, value TEXT)")
|
||||
cur.execute("INSERT INTO metadata (key, value) VALUES ('id_columns_migrated', 'true')")
|
||||
cur.execute("""
|
||||
CREATE TABLE artists (
|
||||
id TEXT PRIMARY KEY,
|
||||
name TEXT NOT NULL
|
||||
)
|
||||
""")
|
||||
cur.execute("""
|
||||
CREATE TABLE albums (
|
||||
id TEXT PRIMARY KEY,
|
||||
artist_id TEXT NOT NULL,
|
||||
title TEXT NOT NULL
|
||||
)
|
||||
""")
|
||||
cur.execute("""
|
||||
CREATE TABLE tracks (
|
||||
id TEXT PRIMARY KEY,
|
||||
album_id TEXT NOT NULL,
|
||||
artist_id TEXT NOT NULL,
|
||||
title TEXT NOT NULL,
|
||||
file_path TEXT,
|
||||
bitrate INTEGER
|
||||
)
|
||||
""")
|
||||
conn.commit()
|
||||
conn.close()
|
||||
|
||||
repaired = MusicDatabase(database_path=str(db_path))
|
||||
conn = repaired._get_connection()
|
||||
cur = conn.cursor()
|
||||
cur.execute("PRAGMA table_info(tracks)")
|
||||
track_cols = {row[1] for row in cur.fetchall()}
|
||||
cur.execute("PRAGMA table_info(albums)")
|
||||
album_cols = {row[1] for row in cur.fetchall()}
|
||||
conn.close()
|
||||
|
||||
assert 'file_size' in track_cols
|
||||
assert 'api_track_count' in album_cols
|
||||
|
||||
|
||||
def test_existing_tracks_have_null_file_size_after_migration(db: MusicDatabase) -> None:
|
||||
"""Backward-compat: rows inserted via the OLD schema (no file_size)
|
||||
must still be readable, and querying file_size returns NULL — not
|
||||
|
|
|
|||
|
|
@ -165,3 +165,132 @@ def test_monitor_waits_for_post_processing_before_batch_success(monkeypatch):
|
|||
dm.download_batches.update(previous_batches)
|
||||
|
||||
|
||||
def test_monitor_matches_release_download_by_id_when_filename_changes(monkeypatch):
|
||||
"""Torrent/usenet rows can expose the completed audio filename, not
|
||||
the original indexer URL/title stored on the task. The monitor must
|
||||
still claim the completed release by stable download_id.
|
||||
"""
|
||||
monkeypatch.setattr(dm, '_make_context_key', lambda u, f: f"{u}::{f}")
|
||||
monkeypatch.setattr(dm.WebUIDownloadMonitor, '_validate_worker_counts', lambda self: None)
|
||||
|
||||
submitted = []
|
||||
|
||||
class FakeExecutor:
|
||||
def submit(self, func, task_id, batch_id):
|
||||
submitted.append((func, task_id, batch_id))
|
||||
|
||||
def fake_post_processing_worker(task_id, batch_id):
|
||||
return None
|
||||
|
||||
monkeypatch.setattr(dm, 'missing_download_executor', FakeExecutor())
|
||||
monkeypatch.setattr(dm, '_run_post_processing_worker', fake_post_processing_worker)
|
||||
monkeypatch.setattr(dm, '_on_download_completed', lambda *args: None)
|
||||
|
||||
with dm.tasks_lock:
|
||||
previous_tasks = dict(dm.download_tasks)
|
||||
previous_batches = dict(dm.download_batches)
|
||||
dm.download_tasks.clear()
|
||||
dm.download_batches.clear()
|
||||
dm.download_tasks['task-1'] = {
|
||||
'track_info': {'name': 'Ran To Atlanta'},
|
||||
'username': 'torrent',
|
||||
'filename': 'http://prowlarr/download?id=123||Drake - ICEMAN',
|
||||
'status': 'downloading',
|
||||
'download_id': 'torrent-1',
|
||||
'status_change_time': time.time(),
|
||||
}
|
||||
dm.download_batches['batch-1'] = {'queue': ['task-1']}
|
||||
|
||||
try:
|
||||
monitor = dm.WebUIDownloadMonitor()
|
||||
monitor.monitoring = True
|
||||
monitor.monitored_batches.add('batch-1')
|
||||
monkeypatch.setattr(
|
||||
monitor,
|
||||
'_get_live_transfers',
|
||||
lambda: {
|
||||
'download_id::torrent-1': {
|
||||
'id': 'torrent-1',
|
||||
'username': 'torrent',
|
||||
'filename': '01. Drake - Make Them Cry.flac',
|
||||
'state': 'Completed, Succeeded',
|
||||
'size': 100,
|
||||
'bytesTransferred': 100,
|
||||
}
|
||||
},
|
||||
)
|
||||
|
||||
monitor._check_all_downloads()
|
||||
|
||||
assert submitted == [(fake_post_processing_worker, 'task-1', 'batch-1')]
|
||||
assert dm.download_tasks['task-1']['status'] == 'post_processing'
|
||||
finally:
|
||||
with dm.tasks_lock:
|
||||
dm.download_tasks.clear()
|
||||
dm.download_tasks.update(previous_tasks)
|
||||
dm.download_batches.clear()
|
||||
dm.download_batches.update(previous_batches)
|
||||
|
||||
|
||||
def test_monitor_recovers_premature_failed_release_download(monkeypatch):
|
||||
monkeypatch.setattr(dm, '_make_context_key', lambda u, f: f"{u}::{f}")
|
||||
monkeypatch.setattr(dm.WebUIDownloadMonitor, '_validate_worker_counts', lambda self: None)
|
||||
|
||||
submitted = []
|
||||
|
||||
class FakeExecutor:
|
||||
def submit(self, func, task_id, batch_id):
|
||||
submitted.append((func, task_id, batch_id))
|
||||
|
||||
def fake_post_processing_worker(task_id, batch_id):
|
||||
return None
|
||||
|
||||
monkeypatch.setattr(dm, 'missing_download_executor', FakeExecutor())
|
||||
monkeypatch.setattr(dm, '_run_post_processing_worker', fake_post_processing_worker)
|
||||
monkeypatch.setattr(dm, '_on_download_completed', lambda *args: None)
|
||||
|
||||
with dm.tasks_lock:
|
||||
previous_tasks = dict(dm.download_tasks)
|
||||
previous_batches = dict(dm.download_batches)
|
||||
dm.download_tasks.clear()
|
||||
dm.download_batches.clear()
|
||||
dm.download_tasks['task-1'] = {
|
||||
'track_info': {'name': 'DAISIES'},
|
||||
'username': 'torrent',
|
||||
'filename': 'http://prowlarr/download?id=123||Justin Bieber - Swag',
|
||||
'status': 'failed',
|
||||
'download_id': 'torrent-1',
|
||||
'status_change_time': time.time(),
|
||||
}
|
||||
dm.download_batches['batch-1'] = {'queue': ['task-1']}
|
||||
|
||||
try:
|
||||
monitor = dm.WebUIDownloadMonitor()
|
||||
monitor.monitoring = True
|
||||
monitor.monitored_batches.add('batch-1')
|
||||
monkeypatch.setattr(
|
||||
monitor,
|
||||
'_get_live_transfers',
|
||||
lambda: {
|
||||
'download_id::torrent-1': {
|
||||
'id': 'torrent-1',
|
||||
'username': 'torrent',
|
||||
'filename': '02. Justin Bieber - DAISIES.flac',
|
||||
'state': 'Completed, Succeeded',
|
||||
'size': 100,
|
||||
'bytesTransferred': 100,
|
||||
}
|
||||
},
|
||||
)
|
||||
|
||||
monitor._check_all_downloads()
|
||||
|
||||
assert submitted == [(fake_post_processing_worker, 'task-1', 'batch-1')]
|
||||
assert dm.download_tasks['task-1']['status'] == 'post_processing'
|
||||
finally:
|
||||
with dm.tasks_lock:
|
||||
dm.download_tasks.clear()
|
||||
dm.download_tasks.update(previous_tasks)
|
||||
dm.download_batches.clear()
|
||||
dm.download_batches.update(previous_batches)
|
||||
|
||||
|
|
|
|||
|
|
@ -44,6 +44,167 @@ if "config.settings" not in sys.modules:
|
|||
from core.repair_worker import RepairWorker
|
||||
|
||||
|
||||
def test_incomplete_album_auto_fill_skips_source_artist_mismatch(tmp_path):
|
||||
"""Album Completeness must never fill a target album with another artist."""
|
||||
existing_path = tmp_path / "Gut" / "Light Years" / "01 - Wound Fuck.flac"
|
||||
existing_path.parent.mkdir(parents=True)
|
||||
existing_path.write_bytes(b"existing")
|
||||
|
||||
candidate_path = tmp_path / "Jamiroquai" / "Light Years.flac"
|
||||
candidate_path.parent.mkdir(parents=True)
|
||||
candidate_path.write_bytes(b"candidate")
|
||||
|
||||
class _FakeDB:
|
||||
def __init__(self):
|
||||
self.search_calls = []
|
||||
self.wishlist = []
|
||||
|
||||
def get_tracks_by_album(self, album_id):
|
||||
if album_id == "target-album":
|
||||
return [
|
||||
SimpleNamespace(
|
||||
id="target-existing-1",
|
||||
album_id="target-album",
|
||||
artist_id="gut",
|
||||
title="Wound Fuck",
|
||||
track_number=1,
|
||||
file_path=str(existing_path),
|
||||
bitrate=9999,
|
||||
),
|
||||
]
|
||||
return []
|
||||
|
||||
def search_tracks(self, title="", artist="", limit=50, server_source=None):
|
||||
self.search_calls.append((title, artist, limit, server_source))
|
||||
return [
|
||||
SimpleNamespace(
|
||||
id="candidate-1",
|
||||
album_id="single-album",
|
||||
artist_id="jamiroquai",
|
||||
artist_name="Jamiroquai",
|
||||
title="Light Years",
|
||||
track_number=1,
|
||||
file_path=str(candidate_path),
|
||||
bitrate=9999,
|
||||
),
|
||||
]
|
||||
|
||||
def add_to_wishlist(self, *args, **kwargs):
|
||||
self.wishlist.append((args, kwargs))
|
||||
|
||||
worker = RepairWorker.__new__(RepairWorker)
|
||||
worker.db = _FakeDB()
|
||||
worker.transfer_folder = str(tmp_path)
|
||||
worker._config_manager = None
|
||||
|
||||
result = worker._fix_incomplete_album(
|
||||
"album",
|
||||
"target-album",
|
||||
None,
|
||||
{
|
||||
"album_id": "target-album",
|
||||
"album_title": "Light Years",
|
||||
"artist": "Gut",
|
||||
"missing_tracks": [
|
||||
{
|
||||
"name": "Light Years",
|
||||
"track_number": 2,
|
||||
"disc_number": 1,
|
||||
"source": "spotify",
|
||||
"source_track_id": "sp-light-years",
|
||||
"artists": ["Jamiroquai"],
|
||||
},
|
||||
],
|
||||
},
|
||||
)
|
||||
|
||||
assert result["success"] is False
|
||||
assert result["fixed"] == 0
|
||||
assert result["skipped"] == 1
|
||||
assert result["details"][0]["reason"] == "source artist does not match target album artist"
|
||||
assert worker.db.search_calls == []
|
||||
assert worker.db.wishlist == []
|
||||
|
||||
|
||||
def test_incomplete_album_auto_fill_rejects_wrong_artist_candidate(tmp_path):
|
||||
"""Exact title is not enough when the candidate artist differs."""
|
||||
existing_path = tmp_path / "album" / "01 - Existing.flac"
|
||||
existing_path.parent.mkdir(parents=True)
|
||||
existing_path.write_bytes(b"existing")
|
||||
candidate_path = tmp_path / "wrong" / "02 - Light Years.flac"
|
||||
candidate_path.parent.mkdir(parents=True)
|
||||
candidate_path.write_bytes(b"wrong")
|
||||
|
||||
class _FakeDB:
|
||||
def __init__(self):
|
||||
self.wishlist = []
|
||||
|
||||
def get_tracks_by_album(self, album_id):
|
||||
if album_id == "target-album":
|
||||
return [
|
||||
SimpleNamespace(
|
||||
id="target-existing-1",
|
||||
album_id="target-album",
|
||||
artist_id="jamiroquai",
|
||||
title="Existing",
|
||||
track_number=1,
|
||||
file_path=str(existing_path),
|
||||
bitrate=9999,
|
||||
),
|
||||
]
|
||||
return []
|
||||
|
||||
def search_tracks(self, title="", artist="", limit=50, server_source=None):
|
||||
return [
|
||||
SimpleNamespace(
|
||||
id="candidate-1",
|
||||
album_id="wrong-album",
|
||||
artist_id="gut",
|
||||
artist_name="Gut",
|
||||
title="Light Years",
|
||||
track_number=1,
|
||||
file_path=str(candidate_path),
|
||||
bitrate=9999,
|
||||
),
|
||||
]
|
||||
|
||||
def add_to_wishlist(self, *args, **kwargs):
|
||||
self.wishlist.append((args, kwargs))
|
||||
|
||||
worker = RepairWorker.__new__(RepairWorker)
|
||||
worker.db = _FakeDB()
|
||||
worker.transfer_folder = str(tmp_path)
|
||||
worker._config_manager = None
|
||||
|
||||
result = worker._fix_incomplete_album(
|
||||
"album",
|
||||
"target-album",
|
||||
None,
|
||||
{
|
||||
"album_id": "target-album",
|
||||
"album_title": "Light Years",
|
||||
"artist": "Jamiroquai",
|
||||
"spotify_album_id": "sp-album",
|
||||
"expected_tracks": 2,
|
||||
"missing_tracks": [
|
||||
{
|
||||
"name": "Light Years",
|
||||
"track_number": 2,
|
||||
"disc_number": 1,
|
||||
"source": "spotify",
|
||||
"source_track_id": "sp-light-years",
|
||||
"artists": ["Jamiroquai"],
|
||||
},
|
||||
],
|
||||
},
|
||||
)
|
||||
|
||||
assert result["success"] is True
|
||||
assert result["fixed"] == 0
|
||||
assert result["wishlisted"] == 1
|
||||
assert worker.db.wishlist
|
||||
|
||||
|
||||
def test_perform_album_fill_copy_branch_generates_track_id(tmp_path, monkeypatch):
|
||||
src_path = tmp_path / "source.flac"
|
||||
src_path.write_bytes(b"fake-audio")
|
||||
|
|
|
|||
198
tests/test_staging_album_provenance.py
Normal file
198
tests/test_staging_album_provenance.py
Normal file
|
|
@ -0,0 +1,198 @@
|
|||
"""Tests for the album-bundle provenance override in
|
||||
``core/downloads/staging.py``.
|
||||
|
||||
Verifies that when ``StagingDeps.get_batch_field`` returns a source
|
||||
override (i.e. the batch was populated by the torrent / usenet
|
||||
album-bundle flow), the staging matcher records that source on the
|
||||
task instead of the generic 'staging' username. Provenance recording
|
||||
downstream uses ``task['username']`` to set ``source_service`` on
|
||||
the persisted download row — so this is the single point that
|
||||
controls whether the history modal shows 'Torrent' / 'Usenet' vs
|
||||
'Staging' / 'Soulseek'.
|
||||
|
||||
Mocks the rest of StagingDeps so the test doesn't touch the
|
||||
filesystem, AcoustID, or post-processing.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
from core.downloads.staging import StagingDeps, try_staging_match
|
||||
from core.runtime_state import download_tasks, tasks_lock
|
||||
|
||||
|
||||
def _make_deps(staging_file, transfer_dir, batch_field_value=None):
|
||||
"""Build a StagingDeps with mocked collaborators. ``batch_field_value``
|
||||
is what get_batch_field returns for ``album_bundle_source`` — None
|
||||
means no override (generic staging match), 'torrent' / 'usenet'
|
||||
means the album-bundle flow seeded the staging folder."""
|
||||
me = MagicMock()
|
||||
me.normalize_string.side_effect = lambda s: (s or '').lower().strip()
|
||||
config = MagicMock()
|
||||
config.get.return_value = transfer_dir
|
||||
return StagingDeps(
|
||||
config_manager=config,
|
||||
matching_engine=me,
|
||||
get_staging_file_cache=lambda _b: [staging_file],
|
||||
docker_resolve_path=lambda p: p,
|
||||
post_process_matched_download_with_verification=lambda *a, **kw: None,
|
||||
get_batch_field=(lambda _b, _f: batch_field_value) if batch_field_value is not None else (lambda _b, _f: None),
|
||||
)
|
||||
|
||||
|
||||
def _seed_task(task_id: str, track_name: str, track_artist: str) -> None:
|
||||
"""Register a task in the runtime_state dict so try_staging_match
|
||||
has something to mark complete."""
|
||||
with tasks_lock:
|
||||
download_tasks[task_id] = {
|
||||
'status': 'searching',
|
||||
'track_info': {
|
||||
'name': track_name,
|
||||
'artists': [{'name': track_artist}],
|
||||
'_is_explicit_album_download': False,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def _cleanup_task(task_id: str) -> None:
|
||||
with tasks_lock:
|
||||
download_tasks.pop(task_id, None)
|
||||
|
||||
|
||||
def test_staging_match_uses_torrent_override_when_present(tmp_path) -> None:
|
||||
src = tmp_path / 'staging' / 'gnx_track_01.flac'
|
||||
src.parent.mkdir()
|
||||
src.write_bytes(b'fLaC')
|
||||
transfer = tmp_path / 'transfer'
|
||||
deps = _make_deps(
|
||||
staging_file={'full_path': str(src), 'title': 'Luther', 'artist': 'Kendrick Lamar'},
|
||||
transfer_dir=str(transfer),
|
||||
batch_field_value='torrent',
|
||||
)
|
||||
track = SimpleNamespace(name='Luther', artists=['Kendrick Lamar'])
|
||||
task_id = 'test_task_torrent_override'
|
||||
_seed_task(task_id, 'Luther', 'Kendrick Lamar')
|
||||
try:
|
||||
ok = try_staging_match(task_id, 'batch_x', track, deps)
|
||||
assert ok is True
|
||||
with tasks_lock:
|
||||
row = download_tasks[task_id]
|
||||
assert row['username'] == 'torrent', \
|
||||
f"Expected provenance override 'torrent', got {row['username']!r}"
|
||||
assert row['staging_match'] is True
|
||||
finally:
|
||||
_cleanup_task(task_id)
|
||||
|
||||
|
||||
def test_staging_match_uses_usenet_override_when_present(tmp_path) -> None:
|
||||
src = tmp_path / 'staging' / 'gnx_track_01.flac'
|
||||
src.parent.mkdir()
|
||||
src.write_bytes(b'fLaC')
|
||||
transfer = tmp_path / 'transfer'
|
||||
deps = _make_deps(
|
||||
staging_file={'full_path': str(src), 'title': 'Luther', 'artist': 'Kendrick Lamar'},
|
||||
transfer_dir=str(transfer),
|
||||
batch_field_value='usenet',
|
||||
)
|
||||
track = SimpleNamespace(name='Luther', artists=['Kendrick Lamar'])
|
||||
task_id = 'test_task_usenet_override'
|
||||
_seed_task(task_id, 'Luther', 'Kendrick Lamar')
|
||||
try:
|
||||
try_staging_match(task_id, 'batch_x', track, deps)
|
||||
with tasks_lock:
|
||||
assert download_tasks[task_id]['username'] == 'usenet'
|
||||
finally:
|
||||
_cleanup_task(task_id)
|
||||
|
||||
|
||||
def test_staging_match_falls_back_to_staging_without_override(tmp_path) -> None:
|
||||
"""When no batch override is present (manual file drop, or
|
||||
batch has no album_bundle_source field), the staging matcher
|
||||
uses the historical 'staging' username."""
|
||||
src = tmp_path / 'staging' / 'gnx_track_01.flac'
|
||||
src.parent.mkdir()
|
||||
src.write_bytes(b'fLaC')
|
||||
transfer = tmp_path / 'transfer'
|
||||
deps = _make_deps(
|
||||
staging_file={'full_path': str(src), 'title': 'Luther', 'artist': 'Kendrick Lamar'},
|
||||
transfer_dir=str(transfer),
|
||||
batch_field_value=None,
|
||||
)
|
||||
track = SimpleNamespace(name='Luther', artists=['Kendrick Lamar'])
|
||||
task_id = 'test_task_no_override'
|
||||
_seed_task(task_id, 'Luther', 'Kendrick Lamar')
|
||||
try:
|
||||
try_staging_match(task_id, 'batch_x', track, deps)
|
||||
with tasks_lock:
|
||||
assert download_tasks[task_id]['username'] == 'staging'
|
||||
finally:
|
||||
_cleanup_task(task_id)
|
||||
|
||||
|
||||
def test_staging_match_handles_missing_batch_field_callable(tmp_path) -> None:
|
||||
"""Backward compat: callers that build StagingDeps without
|
||||
supplying ``get_batch_field`` (it defaults to None) still
|
||||
work — staging matcher falls back to the 'staging' username."""
|
||||
src = tmp_path / 'staging' / 'gnx_track_01.flac'
|
||||
src.parent.mkdir()
|
||||
src.write_bytes(b'fLaC')
|
||||
transfer = tmp_path / 'transfer'
|
||||
|
||||
me = MagicMock()
|
||||
me.normalize_string.side_effect = lambda s: (s or '').lower().strip()
|
||||
config = MagicMock()
|
||||
config.get.return_value = str(transfer)
|
||||
deps = StagingDeps(
|
||||
config_manager=config,
|
||||
matching_engine=me,
|
||||
get_staging_file_cache=lambda _b: [{'full_path': str(src), 'title': 'Luther', 'artist': 'Kendrick Lamar'}],
|
||||
docker_resolve_path=lambda p: p,
|
||||
post_process_matched_download_with_verification=lambda *a, **kw: None,
|
||||
# get_batch_field omitted — defaults to None
|
||||
)
|
||||
track = SimpleNamespace(name='Luther', artists=['Kendrick Lamar'])
|
||||
task_id = 'test_task_no_accessor'
|
||||
_seed_task(task_id, 'Luther', 'Kendrick Lamar')
|
||||
try:
|
||||
try_staging_match(task_id, 'batch_x', track, deps)
|
||||
with tasks_lock:
|
||||
assert download_tasks[task_id]['username'] == 'staging'
|
||||
finally:
|
||||
_cleanup_task(task_id)
|
||||
|
||||
|
||||
def test_staging_match_swallows_accessor_exception(tmp_path) -> None:
|
||||
"""If the injected accessor raises (e.g. the batch was deleted
|
||||
mid-process), the staging matcher should fall back to 'staging'
|
||||
rather than failing the whole match."""
|
||||
src = tmp_path / 'staging' / 'gnx_track_01.flac'
|
||||
src.parent.mkdir()
|
||||
src.write_bytes(b'fLaC')
|
||||
transfer = tmp_path / 'transfer'
|
||||
|
||||
def _boom(_b, _f):
|
||||
raise RuntimeError("batch went away")
|
||||
|
||||
me = MagicMock()
|
||||
me.normalize_string.side_effect = lambda s: (s or '').lower().strip()
|
||||
config = MagicMock()
|
||||
config.get.return_value = str(transfer)
|
||||
deps = StagingDeps(
|
||||
config_manager=config,
|
||||
matching_engine=me,
|
||||
get_staging_file_cache=lambda _b: [{'full_path': str(src), 'title': 'Luther', 'artist': 'Kendrick Lamar'}],
|
||||
docker_resolve_path=lambda p: p,
|
||||
post_process_matched_download_with_verification=lambda *a, **kw: None,
|
||||
get_batch_field=_boom,
|
||||
)
|
||||
track = SimpleNamespace(name='Luther', artists=['Kendrick Lamar'])
|
||||
task_id = 'test_task_accessor_raises'
|
||||
_seed_task(task_id, 'Luther', 'Kendrick Lamar')
|
||||
try:
|
||||
try_staging_match(task_id, 'batch_x', track, deps)
|
||||
with tasks_lock:
|
||||
assert download_tasks[task_id]['username'] == 'staging'
|
||||
finally:
|
||||
_cleanup_task(task_id)
|
||||
479
tests/test_torrent_usenet_plugins.py
Normal file
479
tests/test_torrent_usenet_plugins.py
Normal file
|
|
@ -0,0 +1,479 @@
|
|||
"""Tests for ``core/download_plugins/torrent.py`` and ``usenet.py``.
|
||||
|
||||
Both plugins compose a Prowlarr client + an adapter + the archive
|
||||
pipeline. The tests mock the Prowlarr client and the active adapter
|
||||
factory so we can pin the projection logic, filename encoding /
|
||||
decoding, finalize path, and the cancel / clear lifecycle without
|
||||
touching the network or filesystem (beyond ``tmp_path``).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
from pathlib import Path
|
||||
from typing import Optional
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
from core.download_plugins.torrent import (
|
||||
TorrentDownloadPlugin,
|
||||
_adapter_state_to_display,
|
||||
_decode_filename,
|
||||
_FILENAME_SEP,
|
||||
_guess_quality_from_title,
|
||||
_parse_release_title,
|
||||
)
|
||||
from core.download_plugins.usenet import UsenetDownloadPlugin
|
||||
from core.prowlarr_client import ProwlarrSearchResult
|
||||
from core.torrent_clients.base import TorrentStatus
|
||||
from core.usenet_clients.base import UsenetStatus
|
||||
|
||||
|
||||
def _run(coro):
|
||||
return asyncio.new_event_loop().run_until_complete(coro)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Pure helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_decode_filename_splits_on_separator() -> None:
|
||||
url, display = _decode_filename(f"https://x/y.torrent{_FILENAME_SEP}Album Name")
|
||||
assert url == 'https://x/y.torrent'
|
||||
assert display == 'Album Name'
|
||||
|
||||
|
||||
def test_decode_filename_without_separator_returns_none_url() -> None:
|
||||
url, display = _decode_filename('just a name')
|
||||
assert url is None
|
||||
assert display == 'just a name'
|
||||
|
||||
|
||||
def test_decode_filename_handles_magnet_with_embedded_separators() -> None:
|
||||
"""Magnet URIs contain ``=`` and ``&`` but no ``||`` — so a
|
||||
magnet must round-trip cleanly through the encoder."""
|
||||
magnet = 'magnet:?xt=urn:btih:abc123&dn=Album+Name'
|
||||
encoded = f"{magnet}{_FILENAME_SEP}Display"
|
||||
url, display = _decode_filename(encoded)
|
||||
assert url == magnet
|
||||
assert display == 'Display'
|
||||
|
||||
|
||||
def test_guess_quality_from_title() -> None:
|
||||
assert _guess_quality_from_title('Album [FLAC]') == 'flac'
|
||||
assert _guess_quality_from_title('Album 24-bit Hi-Res') == 'flac'
|
||||
assert _guess_quality_from_title('Album [MP3 320]') == 'mp3'
|
||||
assert _guess_quality_from_title('Album [AAC 256]') == 'aac'
|
||||
assert _guess_quality_from_title('Album [OGG]') == 'ogg'
|
||||
# Default fallback so quality_score doesn't crash on bare titles.
|
||||
assert _guess_quality_from_title('Just A Title') == 'mp3'
|
||||
assert _guess_quality_from_title('') == 'mp3'
|
||||
|
||||
|
||||
def test_parse_release_title_splits_artist_dash_title() -> None:
|
||||
"""Most release titles follow 'Artist - Title' / 'Artist - Album'."""
|
||||
assert _parse_release_title('Danny Brown - Atrocity Exhibition') == ('Danny Brown', 'Atrocity Exhibition')
|
||||
assert _parse_release_title('Kendrick Lamar - DAMN.') == ('Kendrick Lamar', 'DAMN.')
|
||||
|
||||
|
||||
def test_parse_release_title_strips_trailing_tags() -> None:
|
||||
"""Quality / year tags at the end shouldn't pollute the title."""
|
||||
artist, title = _parse_release_title('Danny Brown - Atrocity Exhibition [FLAC]')
|
||||
assert artist == 'Danny Brown'
|
||||
assert title == 'Atrocity Exhibition'
|
||||
artist, title = _parse_release_title('Danny Brown - Atrocity Exhibition (2016)')
|
||||
assert artist == 'Danny Brown'
|
||||
assert title == 'Atrocity Exhibition'
|
||||
|
||||
|
||||
def test_parse_release_title_handles_no_dash() -> None:
|
||||
"""Some indexers post bare titles. Caller should fall back to
|
||||
the indexer name as the 'artist' field."""
|
||||
artist, title = _parse_release_title('JustATitle')
|
||||
assert artist == ''
|
||||
assert title == 'JustATitle'
|
||||
|
||||
|
||||
def test_parse_release_title_handles_dashes_in_title() -> None:
|
||||
"""Track titles can themselves contain dashes — only split on
|
||||
the FIRST one so subtitles survive."""
|
||||
artist, title = _parse_release_title('Artist - Title - Live Version')
|
||||
assert artist == 'Artist'
|
||||
assert title == 'Title - Live Version'
|
||||
|
||||
|
||||
def test_parse_release_title_rejects_url_prefix() -> None:
|
||||
"""Defensive: if a URL somehow lands in the title field, refuse
|
||||
to call it an artist."""
|
||||
artist, title = _parse_release_title('https://example.com/x - Album')
|
||||
assert artist == ''
|
||||
|
||||
|
||||
def test_adapter_state_mapping_covers_complete_states() -> None:
|
||||
assert _adapter_state_to_display('downloading') == 'InProgress, Downloading'
|
||||
assert _adapter_state_to_display('seeding') == 'Completed, Succeeded'
|
||||
assert _adapter_state_to_display('completed') == 'Completed, Succeeded'
|
||||
assert _adapter_state_to_display('error') == 'Completed, Errored'
|
||||
assert _adapter_state_to_display('stalled') == 'InProgress, Stalled'
|
||||
# Unknown state falls through with title-casing rather than crashing.
|
||||
assert _adapter_state_to_display('weird') == 'Weird'
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Torrent plugin — search projection
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _make_torrent_result(**overrides) -> ProwlarrSearchResult:
|
||||
base = dict(
|
||||
guid='guid-1', title='Danny Brown - Atrocity Exhibition [FLAC]', indexer_id=3,
|
||||
indexer_name='Indexer', protocol='torrent',
|
||||
download_url='https://x/y.torrent', magnet_uri=None,
|
||||
info_url=None, size=500_000_000, seeders=12, leechers=3,
|
||||
grabs=100, publish_date='2026-01-01', categories=[3040],
|
||||
raw={},
|
||||
)
|
||||
base.update(overrides)
|
||||
return ProwlarrSearchResult(**base)
|
||||
|
||||
|
||||
def test_torrent_project_results_drops_non_torrent_protocol() -> None:
|
||||
plugin = TorrentDownloadPlugin()
|
||||
results = [
|
||||
_make_torrent_result(),
|
||||
_make_torrent_result(protocol='usenet', title='Usenet Album'),
|
||||
]
|
||||
tracks, albums = plugin._project_results(results)
|
||||
assert len(tracks) == 1
|
||||
assert tracks[0].title == 'Atrocity Exhibition'
|
||||
assert tracks[0].artist == 'Danny Brown'
|
||||
assert len(albums) == 1
|
||||
|
||||
|
||||
def test_torrent_project_results_drops_releases_without_download_url() -> None:
|
||||
plugin = TorrentDownloadPlugin()
|
||||
results = [_make_torrent_result(download_url=None, magnet_uri=None)]
|
||||
tracks, albums = plugin._project_results(results)
|
||||
assert tracks == []
|
||||
assert albums == []
|
||||
|
||||
|
||||
def test_torrent_project_results_prefers_magnet_when_available() -> None:
|
||||
plugin = TorrentDownloadPlugin()
|
||||
magnet = 'magnet:?xt=urn:btih:abc'
|
||||
results = [_make_torrent_result(magnet_uri=magnet, download_url='https://x/y.torrent')]
|
||||
tracks, _ = plugin._project_results(results)
|
||||
url, _ = _decode_filename(tracks[0].filename)
|
||||
assert url == magnet
|
||||
|
||||
|
||||
def test_torrent_project_results_encodes_url_and_title_in_filename() -> None:
|
||||
plugin = TorrentDownloadPlugin()
|
||||
tracks, _ = plugin._project_results([_make_torrent_result()])
|
||||
url, display = _decode_filename(tracks[0].filename)
|
||||
assert url == 'https://x/y.torrent'
|
||||
assert display == 'Danny Brown - Atrocity Exhibition [FLAC]'
|
||||
|
||||
|
||||
def test_torrent_project_falls_back_to_indexer_name_when_title_lacks_dash() -> None:
|
||||
"""When the title has no 'Artist -' prefix we'd auto-parse the
|
||||
filename (which starts with the indexer download URL) and end
|
||||
up showing the URL in the UI's 'by' field. Pre-filling artist
|
||||
with the indexer name avoids that."""
|
||||
plugin = TorrentDownloadPlugin()
|
||||
tracks, _ = plugin._project_results([_make_torrent_result(title='JustATitle')])
|
||||
assert tracks[0].artist == 'Indexer'
|
||||
# And the URL is definitely not the artist.
|
||||
assert 'http' not in tracks[0].artist
|
||||
assert '||' not in tracks[0].artist
|
||||
|
||||
|
||||
def test_torrent_project_results_neutralizes_soulseek_specific_fields() -> None:
|
||||
"""TrackResult.quality_score punishes results with no upload
|
||||
slots; torrent results don't have that concept so the
|
||||
projection has to fill in non-punishing neutral values."""
|
||||
plugin = TorrentDownloadPlugin()
|
||||
tracks, _ = plugin._project_results([_make_torrent_result(seeders=0)])
|
||||
# seeders=0 means we should still hand the picker something
|
||||
# usable. free_upload_slots floors at 1 to avoid the 0-slot
|
||||
# penalty applied to dead Soulseek peers.
|
||||
assert tracks[0].free_upload_slots >= 1
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Torrent plugin — is_configured / check_connection
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_torrent_is_configured_requires_both_sides() -> None:
|
||||
plugin = TorrentDownloadPlugin()
|
||||
with patch.object(plugin._prowlarr, 'is_configured', return_value=False), \
|
||||
patch('core.download_plugins.torrent.get_active_torrent_adapter', return_value=None):
|
||||
assert plugin.is_configured() is False
|
||||
fake_adapter = MagicMock()
|
||||
fake_adapter.is_configured.return_value = False
|
||||
with patch.object(plugin._prowlarr, 'is_configured', return_value=True), \
|
||||
patch('core.download_plugins.torrent.get_active_torrent_adapter', return_value=fake_adapter):
|
||||
assert plugin.is_configured() is False
|
||||
fake_adapter.is_configured.return_value = True
|
||||
with patch.object(plugin._prowlarr, 'is_configured', return_value=True), \
|
||||
patch('core.download_plugins.torrent.get_active_torrent_adapter', return_value=fake_adapter):
|
||||
assert plugin.is_configured() is True
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Torrent plugin — finalize / cancel / clear
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_torrent_finalize_picks_first_audio_file(tmp_path: Path) -> None:
|
||||
plugin = TorrentDownloadPlugin()
|
||||
# Seed an in-flight download row
|
||||
plugin.active_downloads['dl-1'] = {
|
||||
'id': 'dl-1', 'filename': 'x', 'username': 'torrent',
|
||||
'display_name': 'X', 'state': 'InProgress, Downloading',
|
||||
'progress': 50.0, 'size': 0, 'transferred': 0, 'speed': 0,
|
||||
'file_path': None, 'torrent_hash': 'h1', 'error': None,
|
||||
}
|
||||
# Drop two audio files in the save dir
|
||||
(tmp_path / 'b.flac').write_bytes(b'fLaC')
|
||||
(tmp_path / 'a.mp3').write_bytes(b'ID3')
|
||||
plugin._finalize_download('dl-1', str(tmp_path))
|
||||
row = plugin.active_downloads['dl-1']
|
||||
assert row['state'] == 'Completed, Succeeded'
|
||||
assert row['progress'] == 100.0
|
||||
# Walker sorts → 'a.mp3' wins as first.
|
||||
assert row['file_path'].endswith('a.mp3')
|
||||
|
||||
|
||||
def test_torrent_finalize_marks_error_when_no_audio(tmp_path: Path) -> None:
|
||||
plugin = TorrentDownloadPlugin()
|
||||
plugin.active_downloads['dl-1'] = {
|
||||
'id': 'dl-1', 'filename': 'x', 'username': 'torrent',
|
||||
'display_name': 'X', 'state': 'InProgress, Downloading',
|
||||
'progress': 50.0, 'size': 0, 'transferred': 0, 'speed': 0,
|
||||
'file_path': None, 'torrent_hash': 'h1', 'error': None,
|
||||
}
|
||||
# tmp_path has no audio files
|
||||
plugin._finalize_download('dl-1', str(tmp_path))
|
||||
assert plugin.active_downloads['dl-1']['state'] == 'Completed, Errored'
|
||||
assert 'No audio files' in plugin.active_downloads['dl-1']['error']
|
||||
|
||||
|
||||
def test_torrent_finalize_marks_error_when_save_path_missing() -> None:
|
||||
plugin = TorrentDownloadPlugin()
|
||||
plugin.active_downloads['dl-1'] = {
|
||||
'id': 'dl-1', 'filename': 'x', 'username': 'torrent',
|
||||
'display_name': 'X', 'state': 'InProgress, Downloading',
|
||||
'progress': 50.0, 'size': 0, 'transferred': 0, 'speed': 0,
|
||||
'file_path': None, 'torrent_hash': 'h1', 'error': None,
|
||||
}
|
||||
plugin._finalize_download('dl-1', None)
|
||||
assert plugin.active_downloads['dl-1']['state'] == 'Completed, Errored'
|
||||
assert 'no save_path' in plugin.active_downloads['dl-1']['error'].lower()
|
||||
|
||||
|
||||
def test_torrent_clear_completed_drops_only_done_rows() -> None:
|
||||
plugin = TorrentDownloadPlugin()
|
||||
plugin.active_downloads['a'] = {'id': 'a', 'state': 'InProgress, Downloading'}
|
||||
plugin.active_downloads['b'] = {'id': 'b', 'state': 'Completed, Succeeded'}
|
||||
plugin.active_downloads['c'] = {'id': 'c', 'state': 'Completed, Errored'}
|
||||
plugin.active_downloads['d'] = {'id': 'd', 'state': 'Cancelled'}
|
||||
_run(plugin.clear_all_completed_downloads())
|
||||
assert list(plugin.active_downloads.keys()) == ['a']
|
||||
|
||||
|
||||
def test_torrent_get_all_returns_status_objects() -> None:
|
||||
plugin = TorrentDownloadPlugin()
|
||||
plugin.active_downloads['a'] = {
|
||||
'id': 'a', 'filename': 'f', 'username': 'torrent',
|
||||
'state': 'InProgress, Downloading', 'progress': 50.0,
|
||||
'size': 100, 'transferred': 50, 'speed': 1000,
|
||||
'file_path': None,
|
||||
}
|
||||
statuses = _run(plugin.get_all_downloads())
|
||||
assert len(statuses) == 1
|
||||
assert statuses[0].id == 'a'
|
||||
assert statuses[0].progress == 50.0
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Usenet plugin — projection
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _make_usenet_result(**overrides) -> ProwlarrSearchResult:
|
||||
base = dict(
|
||||
guid='guid-u', title='Some Artist - Some Album', indexer_id=5,
|
||||
indexer_name='UsenetIndexer', protocol='usenet',
|
||||
download_url='https://x/y.nzb', magnet_uri=None,
|
||||
info_url=None, size=400_000_000, seeders=None, leechers=None,
|
||||
grabs=42, publish_date='2026-01-01', categories=[3010],
|
||||
raw={},
|
||||
)
|
||||
base.update(overrides)
|
||||
return ProwlarrSearchResult(**base)
|
||||
|
||||
|
||||
def test_usenet_project_drops_torrent_protocol() -> None:
|
||||
plugin = UsenetDownloadPlugin()
|
||||
results = [_make_usenet_result(), _make_usenet_result(protocol='torrent', title='T')]
|
||||
tracks, albums = plugin._project_results(results)
|
||||
assert len(tracks) == 1
|
||||
assert tracks[0].username == 'usenet'
|
||||
|
||||
|
||||
def test_usenet_project_drops_results_without_download_url() -> None:
|
||||
"""Usenet plugins reject magnet-only results entirely — NZBs
|
||||
don't have a magnet equivalent."""
|
||||
plugin = UsenetDownloadPlugin()
|
||||
results = [_make_usenet_result(download_url=None)]
|
||||
tracks, _ = plugin._project_results(results)
|
||||
assert tracks == []
|
||||
|
||||
|
||||
def test_usenet_project_encodes_url_in_filename() -> None:
|
||||
plugin = UsenetDownloadPlugin()
|
||||
tracks, _ = plugin._project_results([_make_usenet_result()])
|
||||
url, display = _decode_filename(tracks[0].filename)
|
||||
assert url == 'https://x/y.nzb'
|
||||
assert display == 'Some Artist - Some Album'
|
||||
# Artist + title should be parsed out, not auto-extracted from filename.
|
||||
assert tracks[0].artist == 'Some Artist'
|
||||
assert tracks[0].title == 'Some Album'
|
||||
|
||||
|
||||
def test_usenet_finalize_picks_first_audio_file(tmp_path: Path) -> None:
|
||||
"""Same finalize contract as torrent — sanity check the shared
|
||||
helper path works for usenet too."""
|
||||
plugin = UsenetDownloadPlugin()
|
||||
plugin.active_downloads['u-1'] = {
|
||||
'id': 'u-1', 'filename': 'x', 'username': 'usenet',
|
||||
'display_name': 'X', 'state': 'InProgress, Downloading',
|
||||
'progress': 50.0, 'size': 0, 'transferred': 0, 'speed': 0,
|
||||
'file_path': None, 'job_id': 'j1', 'error': None,
|
||||
}
|
||||
(tmp_path / 'track1.flac').write_bytes(b'fLaC')
|
||||
plugin._finalize_download('u-1', str(tmp_path))
|
||||
assert plugin.active_downloads['u-1']['state'] == 'Completed, Succeeded'
|
||||
assert plugin.active_downloads['u-1']['file_path'].endswith('track1.flac')
|
||||
|
||||
|
||||
def test_usenet_is_configured_requires_both_sides() -> None:
|
||||
plugin = UsenetDownloadPlugin()
|
||||
fake_adapter = MagicMock()
|
||||
fake_adapter.is_configured.return_value = True
|
||||
with patch.object(plugin._prowlarr, 'is_configured', return_value=False), \
|
||||
patch('core.download_plugins.usenet.get_active_usenet_adapter', return_value=fake_adapter):
|
||||
assert plugin.is_configured() is False
|
||||
with patch.object(plugin._prowlarr, 'is_configured', return_value=True), \
|
||||
patch('core.download_plugins.usenet.get_active_usenet_adapter', return_value=None):
|
||||
assert plugin.is_configured() is False
|
||||
with patch.object(plugin._prowlarr, 'is_configured', return_value=True), \
|
||||
patch('core.download_plugins.usenet.get_active_usenet_adapter', return_value=fake_adapter):
|
||||
assert plugin.is_configured() is True
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Plugin conformance — both must satisfy the DownloadSourcePlugin Protocol
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_plugins_conform_to_protocol() -> None:
|
||||
from core.download_plugins.base import DownloadSourcePlugin
|
||||
assert isinstance(TorrentDownloadPlugin(), DownloadSourcePlugin)
|
||||
assert isinstance(UsenetDownloadPlugin(), DownloadSourcePlugin)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Registry — both should register cleanly
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_torrent_album_pick_prefers_seeded_flac(tmp_path: Path) -> None:
|
||||
"""Album bundle picker prefers high-seeded FLAC over low-seeded MP3
|
||||
of comparable size — protects against picking a dead torrent."""
|
||||
from core.download_plugins.album_bundle import pick_best_album_release
|
||||
from core.download_plugins.torrent import _guess_quality_from_title
|
||||
flac = _make_torrent_result(title='Kendrick Lamar - GNX [FLAC]', size=400_000_000, seeders=120)
|
||||
mp3 = _make_torrent_result(title='Kendrick Lamar - GNX [MP3 320]', size=120_000_000, seeders=5, guid='guid-2')
|
||||
picked = pick_best_album_release([flac, mp3], _guess_quality_from_title)
|
||||
assert picked is flac
|
||||
|
||||
|
||||
def test_torrent_album_pick_drops_too_small() -> None:
|
||||
"""Single-track torrents (~10 MB) shouldn't be picked when the user
|
||||
is downloading a whole album — the size floor (40 MB) catches them."""
|
||||
from core.download_plugins.album_bundle import pick_best_album_release
|
||||
from core.download_plugins.torrent import _guess_quality_from_title
|
||||
single = _make_torrent_result(title='Kendrick Lamar - HUMBLE', size=10_000_000, seeders=500)
|
||||
album = _make_torrent_result(title='Kendrick Lamar - DAMN [MP3]', size=120_000_000, seeders=50, guid='guid-2')
|
||||
picked = pick_best_album_release([single, album], _guess_quality_from_title)
|
||||
assert picked is album
|
||||
|
||||
|
||||
def test_torrent_album_pick_falls_back_when_all_outside_size_range() -> None:
|
||||
"""If every candidate is below the floor (e.g. all results are
|
||||
singles), pick the most-seeded one rather than returning None —
|
||||
user still wants a download even if it's a track torrent."""
|
||||
from core.download_plugins.album_bundle import pick_best_album_release
|
||||
from core.download_plugins.torrent import _guess_quality_from_title
|
||||
small_a = _make_torrent_result(title='X [MP3]', size=8_000_000, seeders=5)
|
||||
small_b = _make_torrent_result(title='Y [MP3]', size=9_000_000, seeders=80, guid='guid-2')
|
||||
picked = pick_best_album_release([small_a, small_b], _guess_quality_from_title)
|
||||
assert picked is small_b
|
||||
|
||||
|
||||
def test_unique_staging_path_handles_collision(tmp_path: Path) -> None:
|
||||
from core.download_plugins.album_bundle import unique_staging_path
|
||||
src = tmp_path / 'src' / 'track.flac'
|
||||
src.parent.mkdir()
|
||||
src.write_bytes(b'fLaC')
|
||||
dest_dir = tmp_path / 'staging'
|
||||
dest_dir.mkdir()
|
||||
# First call returns the natural name.
|
||||
first = unique_staging_path(dest_dir, src)
|
||||
assert first == dest_dir / 'track.flac'
|
||||
first.write_bytes(b'fLaC')
|
||||
# Second call picks a non-colliding suffix.
|
||||
second = unique_staging_path(dest_dir, src)
|
||||
assert second == dest_dir / 'track_1.flac'
|
||||
|
||||
|
||||
def test_torrent_album_to_staging_short_circuits_when_not_configured() -> None:
|
||||
"""The gate must refuse to operate when Prowlarr isn't set up —
|
||||
every later call would hit the network with empty creds."""
|
||||
plugin = TorrentDownloadPlugin()
|
||||
with patch.object(plugin, 'is_configured', return_value=False):
|
||||
outcome = plugin.download_album_to_staging('GNX', 'Kendrick Lamar', '/tmp/staging')
|
||||
assert outcome['success'] is False
|
||||
assert 'not configured' in outcome['error'].lower()
|
||||
|
||||
|
||||
def test_torrent_album_to_staging_ignores_candidates_without_download_url(tmp_path: Path) -> None:
|
||||
plugin = TorrentDownloadPlugin()
|
||||
fake_adapter = MagicMock()
|
||||
fake_adapter.is_configured.return_value = True
|
||||
with patch.object(plugin, 'is_configured', return_value=True), \
|
||||
patch.object(plugin._prowlarr, 'search', new=AsyncMock(return_value=[
|
||||
_make_torrent_result(download_url=None, magnet_uri=None),
|
||||
])), \
|
||||
patch('core.download_plugins.torrent.get_active_torrent_adapter', return_value=fake_adapter):
|
||||
outcome = plugin.download_album_to_staging('GNX', 'Kendrick Lamar', str(tmp_path))
|
||||
|
||||
assert outcome['success'] is False
|
||||
assert 'No torrent results' in outcome['error']
|
||||
fake_adapter.add_torrent.assert_not_called()
|
||||
|
||||
|
||||
def test_registry_includes_torrent_and_usenet() -> None:
|
||||
"""The registry decides what shows up in the orchestrator's
|
||||
iteration helpers. If we forget to register a new plugin the
|
||||
download source dropdown will silently no-op."""
|
||||
from core.download_plugins.registry import build_default_registry
|
||||
registry = build_default_registry()
|
||||
names = registry.names()
|
||||
assert 'torrent' in names
|
||||
assert 'usenet' in names
|
||||
|
|
@ -1,6 +1,8 @@
|
|||
"""Tests for `worker_utils.set_album_api_track_count` — the shared helper
|
||||
enrichment workers call to cache authoritative track counts."""
|
||||
|
||||
import sqlite3
|
||||
|
||||
from core.worker_utils import set_album_api_track_count
|
||||
|
||||
|
||||
|
|
@ -114,3 +116,21 @@ def test_swallows_cursor_execute_errors():
|
|||
cursor = _BrokenCursor()
|
||||
# Should not raise.
|
||||
set_album_api_track_count(cursor, "album-z", 10)
|
||||
|
||||
|
||||
def test_repairs_missing_api_track_count_column():
|
||||
conn = sqlite3.connect(":memory:")
|
||||
cursor = conn.cursor()
|
||||
cursor.execute("CREATE TABLE albums (id TEXT PRIMARY KEY, title TEXT)")
|
||||
cursor.execute("INSERT INTO albums (id, title) VALUES ('album-z', 'Album')")
|
||||
|
||||
set_album_api_track_count(cursor, "album-z", 10)
|
||||
|
||||
cursor.execute("PRAGMA table_info(albums)")
|
||||
cols = {row[1] for row in cursor.fetchall()}
|
||||
cursor.execute("SELECT api_track_count FROM albums WHERE id = 'album-z'")
|
||||
row = cursor.fetchone()
|
||||
conn.close()
|
||||
|
||||
assert 'api_track_count' in cols
|
||||
assert row[0] == 10
|
||||
|
|
|
|||
113
web_server.py
113
web_server.py
|
|
@ -40,7 +40,7 @@ logger = setup_logging(_log_level, _log_path)
|
|||
|
||||
# App version — single source of truth for backup metadata, system-info, update check, etc.
|
||||
# Semver: MAJOR.MINOR.PATCH. Bump at each dev→main release.
|
||||
_SOULSYNC_BASE_VERSION = "2.5.8"
|
||||
_SOULSYNC_BASE_VERSION = "2.5.9"
|
||||
|
||||
def _build_version_string():
|
||||
"""Append short commit hash to version when available (e.g. 2.35+abc1234)."""
|
||||
|
|
@ -2115,10 +2115,15 @@ def get_status():
|
|||
_status_cache_timestamps['media_server'] = current_time
|
||||
# else: use cached value
|
||||
|
||||
# Test Soulseek - only if it's the active source or in the hybrid order
|
||||
if current_time - _status_cache_timestamps['soulseek'] > STATUS_CACHE_TTL:
|
||||
download_mode = config_manager.get('download_source.mode', 'hybrid')
|
||||
hybrid_order = config_manager.get('download_source.hybrid_order', ['hifi', 'youtube', 'soulseek'])
|
||||
download_mode = config_manager.get('download_source.mode', 'hybrid')
|
||||
hybrid_order = config_manager.get('download_source.hybrid_order', ['hifi', 'youtube', 'soulseek'])
|
||||
if isinstance(hybrid_order, str):
|
||||
hybrid_order = [hybrid_order]
|
||||
source_cache_key = f"{download_mode}:{','.join(str(s) for s in (hybrid_order or []))}"
|
||||
|
||||
# Test Soulseek/download source only when the cached source selection is still current.
|
||||
if (current_time - _status_cache_timestamps['soulseek'] > STATUS_CACHE_TTL or
|
||||
_status_cache['soulseek'].get('source_cache_key') != source_cache_key):
|
||||
soulseek_relevant = (download_mode == 'soulseek' or
|
||||
(download_mode == 'hybrid' and 'soulseek' in hybrid_order))
|
||||
|
||||
|
|
@ -2130,11 +2135,24 @@ def get_status():
|
|||
is_serverless = (download_mode in serverless_sources or
|
||||
(download_mode == 'hybrid' and
|
||||
hybrid_order and any(s in serverless_sources for s in hybrid_order)))
|
||||
external_client_sources = ('torrent', 'usenet')
|
||||
external_client_relevant = (
|
||||
download_mode in external_client_sources or
|
||||
(download_mode == 'hybrid' and
|
||||
hybrid_order and any(s in external_client_sources for s in hybrid_order))
|
||||
)
|
||||
|
||||
# Serverless check first — avoids slow slskd timeout when YouTube/HiFi are in hybrid order
|
||||
if is_serverless:
|
||||
soulseek_status = True
|
||||
soulseek_response_time = 0
|
||||
elif external_client_relevant and download_orchestrator:
|
||||
soulseek_start = time.time()
|
||||
try:
|
||||
soulseek_status = run_async(download_orchestrator.check_connection())
|
||||
except Exception:
|
||||
soulseek_status = False
|
||||
soulseek_response_time = (time.time() - soulseek_start) * 1000
|
||||
elif soulseek_relevant and download_orchestrator:
|
||||
soulseek_start = time.time()
|
||||
try:
|
||||
|
|
@ -2148,13 +2166,17 @@ def get_status():
|
|||
|
||||
_status_cache['soulseek'] = {
|
||||
'connected': soulseek_status,
|
||||
'response_time': round(soulseek_response_time, 1)
|
||||
'response_time': round(soulseek_response_time, 1),
|
||||
'source_cache_key': source_cache_key,
|
||||
}
|
||||
_status_cache_timestamps['soulseek'] = current_time
|
||||
|
||||
# Include download source mode so frontend can update labels
|
||||
download_mode = config_manager.get('download_source.mode', 'hybrid')
|
||||
_status_cache['soulseek']['source'] = download_mode
|
||||
soulseek_data = {
|
||||
key: value for key, value in _status_cache['soulseek'].items()
|
||||
if key != 'source_cache_key'
|
||||
}
|
||||
|
||||
# Count active downloads for nav badge
|
||||
active_dl_count = 0
|
||||
|
|
@ -2167,7 +2189,7 @@ def get_status():
|
|||
'metadata_source': metadata_status['metadata_source'],
|
||||
'spotify': metadata_status['spotify'],
|
||||
'media_server': _status_cache['media_server'],
|
||||
'soulseek': _status_cache['soulseek'],
|
||||
'soulseek': soulseek_data,
|
||||
'active_media_server': active_server,
|
||||
'enrichment': _get_enrichment_status(),
|
||||
'active_downloads': active_dl_count,
|
||||
|
|
@ -5887,6 +5909,14 @@ def _find_completed_file_robust(download_dir, api_filename, transfer_dir=None):
|
|||
from difflib import SequenceMatcher
|
||||
from unidecode import unidecode
|
||||
|
||||
audio_extensions = {
|
||||
'.mp3', '.flac', '.m4a', '.aac', '.ogg', '.opus', '.wav', '.wma', '.alac',
|
||||
'.aiff', '.aif', '.dsf', '.dff', '.ape',
|
||||
}
|
||||
|
||||
def _is_audio_candidate(path):
|
||||
return os.path.splitext(str(path or ''))[1].lower() in audio_extensions
|
||||
|
||||
# YOUTUBE/TIDAL SUPPORT: Handle encoded filename format "id||title"
|
||||
# Extract just the title part for file matching
|
||||
if '||' in api_filename:
|
||||
|
|
@ -5921,9 +5951,12 @@ def _find_completed_file_robust(download_dir, api_filename, transfer_dir=None):
|
|||
# Skip quarantine folder — contains known-wrong files from AcoustID verification
|
||||
dirs[:] = [d for d in dirs if d != 'ss_quarantine']
|
||||
for file in files:
|
||||
file_path = os.path.join(root, file)
|
||||
if not _is_audio_candidate(file_path):
|
||||
continue
|
||||
|
||||
# Direct basename match
|
||||
if os.path.basename(file) == target_basename:
|
||||
file_path = os.path.join(root, file)
|
||||
# Fast path: if path aligns with expected directory structure, return now
|
||||
if api_dir_parts and _path_matches_api_dirs(file_path):
|
||||
logger.info(f"Found path-confirmed match in {location_name}: {file_path}")
|
||||
|
|
@ -5940,7 +5973,6 @@ def _find_completed_file_robust(download_dir, api_filename, transfer_dir=None):
|
|||
file_stem, file_ext_part = os.path.splitext(file)
|
||||
stripped_stem = re.sub(r'_\d{10,}$', '', file_stem)
|
||||
if stripped_stem != file_stem and stripped_stem + file_ext_part == target_basename:
|
||||
file_path = os.path.join(root, file)
|
||||
if api_dir_parts and _path_matches_api_dirs(file_path):
|
||||
logger.info(f"Found path-confirmed dedup match in {location_name}: {file_path}")
|
||||
return file_path, 1.0
|
||||
|
|
@ -5956,7 +5988,7 @@ def _find_completed_file_robust(download_dir, api_filename, transfer_dir=None):
|
|||
|
||||
if similarity > highest_fuzzy_similarity:
|
||||
highest_fuzzy_similarity = similarity
|
||||
best_fuzzy_path = os.path.join(root, file)
|
||||
best_fuzzy_path = file_path
|
||||
|
||||
# Return best exact match (disambiguated by path), or fall back to fuzzy
|
||||
if exact_matches:
|
||||
|
|
@ -16463,9 +16495,10 @@ def _get_staging_file_cache(batch_id):
|
|||
"""Scan staging folder once per batch and cache the results."""
|
||||
with _staging_cache_lock:
|
||||
if batch_id in _staging_cache:
|
||||
return _staging_cache[batch_id]
|
||||
cached = _staging_cache[batch_id]
|
||||
return [f for f in cached if os.path.exists(f.get('full_path', ''))]
|
||||
|
||||
staging_path = get_staging_path()
|
||||
staging_path = _get_album_bundle_staging_path(batch_id) or get_staging_path()
|
||||
if not os.path.isdir(staging_path):
|
||||
with _staging_cache_lock:
|
||||
_staging_cache[batch_id] = []
|
||||
|
|
@ -16496,10 +16529,32 @@ def _get_staging_file_cache(batch_id):
|
|||
return files
|
||||
|
||||
|
||||
def _get_album_bundle_staging_path(batch_id):
|
||||
"""Return the private album-bundle staging path for this batch, if any."""
|
||||
if not batch_id:
|
||||
return None
|
||||
with tasks_lock:
|
||||
row = download_batches.get(batch_id)
|
||||
if isinstance(row, dict) and row.get('album_bundle_private_staging'):
|
||||
return row.get('album_bundle_staging_path')
|
||||
return None
|
||||
|
||||
|
||||
# Staging-folder match shortcut lives in core/downloads/staging.py.
|
||||
from core.downloads import staging as _downloads_staging
|
||||
|
||||
|
||||
def _staging_get_batch_field(batch_id, field):
|
||||
"""Accessor injected into StagingDeps so the staging-match helper
|
||||
can read an album-bundle provenance override off the batch state
|
||||
without importing ``download_batches`` directly."""
|
||||
with tasks_lock:
|
||||
row = download_batches.get(batch_id)
|
||||
if isinstance(row, dict):
|
||||
return row.get(field)
|
||||
return None
|
||||
|
||||
|
||||
def _build_staging_deps():
|
||||
"""Build the StagingDeps bundle from web_server.py globals on each call."""
|
||||
return _downloads_staging.StagingDeps(
|
||||
|
|
@ -16508,6 +16563,7 @@ def _build_staging_deps():
|
|||
get_staging_file_cache=_get_staging_file_cache,
|
||||
docker_resolve_path=docker_resolve_path,
|
||||
post_process_matched_download_with_verification=_post_process_matched_download_with_verification,
|
||||
get_batch_field=_staging_get_batch_field,
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -16647,7 +16703,7 @@ def _store_batch_source(batch_id, username, filename):
|
|||
"""Browse the successful download's folder and store results on the batch for reuse."""
|
||||
_sr = source_reuse_logger
|
||||
_sr.info(f"_store_batch_source called: batch={batch_id}, user={username}, file={filename}")
|
||||
if not batch_id or username in ('youtube', 'tidal', 'qobuz', 'hifi', 'deezer_dl', 'lidarr', 'soundcloud', 'amazon'):
|
||||
if not batch_id or username in ('youtube', 'tidal', 'qobuz', 'hifi', 'deezer_dl', 'lidarr', 'soundcloud', 'amazon', 'torrent', 'usenet'):
|
||||
_sr.info(f"Skipped — no batch_id or streaming source ({username})")
|
||||
return
|
||||
|
||||
|
|
@ -19723,39 +19779,12 @@ def soundcloud_status():
|
|||
@app.route('/api/hifi/instances', methods=['GET'])
|
||||
def hifi_instances():
|
||||
"""Check availability of all HiFi API instances."""
|
||||
import requests as req
|
||||
try:
|
||||
hifi = download_orchestrator.client("hifi")
|
||||
instances = list(hifi._instances)
|
||||
results = []
|
||||
for url in instances:
|
||||
entry = {'url': url, 'status': 'unknown', 'version': None, 'can_search': False, 'can_download': False}
|
||||
try:
|
||||
# Check root for version
|
||||
r = req.get(f'{url}/', timeout=5, headers={'Accept': 'application/json'})
|
||||
if r.ok:
|
||||
data = r.json()
|
||||
entry['version'] = data.get('version')
|
||||
entry['status'] = 'online'
|
||||
# Check search
|
||||
sr = req.get(f'{url}/search', params={'s': 'test', 'limit': 1}, timeout=5)
|
||||
entry['can_search'] = sr.ok
|
||||
# Check track (download capability)
|
||||
tr = req.get(f'{url}/track', params={'id': '1550546', 'quality': 'LOSSLESS'}, timeout=5)
|
||||
entry['can_download'] = tr.ok
|
||||
if not tr.ok:
|
||||
entry['download_error'] = f'HTTP {tr.status_code}'
|
||||
else:
|
||||
entry['status'] = f'error (HTTP {r.status_code})'
|
||||
except req.exceptions.SSLError:
|
||||
entry['status'] = 'ssl_error'
|
||||
except req.exceptions.ConnectTimeout:
|
||||
entry['status'] = 'timeout'
|
||||
except req.exceptions.ConnectionError:
|
||||
entry['status'] = 'offline'
|
||||
except Exception as e:
|
||||
entry['status'] = f'error ({type(e).__name__})'
|
||||
results.append(entry)
|
||||
results.append(hifi.check_instance_capabilities(url))
|
||||
return jsonify({'instances': results, 'active': hifi._get_instance()})
|
||||
except Exception as e:
|
||||
return jsonify({'error': str(e)}), 500
|
||||
|
|
|
|||
|
|
@ -4328,6 +4328,8 @@
|
|||
<option value="amazon">Amazon Music Only</option>
|
||||
<option value="lidarr">Lidarr Only</option>
|
||||
<option value="soundcloud">SoundCloud Only</option>
|
||||
<option value="torrent">Torrent Only (via Prowlarr)</option>
|
||||
<option value="usenet">Usenet Only (via Prowlarr)</option>
|
||||
<option value="hybrid">Hybrid (Primary + Fallback)</option>
|
||||
</select>
|
||||
<div class="setting-help-text">
|
||||
|
|
@ -4735,6 +4737,15 @@
|
|||
</div>
|
||||
</div>
|
||||
|
||||
<!-- Torrent / Usenet Settings — settings live on the Indexers tab, just point the user there -->
|
||||
<div id="prowlarr-source-redirect" style="display: none;">
|
||||
<div class="form-group">
|
||||
<div class="setting-help-text" style="padding: 12px 14px; background: rgba(var(--accent-rgb), 0.06); border: 1px solid rgba(var(--accent-rgb), 0.2); border-radius: 10px;">
|
||||
📡 Torrent and Usenet downloads run through your Prowlarr indexers and your configured downloader. Set up Prowlarr + the torrent / usenet client on the <strong>Indexers & Downloaders</strong> tab. Once Test Connection is green on both, this source is ready to use.
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- YouTube Settings (shown when youtube or hybrid mode) -->
|
||||
<div id="youtube-settings-container" style="display: none;">
|
||||
<div class="form-group">
|
||||
|
|
@ -4960,6 +4971,17 @@
|
|||
<div class="ind-hero-sub">
|
||||
Configure your Prowlarr instance to feed search results, and pick a torrent client and/or usenet client to handle the actual downloads. You only need to set up the protocols you actually use.
|
||||
</div>
|
||||
<div class="ind-hero-warning">
|
||||
<div class="ind-hero-warning-title">💡 Filesystem access — easiest setup</div>
|
||||
<div class="ind-hero-warning-body">
|
||||
Torrent and usenet clients write files to <strong>their own download folders</strong>, not Soulseek's. SoulSync needs read access to those folders to import the files. <strong>Simplest fix:</strong> point all your downloaders (Soulseek, qBittorrent, SABnzbd / NZBGet) at the <em>same</em> download folder. One folder, one path to worry about, everything just works.
|
||||
<ul>
|
||||
<li><strong>Bare-metal:</strong> just set every client's download path to the same folder (e.g. your existing slskd folder). Done.</li>
|
||||
<li><strong>Docker:</strong> easiest is to reuse the <code>./downloads</code> mount you already have for slskd — point qBit and SAB at <code>/downloads</code> inside their containers too. Or use the commented placeholders in <code>docker-compose.yml</code> if you want separate folders.</li>
|
||||
<li><strong>Remote client (NAS / different host):</strong> mount the remote folder locally (SMB / NFS / rclone) before SoulSync can see it.</li>
|
||||
</ul>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- ─── INDEXERS section ─── -->
|
||||
|
|
|
|||
|
|
@ -3300,6 +3300,74 @@ function closeCandidatesModal() {
|
|||
}
|
||||
}
|
||||
|
||||
function _downloadModalBundleProgressPercent(bundle) {
|
||||
if (!bundle) return 0;
|
||||
const raw = bundle.progress_percent ?? bundle.progress ?? 0;
|
||||
let progress = Number(raw);
|
||||
if (!Number.isFinite(progress)) progress = 0;
|
||||
if (progress <= 1) progress *= 100;
|
||||
return Math.max(0, Math.min(100, Math.round(progress)));
|
||||
}
|
||||
|
||||
function _downloadModalFormatBytes(bytes) {
|
||||
const value = Number(bytes);
|
||||
if (!Number.isFinite(value) || value <= 0) return '';
|
||||
const units = ['B', 'KB', 'MB', 'GB', 'TB'];
|
||||
let size = value;
|
||||
let unit = 0;
|
||||
while (size >= 1024 && unit < units.length - 1) {
|
||||
size /= 1024;
|
||||
unit += 1;
|
||||
}
|
||||
const decimals = size >= 10 || unit === 0 ? 0 : 1;
|
||||
return `${size.toFixed(decimals)} ${units[unit]}`;
|
||||
}
|
||||
|
||||
function _downloadModalFormatSpeed(bytesPerSecond) {
|
||||
const formatted = _downloadModalFormatBytes(bytesPerSecond);
|
||||
return formatted ? `${formatted}/s` : '';
|
||||
}
|
||||
|
||||
function _downloadModalSourceLabel(source) {
|
||||
const labels = {
|
||||
torrent: 'Torrent',
|
||||
usenet: 'Usenet',
|
||||
soulseek: 'Soulseek',
|
||||
youtube: 'YouTube',
|
||||
tidal: 'Tidal',
|
||||
qobuz: 'Qobuz',
|
||||
hifi: 'HiFi',
|
||||
deezer_dl: 'Deezer',
|
||||
amazon: 'Amazon',
|
||||
lidarr: 'Lidarr',
|
||||
soundcloud: 'SoundCloud'
|
||||
};
|
||||
const key = String(source || '').toLowerCase();
|
||||
return labels[key] || (source ? String(source) : 'Release');
|
||||
}
|
||||
|
||||
function _downloadModalBundleStateLabel(state) {
|
||||
const labels = {
|
||||
searching: 'searching for release',
|
||||
downloading: 'downloading release',
|
||||
staged: 'matching tracks',
|
||||
failed: 'release failed'
|
||||
};
|
||||
const key = String(state || '').toLowerCase();
|
||||
return labels[key] || (state ? String(state).replace(/_/g, ' ') : 'downloading release');
|
||||
}
|
||||
|
||||
function _downloadModalBundleProgressText(bundle) {
|
||||
const percent = _downloadModalBundleProgressPercent(bundle);
|
||||
const source = _downloadModalSourceLabel(bundle && bundle.source);
|
||||
const state = _downloadModalBundleStateLabel(bundle && bundle.state);
|
||||
const release = bundle && bundle.release ? ` - ${bundle.release}` : '';
|
||||
const speed = _downloadModalFormatSpeed(bundle && bundle.speed);
|
||||
const size = _downloadModalFormatBytes(bundle && bundle.size);
|
||||
const detail = speed || size ? ` (${[speed, size].filter(Boolean).join(' of ')})` : '';
|
||||
return `${source} ${state} ${percent}%${release}${detail}`;
|
||||
}
|
||||
|
||||
function processModalStatusUpdate(playlistId, data) {
|
||||
// This function contains ALL the existing polling logic from startModalDownloadPolling
|
||||
// Extracted so it can be called from both individual and batched polling
|
||||
|
|
@ -3348,6 +3416,33 @@ function processModalStatusUpdate(playlistId, data) {
|
|||
// Auto-save M3U file for playlists after analysis
|
||||
autoSavePlaylistM3U(playlistId);
|
||||
}
|
||||
} else if (data.phase === 'album_downloading') {
|
||||
const analysisFill = document.getElementById(`analysis-progress-fill-${playlistId}`);
|
||||
const analysisText = document.getElementById(`analysis-progress-text-${playlistId}`);
|
||||
if (analysisFill) analysisFill.style.width = '100%';
|
||||
if (analysisText) analysisText.textContent = 'Analysis complete!';
|
||||
|
||||
const bundle = data.album_bundle || {};
|
||||
const percent = _downloadModalBundleProgressPercent(bundle);
|
||||
const downloadFill = document.getElementById(`download-progress-fill-${playlistId}`);
|
||||
const downloadText = document.getElementById(`download-progress-text-${playlistId}`);
|
||||
if (downloadFill) downloadFill.style.width = `${percent}%`;
|
||||
if (downloadText) {
|
||||
downloadText.textContent = _downloadModalBundleProgressText(bundle);
|
||||
downloadText.title = 'SoulSync downloads one album release first, then matches the selected tracks from the staged files.';
|
||||
}
|
||||
|
||||
const modal = document.getElementById(`download-missing-modal-${playlistId}`);
|
||||
if (modal) {
|
||||
modal.querySelectorAll('[id^="download-"]').forEach(statusEl => {
|
||||
if (!statusEl.id.startsWith(`download-${playlistId}-`)) return;
|
||||
if (!statusEl.textContent || statusEl.textContent === '-' || statusEl.textContent.includes('Pending')) {
|
||||
statusEl.textContent = 'Waiting for release';
|
||||
statusEl.classList.add('album-bundle-waiting');
|
||||
statusEl.title = 'The album release is downloading first. Tracks will move to processing once SoulSync can match files from it.';
|
||||
}
|
||||
});
|
||||
}
|
||||
} else if (data.phase === 'downloading' || data.phase === 'complete' || data.phase === 'error') {
|
||||
console.debug(`📊 [Status Update] Processing ${data.phase} phase for playlistId: ${playlistId}, tasks: ${(data.tasks || []).length}`);
|
||||
|
||||
|
|
|
|||
|
|
@ -3413,8 +3413,20 @@ function closeHelperSearch() {
|
|||
// projects that span multiple commits before shipping. Strip the flag at
|
||||
// release time and add a real `date:` line at the top of the version block.
|
||||
const WHATS_NEW = {
|
||||
'2.6.0': [
|
||||
{ unreleased: true },
|
||||
'2.5.9': [
|
||||
{ date: 'May 21, 2026 — 2.5.9 release' },
|
||||
{ title: 'Torrent and Usenet release downloads', desc: 'torrent and usenet are now real download sources backed by Prowlarr plus your configured downloader client. album downloads use a release-first staging flow so SoulSync downloads one release, watches live progress, then imports the matching tracks from the staged files. hybrid mode keeps torrent / usenet out of album batches, but still lets them participate for playlist singles and wishlist tracks.' },
|
||||
{ title: 'Fix: HiFi public instance compatibility', desc: 'HiFi instance probing now understands both supported manifest shapes: the newer /trackManifests-style flow and the public hifi-api /track/ legacy flow. instances that can search and return a playable HLS manifest are no longer mislabeled as Search only. browser-openable pages can still be offline from the API side, so HTTP 502 / Offline labels are still shown honestly.' },
|
||||
{ title: 'Fix: Jellyfin full refresh imports tracks on older databases', desc: 'full refresh could import artists and albums but fail every Jellyfin track on upgraded databases that were missing newer media columns. startup repair now adds the missing tracks.file_size and albums.api_track_count columns before refresh work runs, so old databases can accept new Jellyfin track rows again.' },
|
||||
{ title: 'Fix: transient SQLite disk I/O errors retry cleanly', desc: 'cache maintenance and the Tools page full-refresh job now retry short-lived SQLite disk I/O failures around destructive maintenance writes. this targets the reported pattern where the first run failed with disk I/O error and an immediate second run succeeded.' },
|
||||
{ title: 'Fix: Album Completeness blocks wrong-artist fills', desc: 'artist matching now wins before album/single title matching. if Album Completeness is filling a Jamiroquai track, a same-title release by a different artist is rejected instead of being allowed through by a loose title match.' },
|
||||
{ title: 'Fix: Album Completeness no longer cross-contaminates artists', desc: 'reported case: filling a Jamiroquai "Light Years" single brought in tracks from Gut\'s "Light Years" album — completely different artist, completely different genre. Root cause: the auto-fill artist gate was a loose 0.50 SequenceMatcher threshold that could let unrelated candidates through. Two new defenses now block this entirely. <strong>Stage 1</strong> skips the whole track-fill operation up front if the missing track\'s source artist doesn\'t match the target album artist — compilation albums (various artists / soundtrack) still pass through. <strong>Stage 2</strong> replaces the per-candidate 0.50 SequenceMatcher with an alias-aware 0.82 strict matcher that handles diacritics (Beyoncé/Beyonce) and known artist aliases. Both stages logged with structured warnings so future mismatches are diagnosable from the logs.' },
|
||||
{ title: 'Code review refactor pass on the torrent / usenet flow', desc: 'cleanup commits before review. Lifted the shared album-pick + staging-collision helpers out of torrent.py into a new core/download_plugins/album_bundle.py module so usenet.py no longer reaches into a sibling plugin\'s private surface. Lifted the ~90-line inline album-bundle gate out of run_full_missing_tracks_process into core/downloads/album_bundle_dispatch.py with a clean pure-predicate / inject-deps design so the gate is unit-testable in isolation. Replaced the staging matcher\'s direct download_batches import with an injected get_batch_field accessor on StagingDeps. Made the 6h poll timeout configurable via download_source.album_bundle_timeout_seconds. Added atomic .tmp + rename copy so the Auto-Import worker can never observe a partial audio file during the album-bundle copy loop. 49 new tests across album_bundle, album_bundle_dispatch, and staging-provenance modules pin the contracts.' },
|
||||
{ title: 'Hybrid mode skips torrent / usenet on album batches', desc: 'follow-up to the album-bundle flow. When download mode is set to Hybrid AND the batch is an album, the per-track search loop now silently strips torrent and usenet from the source chain — they\'re release-level sources that can\'t match per-track meaningfully, and the album-bundle handling only fires in single-source mode. Without the skip, hybrid + torrent-first would fire N redundant Prowlarr searches per album and rely on Auto-Import sweeping Staging to recover. Cleaner now: hybrid falls straight through to per-track-compatible sources (Soulseek / streaming) for albums, and torrent / usenet still get a shot for single-track / wishlist / basic-search use cases where per-track makes sense.' },
|
||||
{ title: 'Album-bundle flow for torrent / usenet downloads', desc: 'fixes the core architectural problem with indexer-based sources. Prowlarr returns release-level torrents — searching per-track for "Luther (with SZA)" against the GNX album torrent scores near-zero and the orchestrator rejects every candidate. New gated flow: when downloading an album AND torrent or usenet is the single active source (not hybrid), SoulSync now does ONE Prowlarr search for the whole release, picks the best torrent (prefers FLAC, high seeders, reasonable size — drops single-track torrents that snuck in), hands it to your torrent / usenet client, walks the resulting audio files (extracting .zip/.rar/.tar if needed), and drops them all into the staging folder. The existing per-track staging matcher then imports each one to the library by fuzzy title match — same path as the Auto-Import flow. Gate is strictly opt-in: per-track flow is completely untouched for hybrid mode, non-album downloads, and every other source. 5 new tests cover the album picker (seeded-FLAC preference, size floor for single-track torrents, fallback when all candidates are small) and the staging path collision handler.' },
|
||||
{ title: 'Filesystem-access heads-up for torrent / usenet sources', desc: 'new advisory card on the Indexers & Downloaders tab explaining the cleanest setup: point ALL your downloaders (Soulseek, qBittorrent, SABnzbd / NZBGet) at the same download folder. One folder, one mount, everything just works. Bare-metal needs no change; Docker users can reuse the existing ./downloads mount and just configure each client to write there. docker-compose.yml updated to call this out as the easiest path, with optional commented placeholders for users who prefer separate folders per protocol.' },
|
||||
{ title: 'Torrent and Usenet downloads', desc: 'two new download sources live in the Download Source dropdown: <strong>Torrent Only (via Prowlarr)</strong> and <strong>Usenet Only (via Prowlarr)</strong>. they reuse the Prowlarr + torrent client + usenet client you set up on the Indexers & Downloaders tab. searches go through Prowlarr filtered by protocol, picked releases get handed to your torrent client or usenet client, and the resulting files get walked through archive_pipeline (extracts .zip / .rar / .tar when the client didn\'t already do it) and handed to the matching pipeline. both sources are also available in hybrid mode alongside soulseek / youtube / tidal / etc. one caveat: SoulSync needs read access to the torrent / usenet client\'s save_path — works out of the box for everything-on-one-box setups, but remote downloader hosts will need a future sync step.' },
|
||||
{ title: 'Archive pipeline module (groundwork for torrent / usenet downloads)', desc: 'new core/archive_pipeline.py — walks a directory for audio files (recursive, case-insensitive extensions), extracts zip / tar / tar.gz / rar / 7z archives in-place (rar and 7z are optional deps that warn but don\'t crash if absent), and rejects any archive member trying to escape the destination via path traversal. shared helper the upcoming torrent and usenet download plugins both consume — usenet downloaders usually auto-extract, but the occasional torrent ships an album in a .rar and SoulSync handles it now. 21 unit tests cover the walker + zip + tar extraction + path-traversal protection.' },
|
||||
{ title: 'Indexers & Downloaders tab restyled with collapsible sections', desc: 'tab now opens with an intro hero card explaining the flow (indexers find releases → downloader fetches them → SoulSync imports) and folds the rest into three collapsible sections: Indexers, Torrent Client, Usenet Client. each section gets a per-service color accent (Prowlarr orange, torrent sky-blue, usenet violet), a status dot in the header (green when Test Connection succeeds, red on failure, grey before testing), and Lidarr-style indexer cards with protocol badges instead of the inline emoji list. Indexers section is open by default; the downloader sections start collapsed since not everyone uses both protocols.' },
|
||||
{ title: 'Regression tests for the new indexer + downloader plumbing', desc: 'mocked unit tests for Prowlarr + all five downloader adapters (qBittorrent, Transmission, Deluge, SABnzbd, NZBGet). 54 tests pin the state-mapping tables, the parse logic, and the protocol quirks each client needs handled (qBit Referer header, Transmission session-id renegotiation, Deluge magnet-vs-URL method split, SAB queue+history merge, NZBGet 64-bit size fields). next time anyone touches one of these adapters, CI catches breakage before it hits a real downloader.' },
|
||||
{ title: 'Usenet client adapters (SABnzbd, NZBGet)', desc: 'third commit in the torrent + usenet rollout. SoulSync now talks to SABnzbd and NZBGet through a sibling adapter contract that mirrors the torrent adapter set — pick one downloader in Settings → Indexers & Downloaders, fill in its API key (SABnzbd) or username + password (NZBGet), and Test Connection confirms the link. all three layers are now stood up: Prowlarr finds releases, the torrent adapter and the usenet adapter each know how to ship work to the underlying client. next commit wires Prowlarr search → adapter dispatch → archive extraction so the new sources actually download. job state mapping covers SABnzbd queue + history and NZBGet groups + history, including the verify/repair/unpack phases that are unique to usenet.' },
|
||||
|
|
@ -3479,6 +3491,31 @@ const WHATS_NEW = {
|
|||
// Section shape: { title, description, features: [bullet strings],
|
||||
// usage_note?: 'optional hint shown at the bottom' }
|
||||
const VERSION_MODAL_SECTIONS = [
|
||||
{
|
||||
title: "2.5.9 Release Stability Pass",
|
||||
description: "this release ties together the new release-based download sources and a set of fixes from real user reports: HiFi instance detection, Jellyfin full refreshes, transient SQLite disk I/O failures, and wrong-artist Album Completeness fills.",
|
||||
features: [
|
||||
"Torrent and Usenet are now available as Prowlarr-backed download sources with release staging, live progress, and source-aware history labels",
|
||||
"HiFi instances are probed by actually checking whether they can return a playable manifest, including legacy public hifi-api instances",
|
||||
"Jellyfin full refresh repairs older media databases before importing tracks, so stale schemas no longer drop every track row",
|
||||
"Cache maintenance and full refresh retry transient SQLite disk I/O errors instead of failing the whole job on the first attempt",
|
||||
"Album Completeness rejects same-title releases from the wrong artist before importing anything",
|
||||
],
|
||||
usage_note: "version 2.5.9 focuses on safer matching and making the new release-based sources usable without disturbing existing source flows",
|
||||
},
|
||||
{
|
||||
title: "Torrent and Usenet Are Now Live Download Sources",
|
||||
description: "the long-awaited payoff. Two new entries in the Download Source dropdown — Torrent Only and Usenet Only — both backed by your Prowlarr indexers. Searches go through Prowlarr filtered by protocol, picked releases get shipped to your configured torrent client (qBit / Transmission / Deluge) or usenet client (SABnzbd / NZBGet), and the resulting files flow through SoulSync's matching pipeline like any other source.",
|
||||
features: [
|
||||
"• new 'Torrent Only (via Prowlarr)' and 'Usenet Only (via Prowlarr)' options in Settings → Downloads → Download Source",
|
||||
"• both sources also available in hybrid mode — drop them into the priority list alongside soulseek / tidal / etc.",
|
||||
"• shared archive_pipeline walks the downloaded directory and extracts any .zip / .rar / .tar archives the downloader didn't already unpack (with path-traversal protection)",
|
||||
"• torrent state polling handles the qBit / Transmission / Deluge state quirks uniformly; usenet polling covers the verify / repair / unpack phases",
|
||||
"• picking a torrent or usenet source on the Downloads tab now shows a redirect card pointing to the Indexers & Downloaders tab where the actual config lives",
|
||||
"• caveat: SoulSync needs filesystem access to the torrent / usenet client's save_path. fine for everything-on-one-box; remote downloader hosts need a future sync step",
|
||||
],
|
||||
usage_note: "Settings → Downloads → Download Source → Torrent / Usenet Only",
|
||||
},
|
||||
{
|
||||
title: "Usenet Client Adapters (SABnzbd, NZBGet)",
|
||||
description: "third phase of the torrent + usenet rollout. SoulSync now also talks to the two big usenet downloaders through a sibling adapter contract. Prowlarr + torrent + usenet are all stood up — next commit wires them together into actual download sources.",
|
||||
|
|
@ -3850,7 +3887,7 @@ function _getLatestWhatsNewVersion() {
|
|||
const versions = Object.keys(WHATS_NEW)
|
||||
.filter(v => _compareVersions(v, buildVer) <= 0)
|
||||
.sort((a, b) => _compareVersions(b, a));
|
||||
return versions[0] || '2.5.8';
|
||||
return versions[0] || '2.5.9';
|
||||
}
|
||||
|
||||
function openWhatsNew() {
|
||||
|
|
|
|||
|
|
@ -4595,8 +4595,8 @@ async function showTrackSourceInfo(track, anchorEl) {
|
|||
return;
|
||||
}
|
||||
|
||||
const serviceIcons = { soulseek: '🔍', youtube: '▶️', tidal: '🌊', qobuz: '🎵', hifi: '🎧', deezer: '💜' };
|
||||
const serviceLabels = { soulseek: 'Soulseek', youtube: 'YouTube', tidal: 'Tidal', qobuz: 'Qobuz', hifi: 'HiFi', deezer: 'Deezer' };
|
||||
const serviceIcons = { soulseek: '🔍', youtube: '▶️', tidal: '🌊', qobuz: '🎵', hifi: '🎧', deezer: '💜', lidarr: '📦', amazon: '🛒', soundcloud: '☁️', auto_import: '📥', staging: '📥', torrent: '🧲', usenet: '📰' };
|
||||
const serviceLabels = { soulseek: 'Soulseek', youtube: 'YouTube', tidal: 'Tidal', qobuz: 'Qobuz', hifi: 'HiFi', deezer: 'Deezer', lidarr: 'Lidarr', amazon: 'Amazon Music', soundcloud: 'SoundCloud', auto_import: 'Auto-Import', staging: 'Staging', torrent: 'Torrent', usenet: 'Usenet' };
|
||||
|
||||
const dl = data.downloads[0]; // Most recent download
|
||||
const icon = serviceIcons[dl.source_service] || '📦';
|
||||
|
|
@ -4926,8 +4926,8 @@ async function _streamRedownloadSources(overlay, track, metadata) {
|
|||
const startBtn = document.getElementById('redownload-start-btn');
|
||||
if (!columnsEl) return;
|
||||
|
||||
const serviceIcons = { soulseek: '🔍', youtube: '▶️', tidal: '🌊', qobuz: '🎵', hifi: '🎧', deezer_dl: '💜', hybrid: '⚡' };
|
||||
const serviceLabels = { soulseek: 'Soulseek', youtube: 'YouTube', tidal: 'Tidal', qobuz: 'Qobuz', hifi: 'HiFi', deezer_dl: 'Deezer', hybrid: 'Auto' };
|
||||
const serviceIcons = { soulseek: '🔍', youtube: '▶️', tidal: '🌊', qobuz: '🎵', hifi: '🎧', deezer_dl: '💜', hybrid: '⚡', lidarr: '📦', amazon: '🛒', soundcloud: '☁️', torrent: '🧲', usenet: '📰' };
|
||||
const serviceLabels = { soulseek: 'Soulseek', youtube: 'YouTube', tidal: 'Tidal', qobuz: 'Qobuz', hifi: 'HiFi', deezer_dl: 'Deezer', hybrid: 'Auto', lidarr: 'Lidarr', amazon: 'Amazon Music', soundcloud: 'SoundCloud', torrent: 'Torrent', usenet: 'Usenet' };
|
||||
|
||||
let allCandidates = [];
|
||||
let firstResult = true;
|
||||
|
|
@ -5049,8 +5049,8 @@ async function _streamRedownloadSources(overlay, track, metadata) {
|
|||
|
||||
/* _renderRedownloadStep2 removed — replaced by _streamRedownloadSources above */
|
||||
if (false) {
|
||||
const serviceIcons = { soulseek: '🔍', youtube: '▶️', tidal: '🌊', qobuz: '🎵', hifi: '🎧', deezer_dl: '💜', hybrid: '⚡' };
|
||||
const serviceLabels = { soulseek: 'Soulseek', youtube: 'YouTube', tidal: 'Tidal', qobuz: 'Qobuz', hifi: 'HiFi', deezer_dl: 'Deezer', hybrid: 'Auto' };
|
||||
const serviceIcons = { soulseek: '🔍', youtube: '▶️', tidal: '🌊', qobuz: '🎵', hifi: '🎧', deezer_dl: '💜', hybrid: '⚡', lidarr: '📦', amazon: '🛒', soundcloud: '☁️', torrent: '🧲', usenet: '📰' };
|
||||
const serviceLabels = { soulseek: 'Soulseek', youtube: 'YouTube', tidal: 'Tidal', qobuz: 'Qobuz', hifi: 'HiFi', deezer_dl: 'Deezer', hybrid: 'Auto', lidarr: 'Lidarr', amazon: 'Amazon Music', soundcloud: 'SoundCloud', torrent: 'Torrent', usenet: 'Usenet' };
|
||||
|
||||
// Group candidates by source service
|
||||
const grouped = {};
|
||||
|
|
|
|||
|
|
@ -2445,6 +2445,74 @@ function _adlEsc(str) {
|
|||
return String(str).replace(/&/g, '&').replace(/</g, '<').replace(/>/g, '>').replace(/"/g, '"');
|
||||
}
|
||||
|
||||
function _adlBundleProgressPercent(bundle) {
|
||||
if (!bundle) return 0;
|
||||
const raw = bundle.progress_percent ?? bundle.progress ?? 0;
|
||||
let progress = Number(raw);
|
||||
if (!Number.isFinite(progress)) progress = 0;
|
||||
if (progress <= 1) progress *= 100;
|
||||
return Math.max(0, Math.min(100, Math.round(progress)));
|
||||
}
|
||||
|
||||
function _adlFormatBytes(bytes) {
|
||||
const value = Number(bytes);
|
||||
if (!Number.isFinite(value) || value <= 0) return '';
|
||||
const units = ['B', 'KB', 'MB', 'GB', 'TB'];
|
||||
let size = value;
|
||||
let unit = 0;
|
||||
while (size >= 1024 && unit < units.length - 1) {
|
||||
size /= 1024;
|
||||
unit += 1;
|
||||
}
|
||||
const decimals = size >= 10 || unit === 0 ? 0 : 1;
|
||||
return `${size.toFixed(decimals)} ${units[unit]}`;
|
||||
}
|
||||
|
||||
function _adlFormatSpeed(bytesPerSecond) {
|
||||
const formatted = _adlFormatBytes(bytesPerSecond);
|
||||
return formatted ? `${formatted}/s` : '';
|
||||
}
|
||||
|
||||
function _adlSourceLabel(source) {
|
||||
const labels = {
|
||||
torrent: 'Torrent',
|
||||
usenet: 'Usenet',
|
||||
soulseek: 'Soulseek',
|
||||
youtube: 'YouTube',
|
||||
tidal: 'Tidal',
|
||||
qobuz: 'Qobuz',
|
||||
hifi: 'HiFi',
|
||||
deezer_dl: 'Deezer',
|
||||
amazon: 'Amazon',
|
||||
lidarr: 'Lidarr',
|
||||
soundcloud: 'SoundCloud'
|
||||
};
|
||||
const key = String(source || '').toLowerCase();
|
||||
return labels[key] || (source ? String(source) : 'Release');
|
||||
}
|
||||
|
||||
function _adlBundleStateLabel(state) {
|
||||
const labels = {
|
||||
searching: 'searching for release',
|
||||
downloading: 'downloading release',
|
||||
staged: 'matching tracks',
|
||||
failed: 'release failed'
|
||||
};
|
||||
const key = String(state || '').toLowerCase();
|
||||
return labels[key] || (state ? String(state).replace(/_/g, ' ') : 'downloading release');
|
||||
}
|
||||
|
||||
function _adlBundleProgressText(bundle) {
|
||||
const pct = _adlBundleProgressPercent(bundle);
|
||||
const source = _adlSourceLabel(bundle && bundle.source);
|
||||
const state = _adlBundleStateLabel(bundle && bundle.state);
|
||||
const release = bundle && bundle.release ? ` - ${bundle.release}` : '';
|
||||
const speed = _adlFormatSpeed(bundle && bundle.speed);
|
||||
const size = _adlFormatBytes(bundle && bundle.size);
|
||||
const detail = speed || size ? ` (${[speed, size].filter(Boolean).join(' of ')})` : '';
|
||||
return `${source} ${state} ${pct}%${release}${detail}`;
|
||||
}
|
||||
|
||||
async function adlClearCompleted() {
|
||||
try {
|
||||
const resp = await fetch('/api/downloads/clear-completed', { method: 'POST' });
|
||||
|
|
@ -2504,12 +2572,16 @@ function _adlRenderBatchPanel() {
|
|||
const colorStyle = colorIdx >= 0 ? `border-left-color: rgba(var(--batch-color-${colorIdx}), 0.6)` : '';
|
||||
const isExpanded = _adlExpandedBatches.has(batch.batch_id);
|
||||
const isFiltered = _adlFilterBatchId === batch.batch_id;
|
||||
const albumBundle = batch.album_bundle || null;
|
||||
const bundleProgress = _adlBundleProgressPercent(albumBundle);
|
||||
const total = batch.total || 1;
|
||||
const done = batch.completed + batch.failed;
|
||||
const pct = Math.round((done / total) * 100);
|
||||
const pct = batch.phase === 'album_downloading'
|
||||
? bundleProgress
|
||||
: Math.round((done / total) * 100);
|
||||
const hasFailed = batch.failed > 0;
|
||||
const isTerminal = batch.phase === 'complete' || batch.phase === 'cancelled' || batch.phase === 'error';
|
||||
const isActive = batch.phase === 'downloading' && batch.active > 0;
|
||||
const isActive = (batch.phase === 'downloading' && batch.active > 0) || batch.phase === 'album_downloading';
|
||||
|
||||
// Fade progress for completing batches
|
||||
let fadeStyle = '';
|
||||
|
|
@ -2532,6 +2604,9 @@ function _adlRenderBatchPanel() {
|
|||
if (batch.phase === 'analysis') {
|
||||
phaseText = 'Analyzing...';
|
||||
phaseIcon = '<span class="adl-spinner" style="margin-right:4px"></span>';
|
||||
} else if (batch.phase === 'album_downloading') {
|
||||
phaseText = _adlBundleProgressText(albumBundle);
|
||||
phaseIcon = '<span class="adl-spinner" style="margin-right:4px"></span>';
|
||||
} else if (batch.phase === 'downloading') {
|
||||
phaseText = `${batch.completed}/${total} tracks`;
|
||||
if (batch.active > 0) phaseIcon = '<span class="adl-spinner" style="margin-right:4px"></span>';
|
||||
|
|
@ -2595,7 +2670,9 @@ function _adlRenderBatchPanel() {
|
|||
</div>`;
|
||||
}).join('');
|
||||
} else {
|
||||
tracksHtml = '<div style="font-size:0.7rem;color:rgba(255,255,255,0.3);padding:4px 0">No tracks loaded</div>';
|
||||
tracksHtml = batch.phase === 'album_downloading'
|
||||
? '<div class="adl-batch-release-note">Downloading one release first. Track matching starts after staging.</div>'
|
||||
: '<div style="font-size:0.7rem;color:rgba(255,255,255,0.3);padding:4px 0">No tracks loaded</div>';
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -597,10 +597,12 @@ const HYBRID_SOURCES = [
|
|||
{ id: 'amazon', name: 'Amazon Music', icon: null, emoji: '🛒' },
|
||||
{ id: 'lidarr', name: 'Lidarr', icon: null, emoji: '📦' },
|
||||
{ id: 'soundcloud', name: 'SoundCloud', icon: 'https://www.svgrepo.com/show/452219/soundcloud.svg', emoji: '☁️' },
|
||||
{ id: 'torrent', name: 'Torrent', icon: null, emoji: '🧲' },
|
||||
{ id: 'usenet', name: 'Usenet', icon: null, emoji: '📰' },
|
||||
];
|
||||
|
||||
let _hybridSourceOrder = ['soulseek', 'youtube'];
|
||||
let _hybridSourceEnabled = { soulseek: true, youtube: true, tidal: false, qobuz: false, hifi: false, deezer_dl: false, amazon: false, lidarr: false, soundcloud: false };
|
||||
let _hybridSourceEnabled = { soulseek: true, youtube: true, tidal: false, qobuz: false, hifi: false, deezer_dl: false, amazon: false, lidarr: false, soundcloud: false, torrent: false, usenet: false };
|
||||
let _hybridVisualOrder = null; // Full visual order including disabled sources
|
||||
|
||||
function buildHybridSourceList() {
|
||||
|
|
@ -1536,6 +1538,11 @@ function updateDownloadSourceUI() {
|
|||
if (amazonContainer) amazonContainer.style.display = activeSources.has('amazon') ? 'block' : 'none';
|
||||
if (lidarrContainer) lidarrContainer.style.display = activeSources.has('lidarr') ? 'block' : 'none';
|
||||
if (soundcloudContainer) soundcloudContainer.style.display = activeSources.has('soundcloud') ? 'block' : 'none';
|
||||
const prowlarrRedirect = document.getElementById('prowlarr-source-redirect');
|
||||
if (prowlarrRedirect) {
|
||||
const showProwlarr = activeSources.has('torrent') || activeSources.has('usenet');
|
||||
prowlarrRedirect.style.display = showProwlarr ? 'block' : 'none';
|
||||
}
|
||||
|
||||
// Quality profile is Soulseek-only and downloads-tab-only
|
||||
const qualityProfileSection = document.getElementById('quality-profile-section');
|
||||
|
|
|
|||
|
|
@ -3377,8 +3377,8 @@ function updateServiceStatus(service, statusData, spotifyStatus = null) {
|
|||
|
||||
// Update download source title on dashboard card
|
||||
if (service === 'soulseek' && statusData.source) {
|
||||
const sourceNames = { soulseek: 'Soulseek', youtube: 'YouTube', tidal: 'Tidal', qobuz: 'Qobuz', hifi: 'HiFi', deezer_dl: 'Deezer', lidarr: 'Lidarr', soundcloud: 'SoundCloud', hybrid: 'Hybrid' };
|
||||
const displayName = sourceNames[statusData.source] || 'Soulseek';
|
||||
const sourceNames = { soulseek: 'Soulseek', youtube: 'YouTube', tidal: 'Tidal', qobuz: 'Qobuz', hifi: 'HiFi', deezer_dl: 'Deezer', amazon: 'Amazon', lidarr: 'Lidarr', soundcloud: 'SoundCloud', torrent: 'Torrent', usenet: 'Usenet', hybrid: 'Hybrid' };
|
||||
const displayName = sourceNames[statusData.source] || 'Download Source';
|
||||
const titleEl = document.getElementById('download-source-title');
|
||||
if (titleEl) titleEl.textContent = displayName;
|
||||
}
|
||||
|
|
@ -3422,8 +3422,8 @@ function updateSidebarServiceStatus(service, statusData, spotifyStatus = null) {
|
|||
|
||||
// Update download source name based on configured mode
|
||||
if (service === 'soulseek' && statusData.source) {
|
||||
const sourceNames = { soulseek: 'Soulseek', youtube: 'YouTube', tidal: 'Tidal', qobuz: 'Qobuz', hifi: 'HiFi', deezer_dl: 'Deezer', lidarr: 'Lidarr', soundcloud: 'SoundCloud', hybrid: 'Hybrid' };
|
||||
const displayName = sourceNames[statusData.source] || 'Soulseek';
|
||||
const sourceNames = { soulseek: 'Soulseek', youtube: 'YouTube', tidal: 'Tidal', qobuz: 'Qobuz', hifi: 'HiFi', deezer_dl: 'Deezer', amazon: 'Amazon', lidarr: 'Lidarr', soundcloud: 'SoundCloud', torrent: 'Torrent', usenet: 'Usenet', hybrid: 'Hybrid' };
|
||||
const displayName = sourceNames[statusData.source] || 'Download Source';
|
||||
const sidebarName = document.getElementById('download-source-name');
|
||||
if (sidebarName) sidebarName.textContent = displayName;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -3316,6 +3316,45 @@ body.helper-mode-active #dashboard-activity-feed:hover {
|
|||
line-height: 1.55;
|
||||
}
|
||||
|
||||
.ind-hero-warning {
|
||||
margin-top: 14px;
|
||||
padding: 12px 14px;
|
||||
background: rgba(255, 165, 0, 0.06);
|
||||
border: 1px solid rgba(255, 165, 0, 0.2);
|
||||
border-radius: 10px;
|
||||
}
|
||||
|
||||
.ind-hero-warning-title {
|
||||
font-size: 12px;
|
||||
font-weight: 700;
|
||||
color: #ffb155;
|
||||
letter-spacing: 0.4px;
|
||||
margin-bottom: 6px;
|
||||
}
|
||||
|
||||
.ind-hero-warning-body {
|
||||
font-size: 12.5px;
|
||||
color: rgba(255, 255, 255, 0.72);
|
||||
line-height: 1.55;
|
||||
}
|
||||
|
||||
.ind-hero-warning-body ul {
|
||||
margin: 6px 0 0 18px;
|
||||
padding: 0;
|
||||
}
|
||||
|
||||
.ind-hero-warning-body li {
|
||||
margin: 3px 0;
|
||||
}
|
||||
|
||||
.ind-hero-warning-body code {
|
||||
font-family: 'SF Mono', Monaco, monospace;
|
||||
font-size: 11.5px;
|
||||
background: rgba(0, 0, 0, 0.3);
|
||||
padding: 1px 5px;
|
||||
border-radius: 3px;
|
||||
}
|
||||
|
||||
/* Section-header status dots — Lidarr-style green/red/grey indicator */
|
||||
.ind-status-dot {
|
||||
width: 9px;
|
||||
|
|
@ -10294,6 +10333,25 @@ body.helper-mode-active #dashboard-activity-feed:hover {
|
|||
border: 1px solid rgba(100, 200, 130, 0.2);
|
||||
}
|
||||
|
||||
.library-history-badge.download.source-torrent {
|
||||
color: #5dade2;
|
||||
background: rgba(93, 173, 226, 0.1);
|
||||
border-color: rgba(93, 173, 226, 0.25);
|
||||
}
|
||||
|
||||
.library-history-badge.download.source-usenet {
|
||||
color: #a78bfa;
|
||||
background: rgba(167, 139, 250, 0.1);
|
||||
border-color: rgba(167, 139, 250, 0.25);
|
||||
}
|
||||
|
||||
.library-history-badge.download.source-staging,
|
||||
.library-history-badge.download.source-auto-import {
|
||||
color: rgba(255, 255, 255, 0.55);
|
||||
background: rgba(255, 255, 255, 0.05);
|
||||
border-color: rgba(255, 255, 255, 0.12);
|
||||
}
|
||||
|
||||
.library-history-badge.import {
|
||||
color: rgba(120, 160, 230, 0.9);
|
||||
background: rgba(120, 160, 230, 0.1);
|
||||
|
|
@ -19846,6 +19904,11 @@ body.helper-mode-active #dashboard-activity-feed:hover {
|
|||
inset 0 1px 2px rgba(255, 255, 255, 0.3);
|
||||
}
|
||||
|
||||
.album-bundle-waiting {
|
||||
color: rgba(255, 255, 255, 0.55);
|
||||
font-style: italic;
|
||||
}
|
||||
|
||||
/* Track Table Section */
|
||||
.download-tracks-section {
|
||||
/* Premium glassmorphic card matching other sections */
|
||||
|
|
@ -58173,6 +58236,9 @@ body.reduce-effects *::after {
|
|||
font-size: 0.7rem;
|
||||
color: rgba(255, 255, 255, 0.4);
|
||||
margin-top: 2px;
|
||||
white-space: nowrap;
|
||||
overflow: hidden;
|
||||
text-overflow: ellipsis;
|
||||
}
|
||||
|
||||
.adl-batch-card-source {
|
||||
|
|
@ -58315,6 +58381,13 @@ body.reduce-effects *::after {
|
|||
.adl-batch-track-status.failed { color: #ef4444; }
|
||||
.adl-batch-track-status.queued { color: rgba(255, 255, 255, 0.3); }
|
||||
|
||||
.adl-batch-release-note {
|
||||
padding: 6px 0;
|
||||
color: rgba(255, 255, 255, 0.38);
|
||||
font-size: 0.7rem;
|
||||
line-height: 1.4;
|
||||
}
|
||||
|
||||
/* Mini progress bar per track */
|
||||
.adl-batch-track-progress {
|
||||
width: 100%;
|
||||
|
|
|
|||
|
|
@ -3406,7 +3406,7 @@ async function loadLibraryHistory() {
|
|||
const sc = data.stats?.source_counts || {};
|
||||
const srcEntries = Object.entries(sc).sort((a, b) => b[1] - a[1]);
|
||||
if (srcEntries.length > 0 && tab === 'download') {
|
||||
const _srcColors = { Soulseek: '#4caf50', Tidal: '#000', YouTube: '#ff0000', Qobuz: '#4285f4', HiFi: '#00bcd4', Deezer: '#a238ff' };
|
||||
const _srcColors = { Soulseek: '#4caf50', Tidal: '#000', YouTube: '#ff0000', Qobuz: '#4285f4', HiFi: '#00bcd4', Deezer: '#a238ff', Lidarr: '#5dade2', Amazon: '#ff9900', SoundCloud: '#ff7700', Torrent: '#5dade2', Usenet: '#a78bfa', Staging: '#888', 'Auto-Import': '#888' };
|
||||
sourceBar.innerHTML = srcEntries.map(([src, cnt]) =>
|
||||
`<span class="history-source-chip" style="border-color:${_srcColors[src] || '#888'};color:${_srcColors[src] || '#888'}">${src}: ${cnt}</span>`
|
||||
).join('');
|
||||
|
|
@ -3449,7 +3449,10 @@ function renderHistoryEntry(entry) {
|
|||
const parts = [];
|
||||
if (entry.download_source) parts.push(entry.download_source);
|
||||
if (entry.quality) parts.push(entry.quality);
|
||||
badge = parts.map(p => `<span class="library-history-badge download">${escapeHtml(p)}</span>`).join('');
|
||||
badge = parts.map(p => {
|
||||
const cls = String(p || '').toLowerCase().replace(/[^a-z0-9]+/g, '-').replace(/^-|-$/g, '');
|
||||
return `<span class="library-history-badge download source-${escapeHtml(cls)}">${escapeHtml(p)}</span>`;
|
||||
}).join('');
|
||||
} else if (entry.event_type === 'import' && entry.server_source) {
|
||||
const sourceName = { plex: 'Plex', jellyfin: 'Jellyfin', navidrome: 'Navidrome' }[entry.server_source] || entry.server_source;
|
||||
badge = `<span class="library-history-badge import">${escapeHtml(sourceName)}</span>`;
|
||||
|
|
|
|||
Loading…
Reference in a new issue