"""Shared helpers for the album-bundle download flow. The torrent and usenet download plugins both implement a ``download_album_to_staging`` method that searches Prowlarr for a whole release, hands it to the active downloader, walks the resulting audio files, and copies them into the staging folder. The two implementations share the same release-picker heuristic and the same staging-path collision logic. Pulled out of ``core/download_plugins/torrent.py`` so the usenet plugin doesn't have to import private helpers from a sibling plugin (Cin's "no leaky module boundaries" standard). Also exposes ``atomic_copy_to_staging`` — the audio file is copied to a ``.tmp.`` sidecar first and atomically renamed onto its final extension. The Auto-Import worker filters by audio extension so the in-flight ``.tmp`` file is never picked up mid-copy, closing the race between the album-bundle copy loop and Auto-Import's folder scan. """ from __future__ import annotations import shutil import time import uuid from pathlib import Path from typing import Any, Callable, Iterable, Optional from config.settings import config_manager from utils.logging_config import get_logger logger = get_logger("download_plugins.album_bundle") # Album-pick size floor / ceiling. Single-track torrents (~10 MB) # are rejected when bigger candidates exist; anything past 3 GB is # treated as suspicious (multi-disc box-set + scans + extras). ALBUM_PICK_MIN_BYTES = 40 * 1024 * 1024 ALBUM_PICK_MAX_BYTES = 3 * 1024 * 1024 * 1024 # Quality-score weights for the album-pick heuristic. Mirrors the # tier order in ``core/imports/file_ops.py``'s ``quality_tiers`` — # higher number = preferred. _QUALITY_SCORE = {'flac': 4, 'ogg': 3, 'aac': 2, 'mp3': 1} # Default poll cadence + timeout for the album-download poll loop. # Both are overridable through config so users with slow trackers # / large box-sets can extend the deadline without editing code. DEFAULT_POLL_INTERVAL_SECONDS = 2.0 DEFAULT_POLL_TIMEOUT_SECONDS = 6 * 60 * 60 def get_poll_interval() -> float: """Return the per-poll sleep duration (seconds). Configurable via ``download_source.album_bundle_poll_interval_seconds``.""" raw = config_manager.get('download_source.album_bundle_poll_interval_seconds', DEFAULT_POLL_INTERVAL_SECONDS) try: value = float(raw) if value > 0: return value except (TypeError, ValueError): pass return DEFAULT_POLL_INTERVAL_SECONDS def get_poll_timeout() -> float: """Return the total deadline for an album-bundle download (seconds). Configurable via ``download_source.album_bundle_timeout_seconds``.""" raw = config_manager.get('download_source.album_bundle_timeout_seconds', DEFAULT_POLL_TIMEOUT_SECONDS) try: value = float(raw) if value > 0: return value except (TypeError, ValueError): pass return DEFAULT_POLL_TIMEOUT_SECONDS def quality_score(title: str, quality_guess) -> int: """Map a release title's inferred quality to a sortable integer. ``quality_guess`` is the function from each plugin that maps a title string to a quality string ('flac' / 'mp3' / etc.) — passed in so this module doesn't have to import either plugin and risk a circular import.""" return _QUALITY_SCORE.get(quality_guess(title) or '', 0) def pick_best_album_release(candidates, quality_guess) -> Optional[object]: """Pick the single best torrent / NZB for an album-bundle download. Heuristic, in priority order: 1. Reasonable album-ish size (40 MB – 3 GB) — drops single-track releases that snuck in and quarantines suspicious giants. 2. Higher seeders > lower (dead torrents = dead downloads). Usenet releases use ``grabs`` as a popularity proxy when seeders is None. 3. Higher quality (FLAC > AAC > MP3) inferred from title. 4. Larger size as tiebreaker (often = higher bitrate). """ if not candidates: return None sized = [c for c in candidates if ALBUM_PICK_MIN_BYTES <= (c.size or 0) <= ALBUM_PICK_MAX_BYTES] pool = sized or list(candidates) if not pool: return None def _score(c) -> tuple: seeders = c.seeders if c.seeders is not None else (c.grabs or 0) return (seeders, quality_score(c.title or '', quality_guess), c.size or 0) return max(pool, key=_score) def unique_staging_path(staging_dir: Path, src: Path) -> Path: """Return a destination path inside ``staging_dir`` that doesn't collide with an existing file. Appends ``_1``, ``_2``, ... before the extension when needed; gives up after 1000 candidates and returns the unsuffixed path so the caller will overwrite (better than infinite loop or crash).""" dest = staging_dir / src.name if not dest.exists(): return dest stem = dest.stem suffix = dest.suffix for i in range(1, 1000): candidate = staging_dir / f"{stem}_{i}{suffix}" if not candidate.exists(): return candidate return dest def atomic_copy_to_staging(src: Path, dest: Path) -> bool: """Copy ``src`` to ``dest`` without exposing a partial file to folder scanners. The Auto-Import worker filters by audio extension when scanning Staging — see ``AUDIO_EXTENSIONS`` in ``core/auto_import_worker.py``. Naming the in-flight file ``.tmp.`` keeps it invisible until the rename atomically swings it to its final extension. ``os.replace`` (used by ``Path.rename`` on Python 3.x) is atomic on the same filesystem, so Auto-Import either sees the file at its final name (complete) or doesn't see it at all (in flight). Returns True on success, False on copy / rename failure. Caller is expected to log the failure case so we don't double-log here. """ tmp = dest.with_name(f"{dest.name}.tmp.{uuid.uuid4().hex[:8]}") try: shutil.copy2(src, tmp) except Exception: # Best-effort cleanup of the partial file. If unlink fails # (locked, permissions) we leave it — Auto-Import ignores it # anyway because of the .tmp extension. try: if tmp.exists(): tmp.unlink() except Exception as cleanup_exc: logger.debug("album_bundle tmp cleanup failed: %s", cleanup_exc) raise try: tmp.replace(dest) return True except Exception: try: tmp.unlink(missing_ok=True) except Exception as cleanup_exc: logger.debug("album_bundle tmp cleanup failed: %s", cleanup_exc) raise # Number of consecutive None-status reads tolerated before treating the # job as gone. Sized for the SAB queue→history transition window: SAB # removes the slot from the queue before adding it to history, and on a # busy server (par2 verify + unrar) that window can be several poll # intervals. At the default 2s interval, 5 retries = ~10s of tolerance # before we give up and emit a terminal failure. Override via # ``download_source.album_bundle_transient_miss_threshold`` for users # whose servers need more headroom (very large multi-disc box sets, # slow disks, etc.). DEFAULT_TRANSIENT_MISS_THRESHOLD = 5 def get_transient_miss_threshold() -> int: """Return the configured transient-miss threshold for poll loops.""" raw = config_manager.get('download_source.album_bundle_transient_miss_threshold', DEFAULT_TRANSIENT_MISS_THRESHOLD) try: value = int(raw) if value > 0: return value except (TypeError, ValueError): pass return DEFAULT_TRANSIENT_MISS_THRESHOLD # How long to keep polling after the client reports terminal success # but hasn't yet exposed a final save_path. Distinct from the # transient-miss threshold because the two model different things: # a transient miss is "the job vanished — fail fast (~10s) so a deleted # job doesn't hang"; a completed-no-path read is "the download SUCCEEDED # and the files are on disk — SAB just hasn't finished writing the # ``storage`` field." The #706 fix reused the 5-poll (~10s) miss window # here, but #721's own report shows SAB can take 2+ minutes (or, on some # versions, never expose ``storage`` at all) — so a 10s window false-fails # a download that actually completed. Expressed in SECONDS (converted to # a poll count against the live interval) so it's interval-independent. # Override via ``download_source.album_bundle_completed_no_path_seconds``. DEFAULT_COMPLETED_NO_PATH_WINDOW_SECONDS = 120.0 def get_completed_no_path_window_seconds() -> float: """Return the completed-but-no-save_path tolerance window (seconds).""" raw = config_manager.get('download_source.album_bundle_completed_no_path_seconds', DEFAULT_COMPLETED_NO_PATH_WINDOW_SECONDS) try: value = float(raw) if value > 0: return value except (TypeError, ValueError): pass return DEFAULT_COMPLETED_NO_PATH_WINDOW_SECONDS class TransientMissCounter: """Bounded retry counter for adapter status reads. Both the album-bundle poll (in ``poll_album_download``) and the per-track download threads in ``usenet.py`` / ``torrent.py`` need the same "tolerate N consecutive missing or unmapped reads before declaring the job gone" logic. Lifted into one class so the rule is in one place and unit-testable in isolation — the per-track paths used to carry inline counters that mirrored this logic by hand, which is exactly the kind of duplication that drifts.""" def __init__(self, threshold: Optional[int] = None) -> None: self.threshold = threshold if threshold is not None else get_transient_miss_threshold() self.misses = 0 def record_miss(self) -> bool: """Bump the miss counter. Returns True when the counter has reached the threshold (caller should give up).""" self.misses += 1 return self.misses >= self.threshold def reset(self) -> None: """Successful read — reset the counter back to zero.""" self.misses = 0 def poll_album_download( *, get_status: Callable[[], Optional[Any]], title: str, emit: Callable[..., None], complete_states: frozenset, failed_states: frozenset = frozenset(['failed']), is_shutdown: Optional[Callable[[], bool]] = None, transient_miss_threshold: int = DEFAULT_TRANSIENT_MISS_THRESHOLD, completed_no_path_threshold: Optional[int] = None, poll_interval: Optional[float] = None, timeout: Optional[float] = None, sleep: Callable[[float], None] = time.sleep, monotonic: Callable[[], float] = time.monotonic, log_prefix: str = '[album_bundle]', ) -> Optional[str]: """Drive the per-poll status loop for an album-bundle download. Lifted out of ``UsenetDownloadPlugin._poll_album_download`` and the sibling torrent method so the loop is testable in isolation and so both plugins share the same exit semantics. Contract: - ``get_status()`` returns the adapter status object for the bound job, ``None`` when the client doesn't know about the job currently (transient or terminal — disambiguated by retry count). - ``emit(state, **fields)`` is the plugin's progress callback — this function calls it on EVERY successful poll with ``state='downloading'`` and ALWAYS calls it once more with ``state='failed'`` before returning ``None`` on any failure path, so the UI doesn't freeze on the last 'downloading' emit. - ``complete_states`` is the adapter's terminal-success set ('completed' alone for usenet; 'seeding' + 'completed' for torrent because seeding-but-files-on-disk also counts). - ``failed_states`` is the explicit-failure set. The adapter-level 'error' (unmapped state default) is intentionally NOT in here — that's treated as a transient miss because a real SAB / NZBGet / qBit never returns a literal 'error' state on a healthy job; it's only our default fallback for unknown queue strings. Real example: SAB's 'Pp' post-processing state was unmapped → became 'error' → poll infinite-looped until the 6-hour timeout. - ``transient_miss_threshold`` is the number of consecutive None / 'error' reads tolerated before declaring the job gone. Sized for the SAB queue→history gap window (~10s) — a vanished job should fail fast. - ``completed_no_path_threshold`` is a SEPARATE, longer window for the "client says complete but no save_path yet" case. The download already succeeded, so this defaults to ~120s (configurable via ``download_source.album_bundle_completed_no_path_seconds``) instead of reusing the 10s miss window — #721 showed SAB can take 2+ minutes to write ``storage``. When the window is exhausted the loop falls back to the adapter's ``incomplete_path`` (the on-disk in-progress dir) if present, and only emits terminal ``failed`` when there's no path of any kind to scan. Returns the adapter's reported save_path (or, as a last resort, its ``incomplete_path``) on terminal success, or ``None`` on any failure (timeout / disappeared / explicit failed / shutdown). On every failure path emits ``'failed'`` once with an ``error`` field describing why. """ interval = poll_interval if poll_interval is not None else get_poll_interval() deadline = monotonic() + (timeout if timeout is not None else get_poll_timeout()) last_save_path: Optional[str] = None last_incomplete_path: Optional[str] = None misses = TransientMissCounter(transient_miss_threshold) # Separate counter for "client reports terminal-success state but no # save_path field has landed yet." SAB History flips ``status`` to # 'Completed' a few seconds before its post-processing pipeline # writes the final ``storage`` field — see issue #721 (Forty Licks # stuck at 61%): SAB shows Completed in the UI, but # ``_parse_history_slot`` returns ``save_path=None`` for those few # seconds because ``storage`` isn't populated yet. Pre-fix the # poll returned ``None`` on the first such read, the bundle # plugin marked the batch failed, but the UI still displayed the # last ``downloading`` progress emit. # # This window is intentionally LONGER than the transient-miss window: # the download already SUCCEEDED, so being patient here is cheap and # correct, whereas the original 5-poll (~10s) reuse false-failed real # completions (#721 reported SAB taking 2+ minutes). Default ~120s, # converted from seconds to a poll count against the live interval. if completed_no_path_threshold is None: completed_no_path_threshold = max( transient_miss_threshold, int(get_completed_no_path_window_seconds() / max(interval, 0.001)) or 1, ) completed_no_path_misses = TransientMissCounter(completed_no_path_threshold) def _fail(reason: str) -> None: try: emit('failed', release=title, error=reason) except Exception as cb_exc: logger.debug("%s terminal emit failed: %s", log_prefix, cb_exc) # Heartbeat so the otherwise-silent download loop is diagnosable. # The loop emits progress to the UI on every poll but logs nothing # during normal operation — which made the #721 "stuck at N%" reports # impossible to triage from logs alone (we couldn't tell if the poll # was alive, what state SAB returned, or whether it had wedged). Log # the raw adapter read at most once per heartbeat interval. HEARTBEAT_SECONDS = 30.0 last_heartbeat = monotonic() poll_count = 0 while monotonic() < deadline: if is_shutdown and is_shutdown(): # Shutdown is a clean exit — don't paint failure on the UI; # the app is going away anyway. return None try: status = get_status() except Exception as e: logger.warning("%s Poll error: %s", log_prefix, e) status = None poll_count += 1 now = monotonic() if now - last_heartbeat >= HEARTBEAT_SECONDS: last_heartbeat = now if status is None: logger.info("%s '%s' poll #%d: client returned no status (miss %d/%d)", log_prefix, title, poll_count, misses.misses, misses.threshold) else: logger.info( "%s '%s' poll #%d: state=%r progress=%.2f save_path=%r", log_prefix, title, poll_count, getattr(status, 'state', None), getattr(status, 'progress', 0.0) or 0.0, getattr(status, 'save_path', None), ) if status is None: if misses.record_miss(): logger.error( "%s '%s' missing from client for %d consecutive polls — giving up", log_prefix, title, misses.misses, ) _fail('Disappeared from client (no status after retries)') return None sleep(interval) continue # Reset the miss counter only when the adapter returned a state # we actually recognise. The default-fallback 'error' is treated # as a continuing transient miss below, so it must NOT reset # here — otherwise a persistently-unmapped state loops forever. if status.state != 'error': misses.reset() emit('downloading', progress=status.progress, downloaded=status.downloaded, speed=status.download_speed) if status.save_path: last_save_path = status.save_path # Remember the in-progress dir too — never used on a normal # completion, only as the last-resort fallback below when the # final save_path provably never lands. incomplete_path = getattr(status, 'incomplete_path', None) if incomplete_path: last_incomplete_path = incomplete_path if status.state in complete_states: if last_save_path: completed_no_path_misses.reset() return last_save_path # Terminal-success state but no save_path landed yet. # SAB History flips ``Completed`` a few seconds before # ``storage`` is populated — give the adapter a generous # window before declaring this a hard failure. Without this # tolerance, every TAR / unrar-bearing usenet release # would race the path-write window and randomly fail. if completed_no_path_misses.record_miss(): # Last resort before failing: SAB finished and the files # are physically on disk (#721), but the final ``storage`` # field never landed. Fall back to the in-progress dir so # the bundle can still scan + stage the audio, rather than # leaving the user stuck with a completed-in-SAB download # that SoulSync never imports. if last_incomplete_path: logger.warning( "%s '%s' completed on the client but never exposed a final " "save_path after %d polls — falling back to the in-progress " "path %r as a last resort. If staging fails, the SAB job " "likely needs its post-process move to finish first.", log_prefix, title, completed_no_path_misses.misses, last_incomplete_path, ) return last_incomplete_path logger.error( "%s '%s' reported terminal success but no save_path landed " "after %d consecutive polls — bundle cannot stage. Adapter " "may need new history-slot fallback fields (storage / path " "/ download_path / dirname). Last status: state=%r progress=%r", log_prefix, title, completed_no_path_misses.misses, status.state, status.progress, ) _fail('Client reported success but never provided a save_path') return None logger.info( "%s '%s' is %s on the client but save_path not yet set — " "retrying (poll %d/%d)", log_prefix, title, status.state, completed_no_path_misses.misses, completed_no_path_misses.threshold, ) sleep(interval) continue if status.state in failed_states: error = getattr(status, 'error', None) or 'Client reported failure' logger.error("%s '%s' failed: %s", log_prefix, title, error) _fail(error) return None if status.state == 'error': # Unmapped adapter state — see contract docstring. Warn so # we hear about new states the adapter map needs to grow # without breaking the user's download. The miss counter # was intentionally NOT reset above for this branch. logger.warning( "%s '%s' returned unmapped state — treating as transient", log_prefix, title, ) if misses.record_miss(): _fail('Client returned unmapped state repeatedly') return None sleep(interval) logger.error("%s '%s' timed out", log_prefix, title) _fail('Download timed out') return None def _candidate_download_roots(config_get: Callable[..., Any]) -> list: """Directories where THIS process can read finished downloads — used by ``resolve_reported_save_path`` for the basename fallback. Order matters: most-specific usenet/torrent roots first, then the general Soulseek download / transfer dirs, which in the standard shared-volume arr setup are bind-mounted to the very directory the usenet client writes its completed downloads into. Relative values (e.g. ``./downloads``) resolve against the process CWD — the container's ``/app`` — which is exactly where those mounts live. """ roots: list = [] for key in ( 'download_source.usenet_download_path', 'usenet_client.completed_path', 'usenet_client.download_path', 'download_source.torrent_download_path', 'soulseek.download_path', 'soulseek.transfer_path', ): value = config_get(key, None) if value: roots.append(str(value)) seen: set = set() out: list = [] for root in roots: if root not in seen: seen.add(root) out.append(root) return out def resolve_reported_save_path( reported_path: Optional[str], config_get: Optional[Callable[..., Any]] = None, ) -> Optional[str]: """Translate a downloader-reported save_path into one THIS process can read. Usenet / torrent clients report paths from inside THEIR OWN container (e.g. SAB hands back ``/data/downloads/music/``); SoulSync often mounts the very same files at a different point (``/app/downloads/``). Feeding the client's path straight to the audio walker then yields "No audio files found" even though the files are physically present — the classic arr-stack remote-path mismatch. Resolution order: 1. The reported path verbatim, if it's a readable directory here (deployments that mirror the client's mount paths). 2. Explicit prefix mappings from ``download_source.usenet_path_mappings`` — a list of ``{"from": "...", "to": "..."}`` (Sonarr/Radarr-style remote path mapping) for non-shared / oddly-mounted layouts. 3. Basename fallback: a same-named folder under a known SoulSync download root. Zero-config for the standard shared-volume setup — the album folder shows up under SoulSync's own ``./downloads`` mount with the same name the client reported. Returns the best resolved path, or ``reported_path`` unchanged when nothing better is found (so the caller's existing "no audio" error still surfaces, with both paths logged). """ if not reported_path: return reported_path if config_get is None: config_get = config_manager.get def _is_dir(candidate) -> bool: try: return Path(candidate).is_dir() except OSError: return False # 1. Reported path is directly readable — mounts already line up. if _is_dir(reported_path): return reported_path normalized = str(reported_path).replace('\\', '/') # 2. Explicit prefix mappings (remote-path-mapping escape hatch). mappings = config_get('download_source.usenet_path_mappings', None) or [] if isinstance(mappings, (list, tuple)): for mapping in mappings: if not isinstance(mapping, dict): continue frm = str(mapping.get('from') or '').replace('\\', '/').rstrip('/') to = str(mapping.get('to') or '') if not frm or not to: continue if normalized == frm or normalized.startswith(frm + '/'): rest = normalized[len(frm):].lstrip('/') candidate = str(Path(to) / rest) if rest else to if _is_dir(candidate): return candidate # 3. Basename fallback under known download roots — covers the standard # shared-volume layout with zero configuration. basename = Path(normalized).name if basename: for root in _candidate_download_roots(config_get): candidate = Path(root) / basename if _is_dir(candidate): return str(candidate) return reported_path def copy_audio_files_atomically( sources: Iterable[Path], staging_dir: Path, ) -> list: """Convenience wrapper: pick a non-colliding staging path for each source, copy via ``atomic_copy_to_staging``. Returns the list of final destination paths (as strings). Files that fail to copy are logged and skipped; the caller decides what to do with a partial result.""" staging_dir.mkdir(parents=True, exist_ok=True) out: list = [] for src in sources: dest = unique_staging_path(staging_dir, src) try: atomic_copy_to_staging(src, dest) out.append(str(dest)) except Exception as e: logger.warning("[album_bundle] Failed to stage %s -> %s: %s", src, dest, e) return out # Re-export so callers don't have to remember which module owns # what. The ``time`` import is kept so plugins can ``from # core.download_plugins.album_bundle import time`` if they want to, # avoiding a second std-lib import line for a single use. __all__ = [ "ALBUM_PICK_MIN_BYTES", "ALBUM_PICK_MAX_BYTES", "DEFAULT_POLL_INTERVAL_SECONDS", "DEFAULT_POLL_TIMEOUT_SECONDS", "DEFAULT_TRANSIENT_MISS_THRESHOLD", "DEFAULT_COMPLETED_NO_PATH_WINDOW_SECONDS", "TransientMissCounter", "atomic_copy_to_staging", "copy_audio_files_atomically", "get_completed_no_path_window_seconds", "get_poll_interval", "get_poll_timeout", "get_transient_miss_threshold", "resolve_reported_save_path", "pick_best_album_release", "poll_album_download", "quality_score", "time", "unique_staging_path", ]