From 6c9b43225a273b3701757de4fd2a5a557c1ffc8f Mon Sep 17 00:00:00 2001 From: Broque Thomas <26755000+Nezreka@users.noreply.github.com> Date: Thu, 21 May 2026 14:22:21 -0700 Subject: [PATCH] Add torrent and usenet release staging support Adds torrent/usenet as release-oriented download sources with album-bundle staging, live progress reporting, and post-processing that selects the requested audio file from completed releases instead of blindly importing the first file. Keeps album-bundle behavior gated to single-source torrent/usenet album downloads, excludes release sources from hybrid album per-track searches, and allows hybrid non-album tracks to use release results safely. Improves staged-release matching for featured/bonus track filenames while preserving version mismatches, records torrent/usenet provenance in library history, and updates service/status UI labels. Covers the flow with focused lifecycle, status, staging, validation, task worker, post-processing, and import side-effect tests. --- core/connection_test.py | 12 ++ core/download_plugins/torrent.py | 8 +- core/download_plugins/types.py | 1 + core/download_plugins/usenet.py | 2 + core/downloads/lifecycle.py | 45 +++++ core/downloads/monitor.py | 123 ++++++++----- core/downloads/post_processing.py | 158 +++++++++++++++- core/downloads/staging.py | 97 +++++++++- core/downloads/status.py | 47 +++-- core/downloads/task_worker.py | 77 +++++++- core/downloads/validation.py | 49 ++++- core/imports/side_effects.py | 4 + tests/downloads/test_downloads_lifecycle.py | 42 +++++ .../test_downloads_post_processing.py | 155 ++++++++++++++++ tests/downloads/test_downloads_staging.py | 174 ++++++++++++++++++ tests/downloads/test_downloads_status.py | 126 +++++++++++++ tests/downloads/test_downloads_task_worker.py | 54 +++++- tests/downloads/test_downloads_validation.py | 44 +++++ tests/imports/test_import_side_effects.py | 38 ++++ tests/test_manual_pick_no_auto_retry.py | 129 +++++++++++++ web_server.py | 52 +++++- webui/static/downloads.js | 57 ++++++ webui/static/shared-helpers.js | 8 +- 23 files changed, 1399 insertions(+), 103 deletions(-) diff --git a/core/connection_test.py b/core/connection_test.py index c470cfec..745caa2a 100644 --- a/core/connection_test.py +++ b/core/connection_test.py @@ -191,6 +191,12 @@ def run_service_test(service, test_config): 'tidal': "Tidal download source ready.", 'qobuz': "Qobuz download source ready.", 'hifi': "HiFi download source ready.", + 'deezer_dl': "Deezer download source ready.", + 'amazon': "Amazon download source ready.", + 'lidarr': "Lidarr download source ready.", + 'soundcloud': "SoundCloud download source ready.", + 'torrent': "Torrent download source ready.", + 'usenet': "Usenet download source ready.", 'hybrid': "Download sources ready (Hybrid mode)." } message = mode_messages.get(download_mode, "Download source connected.") @@ -203,6 +209,12 @@ def run_service_test(service, test_config): 'tidal': "Tidal download source not available. Check authentication.", 'qobuz': "Qobuz download source not available. Check authentication.", 'hifi': "HiFi download source not available. Public API instances may be down.", + 'deezer_dl': "Deezer download source not available. Check authentication.", + 'amazon': "Amazon download source not available.", + 'lidarr': "Lidarr download source not available. Check Lidarr URL and API key.", + 'soundcloud': "SoundCloud download source not available.", + 'torrent': "Torrent download source not available. Check Prowlarr and torrent client settings.", + 'usenet': "Usenet download source not available. Check Prowlarr and usenet client settings.", 'hybrid': "Could not connect to download sources. Check configuration." } error = mode_errors.get(download_mode, "Download source connection failed.") diff --git a/core/download_plugins/torrent.py b/core/download_plugins/torrent.py index a04c8247..388cbe98 100644 --- a/core/download_plugins/torrent.py +++ b/core/download_plugins/torrent.py @@ -16,8 +16,9 @@ Two flows: polls the adapter for completion. 3. On completion the thread walks the adapter-reported save path via ``archive_pipeline.collect_audio_after_extraction`` and - marks the download succeeded with the first audio file as the - primary ``file_path``. + exposes the full audio-file list. Post-processing can then pick + the requested track from a completed release instead of importing + the first file blindly. **Album-bundle flow** (album-context batch downloads — wired in ``core/downloads/master.py``) — @@ -257,6 +258,7 @@ class TorrentDownloadPlugin(DownloadSourcePlugin): 'transferred': 0, 'speed': 0, 'file_path': None, + 'audio_files': [], 'torrent_hash': None, 'error': None, } @@ -353,6 +355,7 @@ class TorrentDownloadPlugin(DownloadSourcePlugin): row['state'] = 'Completed, Succeeded' row['progress'] = 100.0 row['file_path'] = str(primary) + row['audio_files'] = [str(path) for path in audio_files] logger.info("Torrent download complete: %s -> %s (%d audio files)", download_id[:8], primary.name, len(audio_files)) @@ -659,4 +662,5 @@ def _row_to_status(row: Dict[str, Any]) -> DownloadStatus: speed=int(row.get('speed', 0)), time_remaining=None, file_path=row.get('file_path'), + audio_files=row.get('audio_files') or None, ) diff --git a/core/download_plugins/types.py b/core/download_plugins/types.py index 46e250c3..59d52913 100644 --- a/core/download_plugins/types.py +++ b/core/download_plugins/types.py @@ -197,3 +197,4 @@ class DownloadStatus: speed: int time_remaining: Optional[int] = None file_path: Optional[str] = None + audio_files: Optional[List[str]] = None diff --git a/core/download_plugins/usenet.py b/core/download_plugins/usenet.py index ea0f4397..419bc380 100644 --- a/core/download_plugins/usenet.py +++ b/core/download_plugins/usenet.py @@ -190,6 +190,7 @@ class UsenetDownloadPlugin(DownloadSourcePlugin): 'transferred': 0, 'speed': 0, 'file_path': None, + 'audio_files': [], 'job_id': None, 'error': None, } @@ -281,6 +282,7 @@ class UsenetDownloadPlugin(DownloadSourcePlugin): row['state'] = 'Completed, Succeeded' row['progress'] = 100.0 row['file_path'] = str(primary) + row['audio_files'] = [str(path) for path in audio_files] logger.info("Usenet download complete: %s -> %s (%d audio files)", download_id[:8], primary.name, len(audio_files)) diff --git a/core/downloads/lifecycle.py b/core/downloads/lifecycle.py index 2a29feec..5891e8be 100644 --- a/core/downloads/lifecycle.py +++ b/core/downloads/lifecycle.py @@ -26,9 +26,11 @@ Lifted verbatim from web_server.py. Dependencies injected via from __future__ import annotations import logging +import shutil import time import traceback from dataclasses import dataclass +from pathlib import Path from typing import Any, Callable, Optional from core.downloads.history import record_sync_history_completion @@ -42,6 +44,47 @@ from core.runtime_state import ( logger = logging.getLogger(__name__) +def _safe_batch_dirname(batch_id: str) -> str: + safe = ''.join(ch if ch.isalnum() or ch in ('-', '_') else '_' for ch in str(batch_id or 'batch')) + return safe or 'batch' + + +def _cleanup_private_album_bundle_staging(batch_id: str, batch: dict) -> None: + """Best-effort cleanup for torrent/usenet private staging copies. + + The torrent/usenet clients keep their own completed download folders. + This only removes SoulSync's per-batch copy under album-bundle staging. + """ + if not batch.get('album_bundle_private_staging'): + return + if (batch.get('album_bundle_source') or '').lower() not in ('torrent', 'usenet'): + return + + staging_path = batch.get('album_bundle_staging_path') + if not staging_path: + return + + path = Path(staging_path) + expected_name = _safe_batch_dirname(batch_id) + if path.name != expected_name: + logger.warning( + "[Album Bundle] Refusing to clean private staging path with unexpected name: %s", + staging_path, + ) + return + if not path.exists(): + return + if not path.is_dir(): + logger.warning("[Album Bundle] Refusing to clean non-directory staging path: %s", staging_path) + return + + try: + shutil.rmtree(path) + logger.info("[Album Bundle] Cleaned private staging folder for batch %s: %s", batch_id, staging_path) + except Exception as exc: + logger.warning("[Album Bundle] Could not clean private staging folder %s: %s", staging_path, exc) + + @dataclass class LifecycleDeps: """Bundle of cross-cutting deps the batch lifecycle needs.""" @@ -398,6 +441,7 @@ def on_download_completed(batch_id: str, task_id: str, success: bool, deps: Life logger.info(f"[Batch Manager] Batch {batch_id} complete - stopping monitor") deps.download_monitor.stop_monitoring(batch_id) + _cleanup_private_album_bundle_staging(batch_id, batch) # M3U REGENERATION: Regenerate M3U with real library paths now that # all post-processing (tagging, moving, DB writes) is complete. @@ -606,6 +650,7 @@ def check_batch_completion_v2(batch_id: str, deps: LifecycleDeps) -> Optional[bo logger.info(f"[Completion Check V2] Batch {batch_id} complete - stopping monitor") deps.download_monitor.stop_monitoring(batch_id) + _cleanup_private_album_bundle_staging(batch_id, batch) # REPAIR: Scan all album folders from this batch for track number issues if deps.repair_worker: diff --git a/core/downloads/monitor.py b/core/downloads/monitor.py index 62d515e5..c4b645f5 100644 --- a/core/downloads/monitor.py +++ b/core/downloads/monitor.py @@ -34,6 +34,32 @@ _start_next_batch_of_downloads = None _orphaned_download_keys = None missing_download_executor = None download_orchestrator = None +_RELEASE_SOURCE_NAMES = frozenset(('torrent', 'usenet')) + + +def _download_id_key(download_id): + return f"download_id::{download_id}" if download_id else None + + +def _is_release_task(task): + ti = task.get('track_info') if isinstance(task.get('track_info'), dict) else {} + username = task.get('username') or ti.get('username') + return username in _RELEASE_SOURCE_NAMES + + +def _lookup_live_info(task, live_transfers_lookup): + ti = task.get('track_info') if isinstance(task.get('track_info'), dict) else {} + download_id = task.get('download_id') + if _is_release_task(task): + by_id = live_transfers_lookup.get(_download_id_key(download_id)) + if by_id: + return by_id + + task_filename = task.get('filename') or ti.get('filename') + task_username = task.get('username') or ti.get('username') + if not task_filename or not task_username: + return None + return live_transfers_lookup.get(_make_context_key(task_username, task_filename)) def init( @@ -139,55 +165,64 @@ class WebUIDownloadMonitor: for task_id in download_batches[batch_id].get('queue', []): task = download_tasks.get(task_id) - if not task or task['status'] not in ['downloading', 'queued']: + if not task: + continue + release_recoverable = ( + _is_release_task(task) + and task.get('download_id') + and task.get('status') in ['failed', 'not_found'] + ) + if task['status'] not in ['downloading', 'queued'] and not release_recoverable: continue # Check for timeouts and errors - retries handled directly in _should_retry_task # If _should_retry_task returns True, it means retries were exhausted - retry_exhausted = self._should_retry_task(task_id, task, live_transfers_lookup, current_time, deferred_ops) + retry_exhausted = False + if not release_recoverable: + retry_exhausted = self._should_retry_task(task_id, task, live_transfers_lookup, current_time, deferred_ops) # Collect exhausted tasks to handle outside lock (prevents deadlock) if retry_exhausted: exhausted_tasks.append((batch_id, task_id)) - # ENHANCED: Check for successful completions (especially YouTube) - task_filename = task.get('filename') or task.get('track_info', {}).get('filename') - task_username = task.get('username') or task.get('track_info', {}).get('username') + # ENHANCED: Check for successful completions (especially YouTube). + # Release-style sources can report a completed audio file + # name that differs from the original indexer URL/title + # stored on the task, so prefer the stable download_id. + live_info = _lookup_live_info(task, live_transfers_lookup) - if task_filename and task_username: - lookup_key = _make_context_key(task_username, task_filename) - live_info = live_transfers_lookup.get(lookup_key) - - if live_info: - state = live_info.get('state', '') - # Trigger post-processing if download is completed successfully - # slskd uses compound states like 'Completed, Succeeded' - use substring matching - # Must exclude error states first (matching _build_batch_status_data's prioritized checking) - has_error = ('Errored' in state or 'Failed' in state or 'Rejected' in state or 'TimedOut' in state) - has_completion = ('Completed' in state or 'Succeeded' in state) - # Verify bytes actually transferred before trusting state string. - # slskd can report "Completed" before the full file is flushed to disk, - # or on connection drops that leave a partial file. - if has_completion and not has_error: - expected_size = live_info.get('size', 0) - transferred = live_info.get('bytesTransferred', 0) - if expected_size > 0 and transferred < expected_size: - if not task.get('_incomplete_warned'): - logger.debug(f"Monitor: {task_id} state={state} but bytes incomplete ({transferred}/{expected_size}) — waiting") - task['_incomplete_warned'] = True - continue - if has_completion and not has_error and task['status'] == 'downloading': - task.pop('_incomplete_warned', None) - # CRITICAL FIX: Transition to 'post_processing' HERE so downloads - # don't depend on browser polling to trigger post-processing. - # Previously, post-processing was only submitted by _build_batch_status_data - # (called from browser-polled endpoints), meaning closing the browser - # left tasks stuck in 'downloading' forever. - task['status'] = 'post_processing' - task['status_change_time'] = current_time - logger.info(f"Monitor detected completed download for {task_id} ({state}) - submitting post-processing") - # Collect for handling outside the lock to prevent deadlock. - # _on_download_completed acquires tasks_lock which is non-reentrant. - completed_tasks.append((batch_id, task_id)) + if live_info: + state = live_info.get('state', '') + # Trigger post-processing if download is completed successfully + # slskd uses compound states like 'Completed, Succeeded' - use substring matching + # Must exclude error states first (matching _build_batch_status_data's prioritized checking) + has_error = ('Errored' in state or 'Failed' in state or 'Rejected' in state or 'TimedOut' in state) + has_completion = ('Completed' in state or 'Succeeded' in state) + # Verify bytes actually transferred before trusting state string. + # slskd can report "Completed" before the full file is flushed to disk, + # or on connection drops that leave a partial file. + if has_completion and not has_error: + expected_size = live_info.get('size', 0) + transferred = live_info.get('bytesTransferred', 0) + if expected_size > 0 and transferred < expected_size: + if not task.get('_incomplete_warned'): + logger.debug("Monitor: %s state=%s but bytes incomplete (%s/%s) - waiting", task_id, state, transferred, expected_size) + task['_incomplete_warned'] = True + continue + if has_completion and not has_error and ( + task['status'] == 'downloading' or release_recoverable + ): + task.pop('_incomplete_warned', None) + # CRITICAL FIX: Transition to 'post_processing' HERE so downloads + # don't depend on browser polling to trigger post-processing. + # Previously, post-processing was only submitted by _build_batch_status_data + # (called from browser-polled endpoints), meaning closing the browser + # left tasks stuck in 'downloading' forever. + task['status'] = 'post_processing' + task['status_change_time'] = current_time + logger.info(f"Monitor detected completed download for {task_id} ({state}) - submitting post-processing") + # Collect for handling outside the lock to prevent deadlock. + # _on_download_completed acquires tasks_lock which is non-reentrant. + completed_tasks.append((batch_id, task_id)) # ---- All work below runs WITHOUT tasks_lock held ---- if globals().get('IS_SHUTTING_DOWN', False) or not self.monitoring: @@ -308,7 +343,7 @@ class WebUIDownloadMonitor: for download in all_downloads: key = _make_context_key(download.username, download.filename) # Convert DownloadStatus to transfer dict format for monitor compatibility - live_transfers[key] = { + transfer_row = { 'id': download.id, 'filename': download.filename, 'username': download.username, @@ -318,6 +353,10 @@ class WebUIDownloadMonitor: 'bytesTransferred': download.transferred, 'averageSpeed': download.speed, } + live_transfers[key] = transfer_row + id_key = _download_id_key(download.id) + if id_key: + live_transfers[id_key] = transfer_row except Exception as yt_error: logger.error(f"Monitor: Could not fetch streaming source downloads: {yt_error}") @@ -352,7 +391,7 @@ class WebUIDownloadMonitor: return False lookup_key = _make_context_key(task_username, task_filename) - live_info = live_transfers_lookup.get(lookup_key) + live_info = _lookup_live_info(task, live_transfers_lookup) if not live_info: # User-initiated manual pick — skip auto-retry. The status diff --git a/core/downloads/post_processing.py b/core/downloads/post_processing.py index 1b2d04d2..2c17e90f 100644 --- a/core/downloads/post_processing.py +++ b/core/downloads/post_processing.py @@ -19,9 +19,11 @@ from __future__ import annotations import logging import os +import shutil import time import traceback from dataclasses import dataclass +from difflib import SequenceMatcher from pathlib import Path from typing import Any, Callable, Optional @@ -34,7 +36,7 @@ from core.imports.context import ( get_import_original_search, normalize_import_context, ) -from core.imports.filename import extract_track_number_from_filename +from core.imports.filename import extract_track_number_from_filename, parse_filename_metadata from core.metadata import enrichment as metadata_enrichment from core.runtime_state import ( download_tasks, @@ -46,6 +48,85 @@ from core.runtime_state import ( logger = logging.getLogger(__name__) +_AUDIO_EXTENSIONS = { + '.flac', '.ape', '.wav', '.alac', '.dsf', '.dff', '.aiff', '.aif', + '.opus', '.ogg', '.m4a', '.aac', '.mp3', '.wma', +} + + +def _is_audio_file(path: str) -> bool: + return Path(str(path or '')).suffix.lower() in _AUDIO_EXTENSIONS + + +def _reject_non_audio_found_file(found_file: Optional[str], file_location: Optional[str]) -> tuple[Optional[str], Optional[str]]: + if found_file and not _is_audio_file(found_file): + logger.warning( + "[Post-Processing] Ignoring non-audio candidate found during file search: %s", + found_file, + ) + return None, None + return found_file, file_location + + +def _normalize_match_text(value: str) -> str: + return ''.join(ch.lower() for ch in str(value or '') if ch.isalnum()) + + +def _release_audio_match_score(path: str, expected_title: str, expected_artist: str) -> float: + parsed = parse_filename_metadata(path) + parsed_title = parsed.get('title') or Path(path).stem + parsed_artist = parsed.get('artist') or '' + expected_title_norm = _normalize_match_text(expected_title) + parsed_title_norm = _normalize_match_text(parsed_title) + if expected_title_norm and ( + expected_title_norm in parsed_title_norm or parsed_title_norm in expected_title_norm + ): + title_score = 1.0 + else: + title_score = SequenceMatcher( + None, + expected_title_norm, + parsed_title_norm, + ).ratio() + if expected_artist and parsed_artist: + artist_score = SequenceMatcher( + None, + _normalize_match_text(expected_artist), + _normalize_match_text(parsed_artist), + ).ratio() + return (title_score * 0.75) + (artist_score * 0.25) + return title_score + + +def _track_title_from_task(track_info: Any, context: Optional[dict]) -> str: + if isinstance(track_info, dict): + title = track_info.get('name') or track_info.get('title') + if title: + return str(title) + return get_import_clean_title(context or {}, default='') + + +def _copy_release_audio_to_transfer(source_path: str, transfer_dir: str) -> Optional[str]: + try: + src = Path(source_path) + if not src.exists() or not src.is_file(): + return None + dest_dir = Path(transfer_dir) + dest_dir.mkdir(parents=True, exist_ok=True) + dest = dest_dir / src.name + if dest.exists(): + stem, suffix = src.stem, src.suffix + counter = 1 + while dest.exists(): + dest = dest_dir / f"{stem}_release_{counter}{suffix}" + counter += 1 + shutil.copy2(src, dest) + return str(dest) + except Exception as exc: + logger.warning("[Post-Processing] Could not copy release audio to transfer: %s", exc) + return None + + @dataclass class PostProcessDeps: """Bundle of dependencies the post-processing worker needs. @@ -190,14 +271,71 @@ def run_post_processing_worker(task_id: str, batch_id: str, deps: PostProcessDep # CRITICAL FIX: For YouTube downloads, the filename in task is 'id||title' (metadata), # but the actual file on disk is 'Title.mp3'. We must ask the client for the real path. + # Torrent/usenet also use opaque "url||display" filenames. Their completed + # job may contain a full release, so choose the best matching audio file + # and copy it to transfer before importing instead of moving the client's + # original completed download. if (task.get('username') == 'youtube' or '||' in str(task_filename)) and not found_file: - logger.info(f"[Post-Processing] Detected YouTube download task: {task_id}") + logger.info(f"[Post-Processing] Detected engine-backed download task: {task_id}") try: # Query the download orchestrator for the status which contains the real file path # CRITICAL FIX: Use the actual download_id designated by the client, not the internal task_id actual_download_id = task.get('download_id') or task_id status = deps.run_async(deps.download_orchestrator.get_download_status(actual_download_id)) - if status and status.file_path: + if status and task.get('username') in ('torrent', 'usenet'): + audio_files = list(getattr(status, 'audio_files', None) or []) + if not audio_files and getattr(status, 'file_path', None): + audio_files = [status.file_path] + expected_title = _track_title_from_task(track_info, context) + expected_artist = '' + if isinstance(track_info, dict): + artists = track_info.get('artists', []) + if isinstance(artists, list) and artists: + first_artist = artists[0] + expected_artist = first_artist.get('name', '') if isinstance(first_artist, dict) else str(first_artist) + elif isinstance(artists, str): + expected_artist = artists + if not expected_artist and context: + artist_ctx = get_import_context_artist(context) + expected_artist = artist_ctx.get('name', '') if isinstance(artist_ctx, dict) else '' + + scored_files = [ + (_release_audio_match_score(path, expected_title, expected_artist), path) + for path in audio_files + if _is_audio_file(path) + ] + scored_files.sort(reverse=True) + if scored_files: + best_score, best_path = scored_files[0] + logger.info( + "[Post-Processing] Best %s release file for '%s': %s (score %.2f)", + task.get('username'), expected_title, best_path, best_score, + ) + if best_score >= 0.80: + copied_path = _copy_release_audio_to_transfer(best_path, transfer_dir) + if copied_path: + found_file = copied_path + file_location = 'download' + logger.info( + "[Post-Processing] Copied matched %s release file to transfer: %s", + task.get('username'), copied_path, + ) + else: + logger.warning( + "[Post-Processing] No %s release file met match threshold for '%s' (best %.2f)", + task.get('username'), expected_title, best_score, + ) + if not found_file: + with tasks_lock: + if task_id in download_tasks: + download_tasks[task_id]['status'] = 'failed' + download_tasks[task_id]['error_message'] = ( + f"No matching audio file found in {task.get('username')} release" + ) + deps.on_download_completed(batch_id, task_id, False) + return + + if status and status.file_path and not found_file and task.get('username') not in ('torrent', 'usenet'): real_path = status.file_path if os.path.exists(real_path): # Determine if it's in download or transfer directory @@ -227,11 +365,12 @@ def run_post_processing_worker(task_id: str, batch_id: str, deps: PostProcessDep logger.info(f"[Post-Processing] Resolved actual YouTube filename: {found_file} (Location: {file_location})") else: - logger.warning(f"[Post-Processing] YouTube status reported path but file missing: {real_path}") + logger.warning(f"[Post-Processing] Engine status reported path but file missing: {real_path}") else: - logger.warning(f"[Post-Processing] YouTube status returned no file_path for task {task_id}") + if not found_file: + logger.warning(f"[Post-Processing] Engine status returned no file_path for task {task_id}") except Exception as e: - logger.error(f"[Post-Processing] Failed to retrieve YouTube task status: {e}") + logger.error(f"[Post-Processing] Failed to retrieve engine task status: {e}") _file_search_max_retries = 5 for retry_count in range(_file_search_max_retries): @@ -257,6 +396,7 @@ def run_post_processing_worker(task_id: str, batch_id: str, deps: PostProcessDep # Strategy 1: Try with original filename in both downloads and transfer logger.info("[Post-Processing] Strategy 1: Searching with original filename...") found_file, file_location = deps.find_completed_file(download_dir, task_filename, transfer_dir) + found_file, file_location = _reject_non_audio_found_file(found_file, file_location) if found_file: logger.info(f"[Post-Processing] Strategy 1 SUCCESS: Found file with original filename in {file_location}: {found_file}") @@ -269,7 +409,11 @@ def run_post_processing_worker(task_id: str, batch_id: str, deps: PostProcessDep found_result = deps.find_completed_file(transfer_dir, expected_final_filename) if found_result and found_result[0]: found_file, file_location = found_result[0], 'transfer' - logger.info(f"[Post-Processing] Strategy 2 SUCCESS: Found file with expected final filename: {found_file}") + found_file, file_location = _reject_non_audio_found_file(found_file, file_location) + if found_file: + logger.info(f"[Post-Processing] Strategy 2 SUCCESS: Found file with expected final filename: {found_file}") + else: + logger.error("[Post-Processing] Strategy 2 FAILED: Expected final filename resolved to a non-audio file") else: logger.error("[Post-Processing] Strategy 2 FAILED: Expected final filename not found in transfer folder") elif not expected_final_filename: diff --git a/core/downloads/staging.py b/core/downloads/staging.py index ecd38e40..f7ca8906 100644 --- a/core/downloads/staging.py +++ b/core/downloads/staging.py @@ -34,9 +34,12 @@ from __future__ import annotations import logging import os +import re from dataclasses import dataclass from typing import Any, Callable +from core.imports.filename import extract_track_number_from_filename + # `shutil` and `SequenceMatcher` are imported inline inside try_staging_match() # to keep the lift byte-identical with the original web_server.py function body. @@ -50,6 +53,56 @@ from core.runtime_state import ( logger = logging.getLogger(__name__) +def _coerce_positive_int(value: Any, default: int = 0) -> int: + try: + coerced = int(str(value).split('/')[0]) + except (TypeError, ValueError): + return default + return coerced if coerced > 0 else default + + +def _staging_title_variants(title: Any, normalize: Callable[[str], str]) -> list[str]: + """Return conservative title variants for release-file matching. + + Torrent / usenet release files often encode featured artists in the + filename/title while streaming metadata keeps them in the artist credit. + Strip only feature/bonus noise here; keep version words like remix, + extended, live, acoustic, etc. so distinct recordings do not collapse. + """ + raw = str(title or '').strip() + if not raw: + return [] + + compacted_separators = re.sub(r'[_]+', ' ', raw) + compacted_separators = re.sub(r'\s+', ' ', compacted_separators).strip() + + without_feat = re.sub( + r'\s*[\(\[]\s*(?:feat\.?|ft\.?|featuring)\s+[^)\]]*[\)\]]', + '', + compacted_separators, + flags=re.IGNORECASE, + ) + without_feat = re.sub( + r'\s+(?:feat\.?|ft\.?|featuring)\s+.*$', + '', + without_feat, + flags=re.IGNORECASE, + ) + without_bonus = re.sub( + r'\s*[\(\[]\s*bonus\s+track\s*[\)\]]', + '', + without_feat, + flags=re.IGNORECASE, + ) + + variants: list[str] = [] + for candidate in (raw, compacted_separators, without_feat, without_bonus): + normalized = normalize(candidate) + if normalized and normalized not in variants: + variants.append(normalized) + return variants + + @dataclass class StagingDeps: """Bundle of cross-cutting deps the staging-match helper needs.""" @@ -86,19 +139,24 @@ def try_staging_match(task_id, batch_id, track, deps: StagingDeps): normalize = deps.matching_engine.normalize_string norm_title = normalize(track_title) norm_artist = normalize(track_artist) + title_variants = _staging_title_variants(track_title, normalize) or [norm_title] best_match = None best_score = 0.0 for sf in staging_files: - sf_norm_title = normalize(sf['title']) + sf_title_variants = _staging_title_variants(sf['title'], normalize) sf_norm_artist = normalize(sf['artist']) - if not sf_norm_title: + if not sf_title_variants: continue # Title similarity (primary) - title_sim = SequenceMatcher(None, norm_title, sf_norm_title).ratio() + title_sim = max( + SequenceMatcher(None, expected, candidate).ratio() + for expected in title_variants + for candidate in sf_title_variants + ) if title_sim < 0.80: continue @@ -190,6 +248,8 @@ def try_staging_match(task_id, batch_id, track, deps: StagingDeps): track_info = download_tasks.get(task_id, {}).get('track_info', {}) if not isinstance(track_info, dict): track_info = {} + else: + track_info = dict(track_info) # Build spotify_artist / spotify_album context so post-processing can apply # the path template. Without these, _post_process_matched_download returns @@ -258,20 +318,37 @@ def try_staging_match(task_id, batch_id, track, deps: StagingDeps): ) has_clean_data = bool(track_title and track_artist and track_album_name) - track_number = ( - track_info.get('track_number', 0) or - getattr(track, 'track_number', 0) or 0 - ) - disc_number = ( - track_info.get('disc_number', 1) or - getattr(track, 'disc_number', 1) or 1 + file_track_number = ( + _coerce_positive_int(best_match.get('track_number'), 0) or + extract_track_number_from_filename(best_match.get('full_path', '')) ) + file_disc_number = _coerce_positive_int(best_match.get('disc_number'), 1) + if _private_album_bundle_staging: + track_number = file_track_number + disc_number = file_disc_number + else: + track_number = ( + _coerce_positive_int(track_info.get('track_number'), 0) or + _coerce_positive_int(track_info.get('trackNumber'), 0) or + _coerce_positive_int(getattr(track, 'track_number', 0), 0) or + file_track_number + ) + disc_number = ( + _coerce_positive_int(track_info.get('disc_number'), 0) or + _coerce_positive_int(track_info.get('discNumber'), 0) or + _coerce_positive_int(getattr(track, 'disc_number', 0), 0) or + file_disc_number + ) + track_info['track_number'] = track_number + track_info['disc_number'] = disc_number context = { 'track_info': track_info, 'spotify_artist': spotify_artist_ctx, 'spotify_album': spotify_album_ctx, 'original_search_result': { + 'username': _provenance_username, + 'filename': best_match.get('full_path', ''), 'title': track_title, 'artist': track_artist, 'spotify_clean_title': track_title, diff --git a/core/downloads/status.py b/core/downloads/status.py index 6c9fac4f..6e11a763 100644 --- a/core/downloads/status.py +++ b/core/downloads/status.py @@ -88,7 +88,9 @@ class StatusDeps: # slskd's live_transfers path and must NOT hit the engine fallback. _STREAMING_SOURCE_NAMES = frozenset(( 'youtube', 'tidal', 'qobuz', 'hifi', 'deezer_dl', 'lidarr', 'soundcloud', 'amazon', + 'torrent', 'usenet', )) +_RELEASE_SOURCE_NAMES = frozenset(('torrent', 'usenet')) # Keep these in sync with the engine plugins' state strings. _ENGINE_FAILURE_STATES = ('Errored', 'Failed', 'Rejected', 'TimedOut', 'Aborted') @@ -136,21 +138,12 @@ def _apply_engine_state_fallback( Mutates ``task`` in place (status / error_message) the same way the Soulseek branch does, so the next status poll sees the new state. - Submits post-processing on terminal success and fires - ``on_download_completed`` on terminal failure to free the worker - slot. + Submits post-processing on terminal success. Manual-pick failures + are completed here; automatic failures keep the existing retry path + in charge. """ if deps.download_orchestrator is None or deps.run_async is None: return - if task.get('status') in ('completed', 'failed', 'cancelled', 'not_found', 'post_processing'): - return - # Scope this fallback to user-initiated manual picks. Auto attempts - # already flow through the live_transfers_lookup IF branch (the engine - # pre-populates non-Soulseek records via get_all_downloads), and on - # failure the monitor's existing retry path picks the next candidate. - # Marking auto attempts failed here would short-circuit that fallback. - if not task.get('_user_manual_pick'): - return download_id = task.get('download_id') if not download_id: return @@ -158,6 +151,17 @@ def _apply_engine_state_fallback( username = task.get('username') or ti.get('username') if username not in _STREAMING_SOURCE_NAMES: return + manual_pick = bool(task.get('_user_manual_pick')) + if not manual_pick and username not in _RELEASE_SOURCE_NAMES: + return + release_source = username in _RELEASE_SOURCE_NAMES + terminal_status = task.get('status') in ('completed', 'failed', 'cancelled', 'not_found', 'post_processing') + if terminal_status and not ( + release_source + and task.get('status') in ('failed', 'not_found') + and download_id + ): + return try: record = deps.run_async( @@ -189,6 +193,14 @@ def _apply_engine_state_fallback( return if any(s in state_str for s in _ENGINE_FAILURE_STATES): + if not manual_pick: + task_status['status'] = task.get('status', 'downloading') + task_status['progress'] = _engine_progress_pct(record) + logger.info( + "[Engine Fallback] Task %s engine reports '%s' for auto %s attempt; leaving retry handling to monitor", + task_id, state_str, username, + ) + return if task['status'] != 'failed': task['status'] = 'failed' err = getattr(record, 'error_message', None) or getattr(record, 'error', None) or '' @@ -321,6 +333,17 @@ def build_batch_status_data(batch_id: str, batch: dict, live_transfers_lookup: d _ti = task.get('track_info') if isinstance(task.get('track_info'), dict) else {} task_filename = task.get('filename') or _ti.get('filename') task_username = task.get('username') or _ti.get('username') + if ( + task_username in _RELEASE_SOURCE_NAMES + and task['status'] not in ['completed', 'cancelled', 'post_processing'] + and task.get('download_id') + ): + _apply_engine_state_fallback( + task_id, task, task_status, batch_id, deps, + ) + batch_tasks.append(task_status) + continue + if task_filename and task_username: lookup_key = deps.make_context_key(task_username, task_filename) diff --git a/core/downloads/task_worker.py b/core/downloads/task_worker.py index 8fc42608..67b2a3fe 100644 --- a/core/downloads/task_worker.py +++ b/core/downloads/task_worker.py @@ -25,12 +25,41 @@ import traceback from dataclasses import dataclass from typing import Any, Callable, Optional -from core.runtime_state import download_tasks, tasks_lock +from core.runtime_state import download_batches, download_tasks, tasks_lock from core.spotify_client import Track as SpotifyTrack logger = logging.getLogger(__name__) +def _private_album_bundle_staging_miss_reason(batch_id: Optional[str], deps: Any) -> Optional[str]: + """Return a user-facing miss reason when per-track search should stop. + + Torrent / usenet album batches first download one private staged release, + then each track claims the matching staged file. If that claim fails after + the release is already staged, falling through to the normal per-track + search only retries release-level sources N times and can keep re-adding + the same torrent. Treat the staged release as authoritative for this pass. + """ + if not batch_id: + return None + + batch = download_batches.get(batch_id) + if not isinstance(batch, dict): + return None + + source = (batch.get('album_bundle_source') or '').lower() + mode = (getattr(deps.download_orchestrator, 'mode', '') or '').lower() + if ( + batch.get('album_bundle_private_staging') + and batch.get('album_bundle_state') == 'staged' + and source in ('torrent', 'usenet') + and mode == source + ): + return f'Track was not found in the staged {source} album release' + + return None + + @dataclass class TaskWorkerDeps: """Bundle of cross-cutting deps the per-task download worker needs.""" @@ -128,6 +157,21 @@ def download_track_worker(task_id: str, batch_id: Optional[str], deps: TaskWorke # === STAGING CHECK: Check staging folder for existing file before searching === if deps.try_staging_match(task_id, batch_id, track): return + staging_miss_reason = _private_album_bundle_staging_miss_reason(batch_id, deps) + if staging_miss_reason: + logger.warning( + "[Modal Worker] %s for '%s'; skipping redundant per-track %s search", + staging_miss_reason, + track.name, + getattr(deps.download_orchestrator, 'mode', 'release-source'), + ) + with tasks_lock: + if task_id in download_tasks: + download_tasks[task_id]['status'] = 'not_found' + download_tasks[task_id]['error_message'] = staging_miss_reason + if batch_id: + deps.on_download_completed(batch_id, task_id, False) + return # Initialize task state tracking (like GUI's parallel_search_tracking) with tasks_lock: @@ -147,6 +191,27 @@ def download_track_worker(task_id: str, batch_id: Optional[str], deps: TaskWorke artist_name = track.artists[0] if track.artists else None track_name = track.name + release_queries = [] + try: + _download_mode = (getattr(deps.download_orchestrator, 'mode', '') or '').lower() + _track_album = (getattr(track, 'album', '') or '').strip() + _track_title = (getattr(track, 'name', '') or '').strip() + _track_artists = list(getattr(track, 'artists', []) or []) + _first_artist = _track_artists[0] if _track_artists else '' + _primary_artist = ( + (_first_artist.get('name', '') if isinstance(_first_artist, dict) else str(_first_artist)) + or '' + ).strip() + if ( + _download_mode in ('torrent', 'usenet') + and _primary_artist + and _track_album + and _track_album.lower() not in ('unknown album', _track_title.lower()) + ): + release_queries.append(f"{_primary_artist} {_track_album}".strip()) + except Exception as _release_query_exc: + logger.debug("[Modal Worker] release query hint failed: %s", _release_query_exc) + # Start with matching engine queries search_queries = deps.matching_engine.generate_download_queries(track) @@ -175,8 +240,14 @@ def download_track_worker(task_id: str, batch_id: Optional[str], deps: TaskWorke if cleaned_name and cleaned_name.lower() != track_name.lower(): legacy_queries.append(cleaned_name.strip()) - # Combine enhanced queries with legacy fallbacks - all_queries = search_queries + legacy_queries + # Combine enhanced queries with legacy fallbacks. + # + # Torrent / usenet can use full album releases as a fallback for + # single-track requests, but trying the album release first makes + # playlist batches download whole albums before checking whether a + # track-shaped release exists. Keep release queries last so singles + # stay light when the indexer has a direct result. + all_queries = search_queries + legacy_queries + release_queries # Remove duplicates while preserving order unique_queries = [] diff --git a/core/downloads/validation.py b/core/downloads/validation.py index 714ffb6c..339d0e8e 100644 --- a/core/downloads/validation.py +++ b/core/downloads/validation.py @@ -25,6 +25,20 @@ def init(matching_engine_obj, download_orchestrator_obj): download_orchestrator = download_orchestrator_obj +def _torrent_usenet_artist_is_fallback(result): + """True when a release result has no parsed artist, only indexer filler.""" + if getattr(result, 'username', None) not in ('torrent', 'usenet'): + return False + artist = (getattr(result, 'artist', None) or '').strip() + if not artist: + return True + metadata = getattr(result, '_source_metadata', None) or {} + indexer = str(metadata.get('indexer') or '').strip() + if artist.lower() in ('torrent', 'usenet'): + return True + return bool(indexer and artist.lower() == indexer.lower()) + + def filter_soundcloud_previews(results, expected_track): """Drop SoundCloud preview snippets so they never reach the cache, the modal, or the auto-download attempt. @@ -136,13 +150,18 @@ def get_valid_candidates(results, spotify_track, query): ) continue - # Score using matching engine's generic scorer (same weights as Soulseek) + # Score using matching engine's generic scorer (same weights as Soulseek). + # Torrent/usenet release projections sometimes only have the indexer name + # in the artist field when a title did not parse as "Artist - Release". + # Treat that as unknown artist, not as a real mismatch. + has_only_fallback_artist = _torrent_usenet_artist_is_fallback(r) + candidate_artists = [] if has_only_fallback_artist else ([r.artist] if r.artist else []) confidence, match_type = matching_engine.score_track_match( source_title=expected_title, source_artists=expected_artists, source_duration_ms=expected_duration, candidate_title=r.title or '', - candidate_artists=[r.artist] if r.artist else [], + candidate_artists=candidate_artists, candidate_duration_ms=r.duration or 0, ) @@ -166,13 +185,14 @@ def get_valid_candidates(results, spotify_track, query): # wanted track's ALBUM name and taking the max gives album- # level releases a fair shot. The Auto-Import sweep then picks # the right file out of the downloaded album folder. - if r.username in ('torrent', 'usenet') and spotify_track and spotify_track.album: + expected_album = getattr(spotify_track, 'album', None) if spotify_track else None + if r.username in ('torrent', 'usenet') and expected_album: album_conf, _ = matching_engine.score_track_match( - source_title=spotify_track.album, + source_title=expected_album, source_artists=expected_artists, source_duration_ms=0, # albums don't have one duration candidate_title=r.title or '', - candidate_artists=[r.artist] if r.artist else [], + candidate_artists=candidate_artists, candidate_duration_ms=0, ) if album_conf > confidence: @@ -199,11 +219,10 @@ def get_valid_candidates(results, spotify_track, query): # Artist gate — streaming APIs (Tidal/Qobuz/HiFi/Deezer) have reliable metadata, # so "My Will" by "B. Starr" should never match expected "B小町". - # Skip for YouTube (video-title parsing is unreliable) and torrent/usenet - # (album-level releases legitimately don't expose per-track artist — - # the projection layer fills artist with the indexer name as a fallback, - # which would otherwise fail the gate against every Spotify artist). - if r.username not in ('youtube', 'torrent', 'usenet'): + # YouTube stays excluded because video-title parsing is unreliable. + # Torrent/usenet must also pass this gate so title-only matches + # from the wrong artist do not get downloaded. + if r.username != 'youtube' and not has_only_fallback_artist: from difflib import SequenceMatcher import re as _re _cand_artist_raw = r.artist or '' @@ -235,6 +254,16 @@ def get_valid_candidates(results, spotify_track, query): # so falling to SequenceMatcher means the strings are genuinely # different. 0.5 gives a safer buffer without blocking real # matches that would have scored above 0.85 anyway. + if r.username in ('torrent', 'usenet') and _best_artist < 0.5: + logger.info( + "[%s] Rejecting candidate due to artist mismatch: " + "expected=%s candidate=%r title=%r", + source_label, + list(expected_artists), + _cand_artist_raw, + r.title or '', + ) + continue if _best_artist < 0.5 and confidence < 0.85: continue diff --git a/core/imports/side_effects.py b/core/imports/side_effects.py index d1580564..6513aabf 100644 --- a/core/imports/side_effects.py +++ b/core/imports/side_effects.py @@ -198,6 +198,10 @@ def record_library_history_download(context: Dict[str, Any]) -> None: "deezer_dl": "Deezer", "lidarr": "Lidarr", "soundcloud": "SoundCloud", + "amazon": "Amazon", + "staging": "Staging", + "torrent": "Torrent", + "usenet": "Usenet", # Auto-import isn't a download source, but flows through the # same post-process pipeline (file lands → record provenance # + history → write to library DB). Tagging it as "Auto-Import" diff --git a/tests/downloads/test_downloads_lifecycle.py b/tests/downloads/test_downloads_lifecycle.py index fb67d157..8be74876 100644 --- a/tests/downloads/test_downloads_lifecycle.py +++ b/tests/downloads/test_downloads_lifecycle.py @@ -329,6 +329,48 @@ def test_batch_completion_emits_batch_complete_when_all_done(): assert 'b1' in monitor.stopped +def test_batch_completion_cleans_private_album_bundle_staging(tmp_path): + staging_dir = tmp_path / 'b1' + staging_dir.mkdir() + (staging_dir / 'leftover.flac').write_bytes(b'audio') + + download_tasks['t1'] = {'status': 'completed', 'track_info': {'name': 'X'}} + download_batches['b1'] = { + 'queue': ['t1'], 'queue_index': 1, 'active_count': 1, + 'max_concurrent': 1, 'permanently_failed_tracks': [], + 'cancelled_tracks': set(), + 'album_bundle_private_staging': True, + 'album_bundle_source': 'torrent', + 'album_bundle_staging_path': str(staging_dir), + } + deps, _ = _build_deps() + + lc.on_download_completed('b1', 't1', True, deps) + + assert not staging_dir.exists() + + +def test_batch_completion_keeps_unexpected_staging_path(tmp_path): + staging_dir = tmp_path / 'shared-staging' + staging_dir.mkdir() + (staging_dir / 'leftover.flac').write_bytes(b'audio') + + download_tasks['t1'] = {'status': 'completed', 'track_info': {'name': 'X'}} + download_batches['b1'] = { + 'queue': ['t1'], 'queue_index': 1, 'active_count': 1, + 'max_concurrent': 1, 'permanently_failed_tracks': [], + 'cancelled_tracks': set(), + 'album_bundle_private_staging': True, + 'album_bundle_source': 'torrent', + 'album_bundle_staging_path': str(staging_dir), + } + deps, _ = _build_deps() + + lc.on_download_completed('b1', 't1', True, deps) + + assert staging_dir.exists() + + def test_batch_completion_skips_emit_when_zero_successful(): """Don't emit batch_complete if nothing actually downloaded.""" download_tasks['t1'] = {'status': 'failed', 'track_info': {'name': 'X'}, 'track_index': 0} diff --git a/tests/downloads/test_downloads_post_processing.py b/tests/downloads/test_downloads_post_processing.py index a5f02d8c..455b2237 100644 --- a/tests/downloads/test_downloads_post_processing.py +++ b/tests/downloads/test_downloads_post_processing.py @@ -232,6 +232,28 @@ def test_file_found_in_downloads_with_context_runs_post_process_with_verificatio assert any(c[0] == 'post_process' for c in rec.calls) +def test_file_search_ignores_non_audio_candidates(monkeypatch): + download_tasks['t1'] = { + 'status': 'post_processing', + 'filename': 'Artist - Album.cue', + 'username': 'torrent', + 'track_info': {'name': 'Money'}, + } + matched_downloads_context['torrent::Artist - Album.cue'] = { + 'original_search_result': {'title': 'Money', 'track_number': 1}, + } + monkeypatch.setattr(pp.time, 'sleep', lambda s: None) + deps, rec = _build_deps( + find_completed_file=lambda *a, **kw: ('/downloads/Artist - Album.cue', 'download'), + ) + + pp.run_post_processing_worker('t1', 'b1', deps) + + assert download_tasks['t1']['status'] == 'failed' + assert not any(c[0] == 'post_process' for c in rec.calls) + assert ('on_complete', ('b1', 't1', False), {}) in rec.calls + + def test_file_found_in_downloads_no_context_marks_completed_directly(): """No matched context for the file → just mark completed since file exists.""" download_tasks['t1'] = { @@ -323,6 +345,139 @@ def test_youtube_task_uses_get_download_status_to_resolve_path(monkeypatch): assert any(c[0] == 'mark_completed' for c in rec.calls) +def test_torrent_release_copies_best_matching_audio_to_transfer(tmp_path): + release_dir = tmp_path / 'release' + release_dir.mkdir() + wrong = release_dir / '01 - Intro.flac' + right = release_dir / '02 - Money.flac' + wrong.write_bytes(b'wrong') + right.write_bytes(b'right') + transfer_dir = tmp_path / 'transfer' + + filename = 'magnet:?xt=abc||Artist - Album' + download_tasks['t1'] = { + 'status': 'post_processing', + 'filename': filename, + 'username': 'torrent', + 'download_id': 'dl-torrent-1', + 'track_info': {'name': 'Money', 'artists': [{'name': 'Artist'}]}, + } + matched_downloads_context[f'torrent::{filename}'] = { + 'original_search_result': {'title': 'Money', 'track_number': 2}, + } + + class _FakeStatus: + file_path = str(wrong) + audio_files = [str(wrong), str(right)] + + class _FakeTorrentClient: + def get_download_status(self, dl_id): + assert dl_id == 'dl-torrent-1' + return _FakeStatus() + + deps, rec = _build_deps( + config=_FakeConfig({'soulseek.transfer_path': str(transfer_dir)}), + download_orchestrator=_FakeTorrentClient(), + run_async=lambda coro: coro, + ) + + pp.run_post_processing_worker('t1', 'b1', deps) + + copied = transfer_dir / '02 - Money.flac' + assert copied.exists() + assert right.exists() + assert any(c[0] == 'post_process' and c[1][2] == str(copied) for c in rec.calls) + + +def test_torrent_release_prefers_task_title_over_release_context(tmp_path): + release_dir = tmp_path / 'release' + release_dir.mkdir() + wrong = release_dir / '09.Harry Styles - Pop.flac' + right = release_dir / '10.Harry Styles - American Girls.flac' + wrong.write_bytes(b'wrong') + right.write_bytes(b'right') + transfer_dir = tmp_path / 'transfer' + + filename = 'http://prowlarr/download?id=1||Harry Styles - Kiss All The Time' + download_tasks['t1'] = { + 'status': 'post_processing', + 'filename': filename, + 'username': 'torrent', + 'download_id': 'dl-torrent-1', + 'track_info': {'name': 'American Girls', 'artists': [{'name': 'Harry Styles'}]}, + } + matched_downloads_context[f'torrent::{filename}'] = { + 'original_search_result': {'title': 'Pop', 'clean_title': 'Pop', 'track_number': 9}, + } + + class _FakeStatus: + file_path = str(wrong) + audio_files = [str(wrong), str(right)] + + class _FakeTorrentClient: + def get_download_status(self, dl_id): + assert dl_id == 'dl-torrent-1' + return _FakeStatus() + + deps, rec = _build_deps( + config=_FakeConfig({'soulseek.transfer_path': str(transfer_dir)}), + download_orchestrator=_FakeTorrentClient(), + run_async=lambda coro: coro, + ) + + pp.run_post_processing_worker('t1', 'b1', deps) + + copied = transfer_dir / '10.Harry Styles - American Girls.flac' + assert copied.exists() + assert any(c[0] == 'post_process' and c[1][2] == str(copied) for c in rec.calls) + + +def test_torrent_release_without_matching_file_does_not_fallback_to_generic_search(tmp_path): + release_dir = tmp_path / 'release' + release_dir.mkdir() + wrong = release_dir / '09.Harry Styles - Pop.flac' + wrong.write_bytes(b'wrong') + transfer_dir = tmp_path / 'transfer' + + filename = 'http://prowlarr/download?id=1||Harry Styles - Kiss All The Time' + download_tasks['t1'] = { + 'status': 'post_processing', + 'filename': filename, + 'username': 'torrent', + 'download_id': 'dl-torrent-1', + 'track_info': {'name': 'American Girls', 'artists': [{'name': 'Harry Styles'}]}, + } + matched_downloads_context[f'torrent::{filename}'] = { + 'original_search_result': {'title': 'Pop', 'clean_title': 'Pop', 'track_number': 9}, + } + + class _FakeStatus: + file_path = str(wrong) + audio_files = [str(wrong)] + + class _FakeTorrentClient: + def get_download_status(self, dl_id): + assert dl_id == 'dl-torrent-1' + return _FakeStatus() + + def _unexpected_search(*args, **kwargs): + raise AssertionError("torrent releases should not fall back to generic file search") + + deps, rec = _build_deps( + config=_FakeConfig({'soulseek.transfer_path': str(transfer_dir)}), + download_orchestrator=_FakeTorrentClient(), + run_async=lambda coro: coro, + find_completed_file=_unexpected_search, + ) + + pp.run_post_processing_worker('t1', 'b1', deps) + + assert download_tasks['t1']['status'] == 'failed' + assert 'No matching audio file' in download_tasks['t1']['error_message'] + assert any(c[0] == 'on_complete' and c[1] == ('b1', 't1', False) for c in rec.calls) + assert not list(transfer_dir.glob('*')) + + def test_fuzzy_context_matching_when_exact_key_missing(monkeypatch): """When exact key isn't in matched_downloads_context, worker tries fuzzy match constrained to same Soulseek username.""" diff --git a/tests/downloads/test_downloads_staging.py b/tests/downloads/test_downloads_staging.py index e75e633c..70e86b6b 100644 --- a/tests/downloads/test_downloads_staging.py +++ b/tests/downloads/test_downloads_staging.py @@ -278,6 +278,180 @@ def test_explicit_album_context_uses_real_data(tmp_path): assert ctx['staging_source'] is True +def test_staging_context_falls_back_to_matched_file_track_number(tmp_path): + """Album-bundle staging can recover numbering from the selected audio file.""" + src_file = tmp_path / 'staging' / '03 - Backseat Freestyle.flac' + src_file.parent.mkdir() + src_file.touch() + + deps = _build_deps( + transfer_path=str(tmp_path / 'transfer'), + staging_files=[ + { + 'full_path': str(src_file), + 'title': 'Backseat Freestyle', + 'artist': 'Kendrick Lamar', + 'track_number': 3, + 'disc_number': 1, + }, + ], + ) + _seed_task('t6b', track_info={ + '_is_explicit_album_download': True, + '_explicit_album_context': {'id': 'alb', 'name': 'good kid, m.A.A.d city (Deluxe)'}, + '_explicit_artist_context': {'id': 'art', 'name': 'Kendrick Lamar'}, + }) + + ds.try_staging_match( + 't6b', 'b1', + _Track(name='Backseat Freestyle', artists=['Kendrick Lamar']), + deps, + ) + + ctx = matched_downloads_context['staging_t6b'] + assert ctx['original_search_result']['track_number'] == 3 + assert ctx['original_search_result']['disc_number'] == 1 + + +def test_private_album_bundle_staging_overrides_default_track_info_number(tmp_path): + """Private release staging trusts the selected file number over weak task defaults.""" + src_file = tmp_path / 'staging' / '04-kendrick_lamar-the_art_of_peer_pressure.flac' + src_file.parent.mkdir() + src_file.touch() + + def get_batch_field(_batch_id, field): + if field == 'album_bundle_source': + return 'torrent' + if field == 'album_bundle_private_staging': + return True + return None + + deps = _build_deps( + transfer_path=str(tmp_path / 'transfer'), + staging_files=[ + { + 'full_path': str(src_file), + 'title': 'The Art of Peer Pressure', + 'artist': 'Kendrick Lamar', + }, + ], + get_batch_field=get_batch_field, + ) + _seed_task('t6c', track_info={ + '_is_explicit_album_download': True, + '_explicit_album_context': {'id': 'alb', 'name': 'good kid, m.A.A.d city (Deluxe)'}, + '_explicit_artist_context': {'id': 'art', 'name': 'Kendrick Lamar'}, + 'track_number': 1, + }) + + ds.try_staging_match( + 't6c', 'b1', + _Track(name='The Art of Peer Pressure', artists=['Kendrick Lamar']), + deps, + ) + + ctx = matched_downloads_context['staging_t6c'] + assert ctx['track_info']['track_number'] == 4 + assert ctx['original_search_result']['track_number'] == 4 + assert ctx['original_search_result']['username'] == 'torrent' + assert ctx['original_search_result']['filename'] == str(src_file) + + +def test_staging_title_match_accepts_feature_suffix_from_release_file(tmp_path): + """Album releases can include featured artists in filenames.""" + src_file = tmp_path / 'staging' / '05-kendrick_lamar-money_trees_(feat._jay_rock).flac' + src_file.parent.mkdir() + src_file.touch() + + deps = _build_deps( + transfer_path=str(tmp_path / 'transfer'), + staging_files=[ + { + 'full_path': str(src_file), + 'title': 'money_trees_(feat._jay_rock)', + 'artist': 'Kendrick Lamar', + 'track_number': 5, + }, + ], + ) + _seed_task('t_feature', track_info={ + '_is_explicit_album_download': True, + '_explicit_album_context': {'id': 'alb', 'name': 'good kid, m.A.A.d city (Deluxe)'}, + '_explicit_artist_context': {'id': 'art', 'name': 'Kendrick Lamar'}, + }) + + result = ds.try_staging_match( + 't_feature', 'b1', + _Track(name='Money Trees', artists=['Kendrick Lamar']), + deps, + ) + + assert result is True + assert matched_downloads_context['staging_t_feature']['track_info']['track_number'] == 5 + + +def test_staging_title_match_accepts_bonus_track_against_release_file(tmp_path): + """Expected bonus labels should not block matching the actual release file.""" + src_file = tmp_path / 'staging' / '13-kendrick_lamar-the_recipe_(feat._dr._dre).flac' + src_file.parent.mkdir() + src_file.touch() + + deps = _build_deps( + transfer_path=str(tmp_path / 'transfer'), + staging_files=[ + { + 'full_path': str(src_file), + 'title': 'the_recipe_(feat._dr._dre)', + 'artist': 'Kendrick Lamar', + 'track_number': 13, + }, + ], + ) + _seed_task('t_bonus', track_info={ + '_is_explicit_album_download': True, + '_explicit_album_context': {'id': 'alb', 'name': 'good kid, m.A.A.d city (Deluxe)'}, + '_explicit_artist_context': {'id': 'art', 'name': 'Kendrick Lamar'}, + }) + + result = ds.try_staging_match( + 't_bonus', 'b1', + _Track(name='The Recipe (Bonus Track)', artists=['Kendrick Lamar']), + deps, + ) + + assert result is True + assert matched_downloads_context['staging_t_bonus']['track_info']['track_number'] == 13 + + +def test_staging_title_match_keeps_wrong_versions_separate(tmp_path): + """Do not strip remix/extended wording when matching staged release files.""" + src_file = tmp_path / 'staging' / '17-kendrick_lamar-swimming_pools_(drank)_(black_hippy_remix).flac' + src_file.parent.mkdir() + src_file.touch() + + deps = _build_deps( + transfer_path=str(tmp_path / 'transfer'), + staging_files=[ + { + 'full_path': str(src_file), + 'title': 'swimming_pools_(drank)_(black_hippy_remix)', + 'artist': 'Kendrick Lamar', + 'track_number': 17, + }, + ], + ) + _seed_task('t_wrong_version') + + result = ds.try_staging_match( + 't_wrong_version', 'b1', + _Track(name='Swimming Pools (Drank) (Extended Version)', artists=['Kendrick Lamar']), + deps, + ) + + assert result is False + assert 'staging_t_wrong_version' not in matched_downloads_context + + def test_fallback_context_synthesizes_from_track(tmp_path): """Without explicit context, synthesizes spotify_artist/album from the track.""" src_file = tmp_path / 'staging' / 'Hello.flac' diff --git a/tests/downloads/test_downloads_status.py b/tests/downloads/test_downloads_status.py index cc9a5d15..706f9131 100644 --- a/tests/downloads/test_downloads_status.py +++ b/tests/downloads/test_downloads_status.py @@ -40,6 +40,8 @@ def _build_deps( make_key=None, submit_pp=None, cached_transfers=None, + download_orchestrator=None, + run_async=None, ): submitted = [] @@ -53,6 +55,8 @@ def _build_deps( make_context_key=make_key or (lambda u, f: f"{u}::{f}"), submit_post_processing=submit_pp or _default_submit, get_cached_transfer_data=cached_transfers or (lambda: {}), + download_orchestrator=download_orchestrator, + run_async=run_async, ) return deps, submitted @@ -286,6 +290,128 @@ def test_post_processing_status_progress_is_95(): assert out['tasks'][0]['progress'] == 95 +def test_auto_torrent_without_live_entry_uses_engine_success_fallback(): + class _Record: + state = 'Completed, Succeeded' + progress = 100 + + class _Orchestrator: + def get_download_status(self, download_id): + assert download_id == 'dl1' + return _Record() + + deps, submitted = _build_deps( + download_orchestrator=_Orchestrator(), + run_async=lambda value: value, + ) + download_tasks['t1'] = { + 'track_index': 0, + 'status': 'downloading', + 'track_info': {}, + 'filename': 'song.flac', + 'username': 'torrent', + 'download_id': 'dl1', + } + batch = {'phase': 'downloading', 'queue': ['t1']} + out = st.build_batch_status_data('b1', batch, {}, deps) + assert out['tasks'][0]['status'] == 'post_processing' + assert download_tasks['t1']['status'] == 'post_processing' + assert submitted == [('t1', 'b1')] + + +def test_auto_torrent_prefers_engine_success_over_live_entry(): + class _Record: + state = 'Completed, Succeeded' + progress = 100 + + class _Orchestrator: + def get_download_status(self, download_id): + assert download_id == 'dl1' + return _Record() + + deps, submitted = _build_deps( + download_orchestrator=_Orchestrator(), + run_async=lambda value: value, + ) + download_tasks['t1'] = { + 'track_index': 0, + 'status': 'downloading', + 'track_info': {}, + 'filename': 'song.flac', + 'username': 'torrent', + 'download_id': 'dl1', + } + live = {'torrent::song.flac': { + 'state': 'InProgress, Downloading', + 'percentComplete': 100, + }} + batch = {'phase': 'downloading', 'queue': ['t1']} + out = st.build_batch_status_data('b1', batch, live, deps) + assert out['tasks'][0]['status'] == 'post_processing' + assert download_tasks['t1']['status'] == 'post_processing' + assert submitted == [('t1', 'b1')] + + +def test_auto_torrent_engine_failure_does_not_bypass_monitor_retry(): + class _Record: + state = 'Completed, Errored' + progress = 100 + error = 'client failed' + + class _Orchestrator: + def get_download_status(self, download_id): + assert download_id == 'dl1' + return _Record() + + deps, submitted = _build_deps( + download_orchestrator=_Orchestrator(), + run_async=lambda value: value, + ) + download_tasks['t1'] = { + 'track_index': 0, + 'status': 'downloading', + 'track_info': {}, + 'filename': 'song.flac', + 'username': 'torrent', + 'download_id': 'dl1', + } + batch = {'phase': 'downloading', 'queue': ['t1']} + out = st.build_batch_status_data('b1', batch, {}, deps) + assert out['tasks'][0]['status'] == 'downloading' + assert download_tasks['t1']['status'] == 'downloading' + assert submitted == [] + + +def test_auto_torrent_engine_success_recovers_premature_failed_task(): + class _Record: + state = 'Completed, Succeeded' + progress = 100 + + class _Orchestrator: + def get_download_status(self, download_id): + assert download_id == 'dl1' + return _Record() + + deps, submitted = _build_deps( + download_orchestrator=_Orchestrator(), + run_async=lambda value: value, + ) + download_tasks['t1'] = { + 'track_index': 0, + 'status': 'failed', + 'track_info': {}, + 'filename': 'release-url||Release', + 'username': 'torrent', + 'download_id': 'dl1', + 'error_message': 'premature failure', + } + batch = {'phase': 'downloading', 'queue': ['t1']} + out = st.build_batch_status_data('b1', batch, {}, deps) + assert out['tasks'][0]['status'] == 'post_processing' + assert download_tasks['t1']['status'] == 'post_processing' + assert submitted == [('t1', 'b1')] + + # --------------------------------------------------------------------------- # Safety valve (stuck task handling) # --------------------------------------------------------------------------- diff --git a/tests/downloads/test_downloads_task_worker.py b/tests/downloads/test_downloads_task_worker.py index 11ccdfca..ea605d6e 100644 --- a/tests/downloads/test_downloads_task_worker.py +++ b/tests/downloads/test_downloads_task_worker.py @@ -5,14 +5,16 @@ from __future__ import annotations import pytest from core.downloads import task_worker as tw -from core.runtime_state import download_tasks +from core.runtime_state import download_batches, download_tasks @pytest.fixture(autouse=True) def reset_state(): download_tasks.clear() + download_batches.clear() yield download_tasks.clear() + download_batches.clear() # --------------------------------------------------------------------------- @@ -37,7 +39,7 @@ class _FakeClient: orchestrator); plain attrs (hybrid_order, hybrid_primary, etc.) are set as attributes so getattr() lookups still resolve them.""" _CLIENT_NAMES = {'soulseek', 'youtube', 'tidal', 'qobuz', 'hifi', - 'deezer_dl', 'lidarr', 'soundcloud'} + 'deezer_dl', 'lidarr', 'soundcloud', 'torrent', 'usenet'} def __init__(self, results=None, mode='soulseek', subclients=None): self._results = results if results is not None else [] @@ -54,7 +56,7 @@ class _FakeClient: def client(self, name): return self._client_map.get(name) - async def search(self, query, timeout=30): + async def search(self, query, timeout=30, exclude_sources=None): self.search_calls.append((query, timeout)) return (self._results, None) @@ -194,6 +196,33 @@ def test_staging_match_hit_returns_immediately(): assert rec.calls == [] +def test_private_torrent_album_staging_miss_skips_per_track_search(): + _seed_task(track_info={ + 'id': 'sp-1', 'name': 'Money Trees', 'artists': ['Kendrick Lamar'], + 'album': 'good kid, m.A.A.d city (Deluxe)', 'duration_ms': 387000, + }) + download_batches['b1'] = { + 'album_bundle_private_staging': True, + 'album_bundle_state': 'staged', + 'album_bundle_source': 'torrent', + } + client = _FakeClient(results=['should-not-search'], mode='torrent') + rec = _Recorder() + deps, _ = _build_deps( + soulseek=client, + matching=_FakeMatchEngine(queries=['Kendrick Lamar Money Trees']), + try_staging_match=lambda *a, **kw: False, + on_download_completed=rec('done'), + ) + + tw.download_track_worker('t1', 'b1', deps) + + assert client.search_calls == [] + assert download_tasks['t1']['status'] == 'not_found' + assert 'staged torrent album release' in download_tasks['t1']['error_message'] + assert ('done', ('b1', 't1', False), {}) in rec.calls + + # --------------------------------------------------------------------------- # Search loop happy path # --------------------------------------------------------------------------- @@ -219,6 +248,25 @@ def test_first_query_success_returns_after_storing_source(): assert download_tasks['t1']['status'] == 'searching' +def test_torrent_mode_uses_album_release_after_track_queries(): + _seed_task(track_info={ + 'id': 'sp-1', 'name': 'Money', 'artists': ['Pink Floyd'], + 'album': 'The Dark Side of the Moon', 'duration_ms': 383000, + }) + client = _FakeClient(results=[], mode='torrent') + rec = _Recorder() + deps, _ = _build_deps( + soulseek=client, + matching=_FakeMatchEngine(queries=['Pink Floyd Money']), + on_download_completed=rec('done'), + ) + + tw.download_track_worker('t1', 'b1', deps) + + assert client.search_calls[0][0] == 'Pink Floyd Money' + assert client.search_calls[-1][0] == 'Pink Floyd The Dark Side of the Moon' + + def test_no_results_marks_not_found_and_calls_completion(): _seed_task() rec = _Recorder() diff --git a/tests/downloads/test_downloads_validation.py b/tests/downloads/test_downloads_validation.py index ddb5346a..1acdf91c 100644 --- a/tests/downloads/test_downloads_validation.py +++ b/tests/downloads/test_downloads_validation.py @@ -123,3 +123,47 @@ def test_keeps_tidal_candidate_inside_integrity_duration_tolerance(monkeypatch): result = get_valid_candidates([tidal], expected, 'Artist Song') assert result == [tidal] + + +def test_rejects_torrent_title_match_from_wrong_artist(monkeypatch): + monkeypatch.setattr(validation, 'matching_engine', _MatchingEngine()) + expected = _Track(duration_ms=180_000, name='The Man I Need', artists=('Olivia Dean',)) + wrong_artist = _Candidate( + username='torrent', + duration=None, + title='The Man I Need', + artist='Tinkabelle', + ) + + assert get_valid_candidates([wrong_artist], expected, 'Olivia Dean The Man I Need') == [] + + +def test_keeps_torrent_title_match_from_expected_artist(monkeypatch): + monkeypatch.setattr(validation, 'matching_engine', _MatchingEngine()) + expected = _Track(duration_ms=180_000, name='The Man I Need', artists=('Olivia Dean',)) + correct_artist = _Candidate( + username='torrent', + duration=None, + title='The Man I Need', + artist='Olivia Dean', + ) + + result = get_valid_candidates([correct_artist], expected, 'Olivia Dean The Man I Need') + + assert result == [correct_artist] + + +def test_keeps_torrent_title_match_when_artist_is_indexer_fallback(monkeypatch): + monkeypatch.setattr(validation, 'matching_engine', _MatchingEngine()) + expected = _Track(duration_ms=180_000, name='The Man I Need', artists=('Olivia Dean',)) + candidate = _Candidate( + username='torrent', + duration=None, + title='The Man I Need', + artist='Indexer', + ) + candidate._source_metadata = {'indexer': 'Indexer'} + + result = get_valid_candidates([candidate], expected, 'Olivia Dean The Man I Need') + + assert result == [candidate] diff --git a/tests/imports/test_import_side_effects.py b/tests/imports/test_import_side_effects.py index 455d2d4f..136ee765 100644 --- a/tests/imports/test_import_side_effects.py +++ b/tests/imports/test_import_side_effects.py @@ -2,6 +2,8 @@ import os import sqlite3 from types import SimpleNamespace +import pytest + from core.imports import side_effects @@ -364,6 +366,42 @@ def test_library_history_labels_auto_import(monkeypatch): assert captured["title"] == "Auto-Imported Track" +@pytest.mark.parametrize( + ("username", "expected"), + [ + ("torrent", "Torrent"), + ("usenet", "Usenet"), + ("staging", "Staging"), + ], +) +def test_library_history_labels_release_and_staging_sources(monkeypatch, username, expected): + """Release/staging imports should not fall through to the Soulseek label.""" + captured = {} + + class _DBStub: + def add_library_history_entry(self, **kwargs): + captured.update(kwargs) + + monkeypatch.setattr(side_effects, "get_database", lambda: _DBStub()) + + context = { + "track_info": { + "name": "Imported Track", + "artists": [{"name": "Some Artist"}], + "album": "Some Album", + }, + "original_search_result": { + "username": username, + "filename": "source-file.flac", + }, + "_final_processed_path": "/library/some-album/01.flac", + } + + side_effects.record_library_history_download(context) + + assert captured["download_source"] == expected + + # --------------------------------------------------------------------------- # Album duration parity — must equal sum of all track durations, not whatever # the first imported track happened to be. diff --git a/tests/test_manual_pick_no_auto_retry.py b/tests/test_manual_pick_no_auto_retry.py index efa88359..9c70f0f8 100644 --- a/tests/test_manual_pick_no_auto_retry.py +++ b/tests/test_manual_pick_no_auto_retry.py @@ -165,3 +165,132 @@ def test_monitor_waits_for_post_processing_before_batch_success(monkeypatch): dm.download_batches.update(previous_batches) +def test_monitor_matches_release_download_by_id_when_filename_changes(monkeypatch): + """Torrent/usenet rows can expose the completed audio filename, not + the original indexer URL/title stored on the task. The monitor must + still claim the completed release by stable download_id. + """ + monkeypatch.setattr(dm, '_make_context_key', lambda u, f: f"{u}::{f}") + monkeypatch.setattr(dm.WebUIDownloadMonitor, '_validate_worker_counts', lambda self: None) + + submitted = [] + + class FakeExecutor: + def submit(self, func, task_id, batch_id): + submitted.append((func, task_id, batch_id)) + + def fake_post_processing_worker(task_id, batch_id): + return None + + monkeypatch.setattr(dm, 'missing_download_executor', FakeExecutor()) + monkeypatch.setattr(dm, '_run_post_processing_worker', fake_post_processing_worker) + monkeypatch.setattr(dm, '_on_download_completed', lambda *args: None) + + with dm.tasks_lock: + previous_tasks = dict(dm.download_tasks) + previous_batches = dict(dm.download_batches) + dm.download_tasks.clear() + dm.download_batches.clear() + dm.download_tasks['task-1'] = { + 'track_info': {'name': 'Ran To Atlanta'}, + 'username': 'torrent', + 'filename': 'http://prowlarr/download?id=123||Drake - ICEMAN', + 'status': 'downloading', + 'download_id': 'torrent-1', + 'status_change_time': time.time(), + } + dm.download_batches['batch-1'] = {'queue': ['task-1']} + + try: + monitor = dm.WebUIDownloadMonitor() + monitor.monitoring = True + monitor.monitored_batches.add('batch-1') + monkeypatch.setattr( + monitor, + '_get_live_transfers', + lambda: { + 'download_id::torrent-1': { + 'id': 'torrent-1', + 'username': 'torrent', + 'filename': '01. Drake - Make Them Cry.flac', + 'state': 'Completed, Succeeded', + 'size': 100, + 'bytesTransferred': 100, + } + }, + ) + + monitor._check_all_downloads() + + assert submitted == [(fake_post_processing_worker, 'task-1', 'batch-1')] + assert dm.download_tasks['task-1']['status'] == 'post_processing' + finally: + with dm.tasks_lock: + dm.download_tasks.clear() + dm.download_tasks.update(previous_tasks) + dm.download_batches.clear() + dm.download_batches.update(previous_batches) + + +def test_monitor_recovers_premature_failed_release_download(monkeypatch): + monkeypatch.setattr(dm, '_make_context_key', lambda u, f: f"{u}::{f}") + monkeypatch.setattr(dm.WebUIDownloadMonitor, '_validate_worker_counts', lambda self: None) + + submitted = [] + + class FakeExecutor: + def submit(self, func, task_id, batch_id): + submitted.append((func, task_id, batch_id)) + + def fake_post_processing_worker(task_id, batch_id): + return None + + monkeypatch.setattr(dm, 'missing_download_executor', FakeExecutor()) + monkeypatch.setattr(dm, '_run_post_processing_worker', fake_post_processing_worker) + monkeypatch.setattr(dm, '_on_download_completed', lambda *args: None) + + with dm.tasks_lock: + previous_tasks = dict(dm.download_tasks) + previous_batches = dict(dm.download_batches) + dm.download_tasks.clear() + dm.download_batches.clear() + dm.download_tasks['task-1'] = { + 'track_info': {'name': 'DAISIES'}, + 'username': 'torrent', + 'filename': 'http://prowlarr/download?id=123||Justin Bieber - Swag', + 'status': 'failed', + 'download_id': 'torrent-1', + 'status_change_time': time.time(), + } + dm.download_batches['batch-1'] = {'queue': ['task-1']} + + try: + monitor = dm.WebUIDownloadMonitor() + monitor.monitoring = True + monitor.monitored_batches.add('batch-1') + monkeypatch.setattr( + monitor, + '_get_live_transfers', + lambda: { + 'download_id::torrent-1': { + 'id': 'torrent-1', + 'username': 'torrent', + 'filename': '02. Justin Bieber - DAISIES.flac', + 'state': 'Completed, Succeeded', + 'size': 100, + 'bytesTransferred': 100, + } + }, + ) + + monitor._check_all_downloads() + + assert submitted == [(fake_post_processing_worker, 'task-1', 'batch-1')] + assert dm.download_tasks['task-1']['status'] == 'post_processing' + finally: + with dm.tasks_lock: + dm.download_tasks.clear() + dm.download_tasks.update(previous_tasks) + dm.download_batches.clear() + dm.download_batches.update(previous_batches) + diff --git a/web_server.py b/web_server.py index 0e7e2d8e..d6269314 100644 --- a/web_server.py +++ b/web_server.py @@ -2115,10 +2115,15 @@ def get_status(): _status_cache_timestamps['media_server'] = current_time # else: use cached value - # Test Soulseek - only if it's the active source or in the hybrid order - if current_time - _status_cache_timestamps['soulseek'] > STATUS_CACHE_TTL: - download_mode = config_manager.get('download_source.mode', 'hybrid') - hybrid_order = config_manager.get('download_source.hybrid_order', ['hifi', 'youtube', 'soulseek']) + download_mode = config_manager.get('download_source.mode', 'hybrid') + hybrid_order = config_manager.get('download_source.hybrid_order', ['hifi', 'youtube', 'soulseek']) + if isinstance(hybrid_order, str): + hybrid_order = [hybrid_order] + source_cache_key = f"{download_mode}:{','.join(str(s) for s in (hybrid_order or []))}" + + # Test Soulseek/download source only when the cached source selection is still current. + if (current_time - _status_cache_timestamps['soulseek'] > STATUS_CACHE_TTL or + _status_cache['soulseek'].get('source_cache_key') != source_cache_key): soulseek_relevant = (download_mode == 'soulseek' or (download_mode == 'hybrid' and 'soulseek' in hybrid_order)) @@ -2130,11 +2135,24 @@ def get_status(): is_serverless = (download_mode in serverless_sources or (download_mode == 'hybrid' and hybrid_order and any(s in serverless_sources for s in hybrid_order))) + external_client_sources = ('torrent', 'usenet') + external_client_relevant = ( + download_mode in external_client_sources or + (download_mode == 'hybrid' and + hybrid_order and any(s in external_client_sources for s in hybrid_order)) + ) # Serverless check first — avoids slow slskd timeout when YouTube/HiFi are in hybrid order if is_serverless: soulseek_status = True soulseek_response_time = 0 + elif external_client_relevant and download_orchestrator: + soulseek_start = time.time() + try: + soulseek_status = run_async(download_orchestrator.check_connection()) + except Exception: + soulseek_status = False + soulseek_response_time = (time.time() - soulseek_start) * 1000 elif soulseek_relevant and download_orchestrator: soulseek_start = time.time() try: @@ -2148,13 +2166,17 @@ def get_status(): _status_cache['soulseek'] = { 'connected': soulseek_status, - 'response_time': round(soulseek_response_time, 1) + 'response_time': round(soulseek_response_time, 1), + 'source_cache_key': source_cache_key, } _status_cache_timestamps['soulseek'] = current_time # Include download source mode so frontend can update labels - download_mode = config_manager.get('download_source.mode', 'hybrid') _status_cache['soulseek']['source'] = download_mode + soulseek_data = { + key: value for key, value in _status_cache['soulseek'].items() + if key != 'source_cache_key' + } # Count active downloads for nav badge active_dl_count = 0 @@ -2167,7 +2189,7 @@ def get_status(): 'metadata_source': metadata_status['metadata_source'], 'spotify': metadata_status['spotify'], 'media_server': _status_cache['media_server'], - 'soulseek': _status_cache['soulseek'], + 'soulseek': soulseek_data, 'active_media_server': active_server, 'enrichment': _get_enrichment_status(), 'active_downloads': active_dl_count, @@ -5887,6 +5909,14 @@ def _find_completed_file_robust(download_dir, api_filename, transfer_dir=None): from difflib import SequenceMatcher from unidecode import unidecode + audio_extensions = { + '.mp3', '.flac', '.m4a', '.aac', '.ogg', '.opus', '.wav', '.wma', '.alac', + '.aiff', '.aif', '.dsf', '.dff', '.ape', + } + + def _is_audio_candidate(path): + return os.path.splitext(str(path or ''))[1].lower() in audio_extensions + # YOUTUBE/TIDAL SUPPORT: Handle encoded filename format "id||title" # Extract just the title part for file matching if '||' in api_filename: @@ -5921,9 +5951,12 @@ def _find_completed_file_robust(download_dir, api_filename, transfer_dir=None): # Skip quarantine folder — contains known-wrong files from AcoustID verification dirs[:] = [d for d in dirs if d != 'ss_quarantine'] for file in files: + file_path = os.path.join(root, file) + if not _is_audio_candidate(file_path): + continue + # Direct basename match if os.path.basename(file) == target_basename: - file_path = os.path.join(root, file) # Fast path: if path aligns with expected directory structure, return now if api_dir_parts and _path_matches_api_dirs(file_path): logger.info(f"Found path-confirmed match in {location_name}: {file_path}") @@ -5940,7 +5973,6 @@ def _find_completed_file_robust(download_dir, api_filename, transfer_dir=None): file_stem, file_ext_part = os.path.splitext(file) stripped_stem = re.sub(r'_\d{10,}$', '', file_stem) if stripped_stem != file_stem and stripped_stem + file_ext_part == target_basename: - file_path = os.path.join(root, file) if api_dir_parts and _path_matches_api_dirs(file_path): logger.info(f"Found path-confirmed dedup match in {location_name}: {file_path}") return file_path, 1.0 @@ -5956,7 +5988,7 @@ def _find_completed_file_robust(download_dir, api_filename, transfer_dir=None): if similarity > highest_fuzzy_similarity: highest_fuzzy_similarity = similarity - best_fuzzy_path = os.path.join(root, file) + best_fuzzy_path = file_path # Return best exact match (disambiguated by path), or fall back to fuzzy if exact_matches: diff --git a/webui/static/downloads.js b/webui/static/downloads.js index 678c1796..7df12aff 100644 --- a/webui/static/downloads.js +++ b/webui/static/downloads.js @@ -3300,6 +3300,34 @@ function closeCandidatesModal() { } } +function _downloadModalBundleProgressPercent(bundle) { + if (!bundle) return 0; + const raw = bundle.progress_percent ?? bundle.progress ?? 0; + let progress = Number(raw); + if (!Number.isFinite(progress)) progress = 0; + if (progress <= 1) progress *= 100; + return Math.max(0, Math.min(100, Math.round(progress))); +} + +function _downloadModalFormatBytes(bytes) { + const value = Number(bytes); + if (!Number.isFinite(value) || value <= 0) return ''; + const units = ['B', 'KB', 'MB', 'GB', 'TB']; + let size = value; + let unit = 0; + while (size >= 1024 && unit < units.length - 1) { + size /= 1024; + unit += 1; + } + const decimals = size >= 10 || unit === 0 ? 0 : 1; + return `${size.toFixed(decimals)} ${units[unit]}`; +} + +function _downloadModalFormatSpeed(bytesPerSecond) { + const formatted = _downloadModalFormatBytes(bytesPerSecond); + return formatted ? `${formatted}/s` : ''; +} + function processModalStatusUpdate(playlistId, data) { // This function contains ALL the existing polling logic from startModalDownloadPolling // Extracted so it can be called from both individual and batched polling @@ -3348,6 +3376,35 @@ function processModalStatusUpdate(playlistId, data) { // Auto-save M3U file for playlists after analysis autoSavePlaylistM3U(playlistId); } + } else if (data.phase === 'album_downloading') { + const analysisFill = document.getElementById(`analysis-progress-fill-${playlistId}`); + const analysisText = document.getElementById(`analysis-progress-text-${playlistId}`); + if (analysisFill) analysisFill.style.width = '100%'; + if (analysisText) analysisText.textContent = 'Analysis complete!'; + + const bundle = data.album_bundle || {}; + const percent = _downloadModalBundleProgressPercent(bundle); + const source = bundle.source ? `${bundle.source} ` : ''; + const release = bundle.release ? ` - ${bundle.release}` : ''; + const speed = _downloadModalFormatSpeed(bundle.speed); + const size = _downloadModalFormatBytes(bundle.size); + const detail = speed || size ? ` (${[speed, size].filter(Boolean).join(' of ')})` : ''; + const downloadFill = document.getElementById(`download-progress-fill-${playlistId}`); + const downloadText = document.getElementById(`download-progress-text-${playlistId}`); + if (downloadFill) downloadFill.style.width = `${percent}%`; + if (downloadText) { + downloadText.textContent = `${source}album ${bundle.state || 'downloading'} ${percent}%${release}${detail}`; + } + + const modal = document.getElementById(`download-missing-modal-${playlistId}`); + if (modal) { + modal.querySelectorAll('[id^="download-"]').forEach(statusEl => { + if (!statusEl.id.startsWith(`download-${playlistId}-`)) return; + if (!statusEl.textContent || statusEl.textContent === '-' || statusEl.textContent.includes('Pending')) { + statusEl.textContent = 'Waiting for album bundle'; + } + }); + } } else if (data.phase === 'downloading' || data.phase === 'complete' || data.phase === 'error') { console.debug(`📊 [Status Update] Processing ${data.phase} phase for playlistId: ${playlistId}, tasks: ${(data.tasks || []).length}`); diff --git a/webui/static/shared-helpers.js b/webui/static/shared-helpers.js index 9840fea3..05ab4fa2 100644 --- a/webui/static/shared-helpers.js +++ b/webui/static/shared-helpers.js @@ -3377,8 +3377,8 @@ function updateServiceStatus(service, statusData, spotifyStatus = null) { // Update download source title on dashboard card if (service === 'soulseek' && statusData.source) { - const sourceNames = { soulseek: 'Soulseek', youtube: 'YouTube', tidal: 'Tidal', qobuz: 'Qobuz', hifi: 'HiFi', deezer_dl: 'Deezer', lidarr: 'Lidarr', soundcloud: 'SoundCloud', hybrid: 'Hybrid' }; - const displayName = sourceNames[statusData.source] || 'Soulseek'; + const sourceNames = { soulseek: 'Soulseek', youtube: 'YouTube', tidal: 'Tidal', qobuz: 'Qobuz', hifi: 'HiFi', deezer_dl: 'Deezer', amazon: 'Amazon', lidarr: 'Lidarr', soundcloud: 'SoundCloud', torrent: 'Torrent', usenet: 'Usenet', hybrid: 'Hybrid' }; + const displayName = sourceNames[statusData.source] || 'Download Source'; const titleEl = document.getElementById('download-source-title'); if (titleEl) titleEl.textContent = displayName; } @@ -3422,8 +3422,8 @@ function updateSidebarServiceStatus(service, statusData, spotifyStatus = null) { // Update download source name based on configured mode if (service === 'soulseek' && statusData.source) { - const sourceNames = { soulseek: 'Soulseek', youtube: 'YouTube', tidal: 'Tidal', qobuz: 'Qobuz', hifi: 'HiFi', deezer_dl: 'Deezer', lidarr: 'Lidarr', soundcloud: 'SoundCloud', hybrid: 'Hybrid' }; - const displayName = sourceNames[statusData.source] || 'Soulseek'; + const sourceNames = { soulseek: 'Soulseek', youtube: 'YouTube', tidal: 'Tidal', qobuz: 'Qobuz', hifi: 'HiFi', deezer_dl: 'Deezer', amazon: 'Amazon', lidarr: 'Lidarr', soundcloud: 'SoundCloud', torrent: 'Torrent', usenet: 'Usenet', hybrid: 'Hybrid' }; + const displayName = sourceNames[statusData.source] || 'Download Source'; const sidebarName = document.getElementById('download-source-name'); if (sidebarName) sidebarName.textContent = displayName; }