Merge pull request #495 from Nezreka/refactor/download-source-plugins

Refactor/download source plugins
This commit is contained in:
BoulderBadgeDad 2026-05-05 15:43:18 -07:00 committed by GitHub
commit b53d6f1b95
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
64 changed files with 6291 additions and 1995 deletions

View file

@ -121,7 +121,7 @@ def register_routes(bp):
try:
from utils.async_helpers import run_async
soulseek = current_app.soulsync.get("soulseek_client")
soulseek = current_app.soulsync.get("download_orchestrator")
if not soulseek:
return api_error("NOT_AVAILABLE", "Soulseek client not configured.", 503)
@ -138,7 +138,7 @@ def register_routes(bp):
"""Cancel all active downloads and clear completed ones."""
try:
from utils.async_helpers import run_async
soulseek = current_app.soulsync.get("soulseek_client")
soulseek = current_app.soulsync.get("download_orchestrator")
if not soulseek:
return api_error("NOT_AVAILABLE", "Soulseek client not configured.", 503)

View file

@ -104,7 +104,7 @@ def _run_search_and_download(request_id, query, notify_url):
if request_id in _pending_requests:
_pending_requests[request_id]['status'] = 'searching'
soulseek = current_app._get_current_object().soulsync.get('soulseek_client')
soulseek = current_app._get_current_object().soulsync.get('download_orchestrator')
if not soulseek:
with _requests_lock:
if request_id in _pending_requests:

View file

@ -26,7 +26,7 @@ def register_routes(bp):
spotify = ctx.get("spotify_client")
spotify_ok = bool(spotify and spotify.is_authenticated())
soulseek = ctx.get("soulseek_client")
soulseek = ctx.get("download_orchestrator")
soulseek_ok = bool(soulseek)
hydrabase = ctx.get("hydrabase_client")

View file

@ -1,6 +1,6 @@
"""Service connection test — lifted from web_server.py.
The function body is byte-identical to the original. soulseek_client,
The function body is byte-identical to the original. download_orchestrator,
qobuz_enrichment_worker, hydrabase_client, docker_resolve_url, and
docker_resolve_path are injected at runtime because they live in
web_server.py and are constructed there.
@ -27,7 +27,7 @@ def _get_metadata_fallback_source():
# Injected at runtime via init().
soulseek_client = None
download_orchestrator = None
qobuz_enrichment_worker = None
hydrabase_client = None
docker_resolve_url = None
@ -35,16 +35,16 @@ docker_resolve_path = None
def init(
soulseek_client_obj,
download_orchestrator_obj,
qobuz_worker,
hydrabase_client_obj,
docker_resolve_url_fn,
docker_resolve_path_fn,
):
"""Bind web_server-side helpers/globals so the lifted body can resolve them."""
global soulseek_client, qobuz_enrichment_worker, hydrabase_client
global download_orchestrator, qobuz_enrichment_worker, hydrabase_client
global docker_resolve_url, docker_resolve_path
soulseek_client = soulseek_client_obj
download_orchestrator = download_orchestrator_obj
qobuz_enrichment_worker = qobuz_worker
hydrabase_client = hydrabase_client_obj
docker_resolve_url = docker_resolve_url_fn
@ -177,13 +177,13 @@ def run_service_test(service, test_config):
else:
return False, f"Output folder not found: {transfer_path}"
elif service == "soulseek":
if soulseek_client is None:
if download_orchestrator is None:
return False, "Download orchestrator failed to initialize. Check server logs for startup errors."
# Test the orchestrator's configured download source (not just Soulseek)
download_mode = config_manager.get('download_source.mode', 'hybrid')
if run_async(soulseek_client.check_connection()):
if run_async(download_orchestrator.check_connection()):
# Success message based on active mode
mode_messages = {
'soulseek': "Successfully connected to Soulseek network via slskd.",

View file

@ -64,7 +64,7 @@ download_batches = None
sync_states = None
youtube_playlist_states = None
tidal_discovery_states = None
soulseek_client = None
download_orchestrator = None
_log_path = None
_log_dir = None
app = None
@ -80,7 +80,7 @@ def init(
sync_states_dict,
youtube_playlist_states_dict,
tidal_discovery_states_dict,
soulseek_client_obj,
download_orchestrator_obj,
log_path,
log_dir,
flask_app,
@ -90,7 +90,7 @@ def init(
"""Bind shared state/helpers from web_server."""
global SOULSYNC_VERSION, _DIRECT_RUN, _status_cache, qobuz_enrichment_worker
global download_batches, sync_states, youtube_playlist_states
global tidal_discovery_states, soulseek_client, _log_path, _log_dir
global tidal_discovery_states, download_orchestrator, _log_path, _log_dir
global app, get_database, _get_tidal_client
SOULSYNC_VERSION = soulsync_version
_DIRECT_RUN = direct_run
@ -100,7 +100,7 @@ def init(
sync_states = sync_states_dict
youtube_playlist_states = youtube_playlist_states_dict
tidal_discovery_states = tidal_discovery_states_dict
soulseek_client = soulseek_client_obj
download_orchestrator = download_orchestrator_obj
_log_path = log_path
_log_dir = log_dir
app = flask_app
@ -292,9 +292,9 @@ def get_debug_info():
# Download client init failures
info['download_client_failures'] = []
if soulseek_client and hasattr(soulseek_client, '_init_failures'):
info['download_client_failures'] = soulseek_client._init_failures
elif not soulseek_client:
if download_orchestrator and hasattr(download_orchestrator, '_init_failures'):
info['download_client_failures'] = download_orchestrator._init_failures
elif not download_orchestrator:
info['download_client_failures'] = ['ALL (orchestrator failed to initialize)']
# API rate monitor — current calls/min, 24h totals, peaks, rate limit events

View file

@ -20,7 +20,7 @@ from typing import Any, Dict, List, Optional, Tuple
import requests
from core.soulseek_client import AlbumResult, DownloadStatus, TrackResult
from core.download_plugins.types import AlbumResult, DownloadStatus, TrackResult
from utils.logging_config import get_logger
logger = get_logger("deezer_download")
@ -79,7 +79,10 @@ def _decrypt_chunk(chunk: bytes, key: bytes) -> bytes:
) from exc
class DeezerDownloadClient:
from core.download_plugins.base import DownloadSourcePlugin
class DeezerDownloadClient(DownloadSourcePlugin):
"""Deezer download client using ARL token authentication."""
def __init__(self, download_path: str = None):
@ -91,9 +94,9 @@ class DeezerDownloadClient:
self.download_path = Path(download_path)
self.download_path.mkdir(parents=True, exist_ok=True)
# Download tracking (same pattern as Tidal/Qobuz/HiFi)
self.active_downloads: Dict[str, Dict[str, Any]] = {}
self._download_lock = threading.Lock()
# Engine reference is populated by set_engine() at registration
# time. None until orchestrator wires the registry.
self._engine = None
# Shutdown check callback (set by web_server)
self.shutdown_check = None
@ -125,6 +128,10 @@ class DeezerDownloadClient:
logger.info(f"Deezer download client initialized (download path: {self.download_path})")
def set_engine(self, engine):
"""Engine callback — wires the central thread worker + state store."""
self._engine = engine
# ─── Authentication ──────────────────────────────────────────
def _authenticate(self, arl: str) -> bool:
@ -605,87 +612,67 @@ class DeezerDownloadClient:
if not self._authenticated:
logger.error("Deezer not authenticated — cannot download")
return None
if self._engine is None:
# Raise rather than return None so the orchestrator's
# download_with_fallback surfaces a real warning + tries
# the next source. Returning None silently dropped the
# download with no user feedback (per JohnBaumb).
raise RuntimeError("Deezer client has no engine reference — cannot dispatch download")
# Parse filename: "track_id||display_name"
parts = filename.split('||', 1)
track_id = parts[0]
display_name = parts[1] if len(parts) > 1 else f"Track {track_id}"
download_id = str(uuid.uuid4())
with self._download_lock:
self.active_downloads[download_id] = {
'id': download_id,
return self._engine.worker.dispatch(
source_name='deezer',
target_id=track_id,
display_name=display_name,
original_filename=filename,
impl_callable=self._download_sync,
extra_record_fields={
'track_id': track_id,
'display_name': display_name,
'filename': filename,
'username': 'deezer_dl',
'state': 'Initializing',
'progress': 0.0,
'size': file_size,
'transferred': 0,
'speed': 0,
'file_path': None,
'error': None,
}
thread = threading.Thread(
target=self._download_thread_worker,
args=(download_id, track_id, display_name),
daemon=True,
name=f'deezer-dl-{track_id}'
},
# Legacy username slot — frontend status indicators key off
# ``deezer_dl``, not the canonical ``deezer``.
username_override='deezer_dl',
# Diagnostic thread name for multi-thread debugging.
thread_name=f'deezer-dl-{track_id}',
)
thread.start()
logger.info(f"Started Deezer download {download_id}: {display_name}")
return download_id
def _set_error(self, download_id: str, message: str) -> None:
"""Helper: set the engine record's `error` slot. No-op if
engine isn't wired or record was already removed."""
if self._engine is None:
return
self._engine.update_record('deezer', download_id, {'error': message})
def _download_thread_worker(self, download_id: str, track_id: str, display_name: str):
"""Background worker for a single download."""
try:
result_path = self._download_sync(download_id, track_id, display_name)
with self._download_lock:
if download_id in self.active_downloads:
dl = self.active_downloads[download_id]
if dl['state'] == 'Cancelled':
return
if result_path:
dl['state'] = 'Completed, Succeeded'
dl['progress'] = 100.0
dl['file_path'] = result_path
logger.info(f"Deezer download {download_id} completed: {result_path}")
else:
dl['state'] = 'Errored'
logger.error(f"Deezer download {download_id} failed: {dl.get('error', 'unknown')}")
except Exception as e:
logger.error(f"Deezer download thread error: {e}")
with self._download_lock:
if download_id in self.active_downloads:
self.active_downloads[download_id]['state'] = 'Errored'
self.active_downloads[download_id]['error'] = str(e)
def _is_cancelled(self, download_id: str) -> bool:
if self._engine is None:
return False
record = self._engine.get_record('deezer', download_id)
return record is not None and record.get('state') == 'Cancelled'
def _download_sync(self, download_id: str, track_id: str, display_name: str) -> Optional[str]:
"""Synchronous download: get URL, download, decrypt, save."""
# Check for shutdown
if self.shutdown_check and self.shutdown_check():
with self._download_lock:
if download_id in self.active_downloads:
self.active_downloads[download_id]['state'] = 'Aborted'
if self._engine is not None:
self._engine.update_record('deezer', download_id, {'state': 'Aborted'})
return None
# Get track data from private API
track_data = self._get_track_data(track_id)
if not track_data:
with self._download_lock:
if download_id in self.active_downloads:
self.active_downloads[download_id]['error'] = 'Failed to get track data'
self._set_error(download_id, 'Failed to get track data')
return None
track_token = track_data.get('TRACK_TOKEN', '')
if not track_token:
with self._download_lock:
if download_id in self.active_downloads:
self.active_downloads[download_id]['error'] = 'No track token available'
self._set_error(download_id, 'No track token available')
return None
# Determine quality and get media URL with fallback
@ -695,7 +682,6 @@ class DeezerDownloadClient:
if allow_fallback:
quality_order = _QUALITY_ORDER.copy()
# Start from user's preferred quality
try:
pref_idx = quality_order.index(self._quality)
quality_order = quality_order[pref_idx:] + quality_order[:pref_idx]
@ -712,27 +698,16 @@ class DeezerDownloadClient:
break
if not media_url:
with self._download_lock:
if download_id in self.active_downloads:
self.active_downloads[download_id]['error'] = 'No media URL available (may require higher subscription tier)'
self._set_error(download_id, 'No media URL available (may require higher subscription tier)')
return None
if actual_quality != self._quality:
logger.info(f"Quality fallback: {self._quality}{actual_quality} for {display_name}")
# Determine file extension
ext = '.flac' if actual_quality == 'flac' else '.mp3'
# Sanitize filename
safe_name = self._sanitize_filename(display_name)
out_path = str(self.download_path / f"{safe_name}{ext}")
# Update state
with self._download_lock:
if download_id in self.active_downloads:
dl = self.active_downloads[download_id]
dl['state'] = 'InProgress, Downloading'
# Download and decrypt
try:
bf_key = _get_blowfish_key(track_id)
@ -740,9 +715,8 @@ class DeezerDownloadClient:
resp.raise_for_status()
total_size = int(resp.headers.get('content-length', 0))
with self._download_lock:
if download_id in self.active_downloads:
self.active_downloads[download_id]['size'] = total_size
if self._engine is not None:
self._engine.update_record('deezer', download_id, {'size': total_size})
downloaded = 0
chunk_index = 0
@ -755,23 +729,20 @@ class DeezerDownloadClient:
# Check for cancellation/shutdown
if self.shutdown_check and self.shutdown_check():
with self._download_lock:
if download_id in self.active_downloads:
self.active_downloads[download_id]['state'] = 'Aborted'
if self._engine is not None:
self._engine.update_record('deezer', download_id, {'state': 'Aborted'})
try:
os.remove(out_path)
except OSError:
pass
return None
with self._download_lock:
if download_id in self.active_downloads:
if self.active_downloads[download_id]['state'] == 'Cancelled':
try:
os.remove(out_path)
except OSError:
pass
return None
if self._is_cancelled(download_id):
try:
os.remove(out_path)
except OSError:
pass
return None
# Decrypt every 3rd chunk (Deezer's encryption pattern)
if chunk_index % 3 == 0 and len(raw_chunk) == _CHUNK_SIZE:
@ -788,12 +759,12 @@ class DeezerDownloadClient:
speed = int(downloaded / elapsed) if elapsed > 0 else 0
progress = (downloaded / total_size * 100) if total_size > 0 else 0
with self._download_lock:
if download_id in self.active_downloads:
dl = self.active_downloads[download_id]
dl['transferred'] = downloaded
dl['progress'] = min(progress, 99.9)
dl['speed'] = speed
if self._engine is not None:
self._engine.update_record('deezer', download_id, {
'transferred': downloaded,
'progress': min(progress, 99.9),
'speed': speed,
})
# Validate file size
file_size = os.path.getsize(out_path)
@ -803,9 +774,7 @@ class DeezerDownloadClient:
os.remove(out_path)
except OSError:
pass
with self._download_lock:
if download_id in self.active_downloads:
self.active_downloads[download_id]['error'] = f'File too small ({file_size} bytes)'
self._set_error(download_id, f'File too small ({file_size} bytes)')
return None
logger.info(f"Deezer download complete: {out_path} ({file_size / 1048576:.1f} MB, {actual_quality})")
@ -817,59 +786,58 @@ class DeezerDownloadClient:
os.remove(out_path)
except OSError:
pass
with self._download_lock:
if download_id in self.active_downloads:
self.active_downloads[download_id]['error'] = str(e)
self._set_error(download_id, str(e))
return None
# ─── Download Status ─────────────────────────────────────────
def _record_to_status(self, record: dict) -> DownloadStatus:
return DownloadStatus(
id=record['id'],
filename=record['filename'],
username=record['username'],
state=record['state'],
progress=record['progress'],
size=record.get('size', 0),
transferred=record.get('transferred', 0),
speed=record.get('speed', 0),
file_path=record.get('file_path'),
)
async def get_all_downloads(self) -> List[DownloadStatus]:
"""Return all active downloads."""
with self._download_lock:
return [self._to_status(dl) for dl in self.active_downloads.values()]
if self._engine is None:
return []
return [
self._record_to_status(record)
for record in self._engine.iter_records_for_source('deezer')
]
async def get_download_status(self, download_id: str) -> Optional[DownloadStatus]:
"""Get status of a specific download."""
with self._download_lock:
dl = self.active_downloads.get(download_id)
return self._to_status(dl) if dl else None
if self._engine is None:
return None
record = self._engine.get_record('deezer', download_id)
return self._record_to_status(record) if record is not None else None
async def cancel_download(self, download_id: str, username: str = None,
remove: bool = False) -> bool:
"""Cancel a download."""
with self._download_lock:
dl = self.active_downloads.get(download_id)
if not dl:
return False
dl['state'] = 'Cancelled'
if remove:
del self.active_downloads[download_id]
if self._engine is None:
return False
if self._engine.get_record('deezer', download_id) is None:
return False
self._engine.update_record('deezer', download_id, {'state': 'Cancelled'})
if remove:
self._engine.remove_record('deezer', download_id)
return True
async def clear_all_completed_downloads(self) -> bool:
"""Remove all terminal downloads."""
terminal_states = {'Completed, Succeeded', 'Cancelled', 'Errored', 'Aborted'}
with self._download_lock:
to_remove = [k for k, v in self.active_downloads.items() if v['state'] in terminal_states]
for k in to_remove:
del self.active_downloads[k]
if self._engine is None:
return True
terminal = {'Completed, Succeeded', 'Cancelled', 'Errored', 'Aborted'}
for record in list(self._engine.iter_records_for_source('deezer')):
if record.get('state') in terminal:
self._engine.remove_record('deezer', record['id'])
return True
def _to_status(self, dl: dict) -> DownloadStatus:
"""Convert internal dict to DownloadStatus."""
return DownloadStatus(
id=dl['id'],
filename=dl['filename'],
username=dl['username'],
state=dl['state'],
progress=dl['progress'],
size=dl['size'],
transferred=dl['transferred'],
speed=dl['speed'],
file_path=dl.get('file_path'),
)
# ─── Utilities ───────────────────────────────────────────────
@staticmethod

View file

@ -0,0 +1,31 @@
"""Download Engine — central owner of cross-source download state,
thread workers, search retry, rate-limits, and fallback chains.
This is the second leg of the multi-source download dispatcher
refactor (the first leg, ``core/download_plugins/``, defined the
contract). The engine takes ownership of everything that used to
be duplicated across the per-source clients (background thread
workers, active_downloads dicts, search retry ladders, quality
filtering, hybrid fallback). Clients become DUMB just hit the
API for their source, manage their own auth state, and let the
engine drive everything else.
This package is built up in phases (see
``docs/download-engine-refactor-plan.md`` for the full plan):
- Phase B (current) engine skeleton + state lift.
- Phase C background download worker.
- Phase D search retry + quality filter.
- Phase E rate-limit pool.
- Phase F fallback chain.
Each phase is purely additive at first (engine grows, clients
unchanged). Migration to the new shape happens one source per
commit so behavior never breaks across the suite.
"""
from core.download_engine.engine import DownloadEngine
from core.download_engine.rate_limit import RateLimitPolicy
from core.download_engine.worker import BackgroundDownloadWorker
__all__ = ["DownloadEngine", "BackgroundDownloadWorker", "RateLimitPolicy"]

View file

@ -0,0 +1,457 @@
"""DownloadEngine — central owner of cross-source download state.
Phase B scope: skeleton only. The engine exposes a place for
plugins to register, a single ``active_downloads`` dict keyed by
``(source, download_id)``, and per-source RLocks that guard mutations
without serializing workers across different sources.
Subsequent phases bolt more capability on top:
- ``dispatch_download(plugin, target_id)`` (Phase C replaces every
client's ``_download_thread_worker`` boilerplate).
- ``search(query, source_chain)`` (Phase D replaces every client's
retry ladder + quality filter).
- ``rate_limit.acquire(source)`` (Phase E replaces every client's
semaphore + last-download-timestamp dance).
- ``search_with_fallback`` / ``download_with_fallback`` (Phase F
unifies hybrid mode across search and download).
The engine is constructed by ``DownloadOrchestrator.__init__`` and
each plugin from the registry is registered with it. In Phase B
nothing in the existing code paths goes through the engine yet
this commit is pure additive scaffolding so subsequent commits can
introduce engine-driven behavior one piece at a time without a
big-bang switchover.
"""
from __future__ import annotations
import threading
from typing import Any, Dict, Iterator, List, Optional, Tuple
from utils.logging_config import get_logger
logger = get_logger("download_engine")
# Type alias for the per-download state dict. Today's clients each
# define their own slightly-different shape (see Phase A pinning
# tests); the engine stores them as opaque dicts and the per-plugin
# accessor preserves the source-specific fields.
DownloadRecord = Dict[str, Any]
class DownloadEngine:
"""Central state for every active download across every source.
State is keyed by ``(source_name, download_id)`` so the same
UUID could hypothetically appear in two sources without
collision (in practice each source generates its own UUID4
so collisions are negligible the source qualifier exists
so the engine can answer "which plugin owns this download" in
O(1) without iterating every plugin).
Thread safety: per-source lock sharding. Each source gets its own
RLock progress callbacks on Deezer don't block Tidal's worker
and vice versa, matching the pre-refactor behavior where each
client owned its own download lock. Read-only accessors
(``get_record``, ``iter_records_for_source``) take the source's
lock briefly and return a SHALLOW COPY so the caller can iterate
without holding the lock. Callers that need to mutate a record
should use ``update_record`` which takes the lock and applies the
patch atomically.
"""
def __init__(self) -> None:
# Nested dict: source_name → {download_id → record}. Replaces
# the original single-dict composite-key layout so
# ``iter_records_for_source`` is O(source_records) instead of
# O(total_records).
self._records: Dict[str, Dict[str, DownloadRecord]] = {}
# Per-source RLocks. Each source gets its own so progress
# updates on one source never block writes on another. RLock
# so a plugin's worker callback can re-enter while holding the
# lock for its own update. Lazily created via ``_source_lock``;
# the meta-lock guards creation against the create-race window
# where two threads could both miss + both create.
self._source_locks: Dict[str, threading.RLock] = {}
self._source_locks_lock = threading.Lock()
# Plugins that have registered with the engine. Source name
# → plugin instance.
self._plugins: Dict[str, Any] = {}
# Alias → canonical-name map. Lets engine resolve legacy
# source-name strings (e.g. ``'deezer_dl'`` for Deezer) to
# the canonical key in ``_plugins``. Cin's review caught
# that engine.cancel_download(source_hint='deezer_dl')
# silently fell through to Soulseek because alias resolution
# only existed at the registry, not on the engine.
self._aliases: Dict[str, str] = {}
# Background download worker — lives on the engine because
# it owns the cross-source state the worker mutates. Lazy
# import keeps the engine module standalone.
from core.download_engine.worker import BackgroundDownloadWorker
self.worker = BackgroundDownloadWorker(self)
# ------------------------------------------------------------------
# Plugin registration
# ------------------------------------------------------------------
def register_plugin(self, source_name: str, plugin: Any,
aliases: Tuple[str, ...] = ()) -> None:
"""Register a plugin under its canonical source name. Called
once per source by the orchestrator after the registry's
``initialize`` builds the client instances.
``aliases`` is the list of legacy source-name strings that
should resolve to this plugin (e.g. ``'deezer_dl'`` for
Deezer). Without alias resolution the engine couldn't route
cancel/lookup calls that came in with the legacy name.
If the plugin exposes ``set_engine(engine)``, the engine
passes a self-reference so the plugin can dispatch into
``engine.worker`` / read state / etc. Plugins that haven't
been migrated to the engine yet simply don't define
``set_engine`` they keep their pre-engine behavior
unchanged.
Also reads the plugin's declared ``RateLimitPolicy`` (via
the ``rate_limit_policy()`` method or ``RATE_LIMIT_POLICY``
class attribute) and applies it to the worker. Plugins that
don't declare a policy get the conservative default
(concurrency=1, delay=0).
"""
if source_name in self._plugins:
logger.warning("Plugin %s already registered with engine — overwriting", source_name)
self._plugins[source_name] = plugin
for alias in aliases:
self._aliases[alias] = source_name
# Apply the plugin's rate-limit policy BEFORE set_engine so
# set_engine callbacks can override per-source if they need
# config-driven values (e.g. YouTube's user-tunable delay).
from core.download_engine.rate_limit import resolve_policy
policy = resolve_policy(plugin)
self.worker.set_concurrency(source_name, policy.download_concurrency)
self.worker.set_delay(source_name, policy.download_delay_seconds)
set_engine = getattr(plugin, 'set_engine', None)
if callable(set_engine):
try:
set_engine(self)
except Exception as exc:
logger.warning(
"Plugin %s set_engine callback failed: %s", source_name, exc,
)
def get_plugin(self, source_name: str) -> Optional[Any]:
"""Return the plugin instance for the given source name.
Resolves through aliases e.g. ``get_plugin('deezer_dl')``
returns the same instance as ``get_plugin('deezer')``."""
if source_name in self._plugins:
return self._plugins[source_name]
canonical = self._aliases.get(source_name)
if canonical:
return self._plugins.get(canonical)
return None
def _resolve_canonical(self, source_name: str) -> Optional[str]:
"""Return the canonical source name for an input that may be
an alias. Returns None if the input matches neither a
canonical name nor an alias."""
if source_name in self._plugins:
return source_name
return self._aliases.get(source_name)
def registered_sources(self) -> List[str]:
return list(self._plugins.keys())
def _source_lock(self, source_name: str) -> threading.RLock:
"""Return the per-source RLock, lazy-creating it on first use.
The meta-lock around the cache lookup closes the create-race
window where two threads both miss + both create a fresh lock.
"""
with self._source_locks_lock:
lock = self._source_locks.get(source_name)
if lock is None:
lock = threading.RLock()
self._source_locks[source_name] = lock
return lock
# ------------------------------------------------------------------
# Active-downloads state — Phase B core surface
# ------------------------------------------------------------------
def add_record(self, source_name: str, download_id: str, record: DownloadRecord) -> None:
"""Insert a fresh download record. Used by clients (today
directly via their own dicts; Phase B2 routes them through
here)."""
with self._source_lock(source_name):
source_bucket = self._records.setdefault(source_name, {})
if download_id in source_bucket:
logger.warning("Replacing existing download record for %s/%s", source_name, download_id)
source_bucket[download_id] = dict(record)
def update_record(self, source_name: str, download_id: str, patch: DownloadRecord) -> None:
"""Apply a partial patch to an existing record. No-op if the
record was already removed (e.g. cancelled mid-update)."""
with self._source_lock(source_name):
existing = self._records.get(source_name, {}).get(download_id)
if existing is None:
return
existing.update(patch)
def update_record_unless_state(self, source_name: str, download_id: str,
patch: DownloadRecord,
skip_if_state_in: Tuple[str, ...] = ()) -> bool:
"""Atomically check the record's state and apply ``patch`` only
if the current state is NOT in ``skip_if_state_in``. Returns
True if the patch was applied, False if it was skipped (or
the record didn't exist).
Used by the background download worker's ``_mark_terminal``
to avoid the read-then-write race Cin flagged: a cancel
landing between the snapshot and update could be overwritten
back to Errored / Completed. Holding the source's lock across
the check + write closes the window.
"""
with self._source_lock(source_name):
existing = self._records.get(source_name, {}).get(download_id)
if existing is None:
return False
if existing.get('state') in skip_if_state_in:
return False
existing.update(patch)
return True
def remove_record(self, source_name: str, download_id: str) -> Optional[DownloadRecord]:
"""Delete a record (cancellation cleanup). Returns the
removed record or None if not found."""
with self._source_lock(source_name):
source_bucket = self._records.get(source_name)
if not source_bucket:
return None
removed = source_bucket.pop(download_id, None)
# Drop the empty source bucket so iteration / membership
# checks don't see a stale source key.
if not source_bucket:
self._records.pop(source_name, None)
return removed
def get_record(self, source_name: str, download_id: str) -> Optional[DownloadRecord]:
"""Return a SHALLOW COPY of the record. Caller mutations
don't affect engine state — use ``update_record`` for that."""
with self._source_lock(source_name):
record = self._records.get(source_name, {}).get(download_id)
return dict(record) if record is not None else None
def iter_records_for_source(self, source_name: str) -> Iterator[DownloadRecord]:
"""Yield SHALLOW COPIES of every record owned by a source.
Holds the source's lock briefly to snapshot, then yields
outside the lock so callers can spend arbitrary time on each
record.
With the nested-dict layout this is O(source_records) only
touches the bucket for the requested source, not every record
across every source.
"""
with self._source_lock(source_name):
source_bucket = self._records.get(source_name, {})
snapshot = [dict(record) for record in source_bucket.values()]
for record in snapshot:
yield record
# ------------------------------------------------------------------
# Cross-source query dispatch — Phase B2 surface
# ------------------------------------------------------------------
#
# The orchestrator historically iterated every plugin in its own
# ``get_all_downloads`` / ``get_download_status`` / ``cancel_download``
# methods (with hand-maintained client lists, before the registry
# came along). That iteration logic moves into the engine here so
# the orchestrator becomes a thin pass-through (Phase B3).
#
# In Phase B these methods iterate the registered plugins and call
# their existing ``get_all_downloads`` / ``cancel_download``
# methods — same behavior as today, just in a new home. Phase C/D
# will replace plugin-iteration with direct engine-state queries
# once the thread worker is also lifted.
#
# All methods are async to match the per-plugin contract.
async def get_all_downloads(self, exclude: Tuple[str, ...] = ()):
"""Aggregated view across every registered plugin's active
downloads. Per-plugin exceptions are swallowed (one source
failing shouldn't take down cross-source aggregation) but
logged at debug level same defensive shape the legacy
orchestrator had.
``exclude`` skips named sources entirely. The download monitor
passes ``('soulseek',)`` so it doesn't double-fetch slskd
transfers (it already pulled them via the slskd transfers
endpoint earlier in the same loop).
"""
all_downloads = []
for source_name, plugin in self._plugins.items():
if plugin is None or source_name in exclude:
continue
try:
all_downloads.extend(await plugin.get_all_downloads())
except Exception as exc:
logger.debug("%s get_all_downloads failed: %s", source_name, exc)
return all_downloads
async def get_download_status(self, download_id: str):
"""Find a download_id across every plugin. Returns the first
plugin's response or None if no plugin owns it."""
for source_name, plugin in self._plugins.items():
if plugin is None:
continue
try:
status = await plugin.get_download_status(download_id)
if status:
return status
except Exception as exc:
logger.debug("%s get_download_status failed: %s", source_name, exc)
return None
async def cancel_download(self, download_id: str,
source_hint: Optional[str] = None,
remove: bool = False) -> bool:
"""Cancel a download. ``source_hint`` is the source name (or
legacy alias like ``'deezer_dl'``, or a real Soulseek peer
username) when provided, routes directly to that plugin.
When omitted, every plugin is asked in turn until one accepts.
Cin's review caught a bug here: legacy alias strings like
``'deezer_dl'`` weren't resolved to the canonical ``'deezer'``
plugin name, so the cancel silently fell through to Soulseek.
Resolution now goes through ``_resolve_canonical`` first.
"""
# Direct routing when the caller knows the source.
if source_hint:
canonical = self._resolve_canonical(source_hint)
# Streaming source names (or aliases) resolve to a
# registered plugin. Anything else (real Soulseek peer
# name not in our registry) routes to Soulseek.
if canonical and canonical != 'soulseek':
target_plugin = self._plugins.get(canonical)
if target_plugin is not None:
try:
return await target_plugin.cancel_download(
download_id, source_hint, remove,
)
except Exception as exc:
logger.debug("%s cancel_download failed: %s", canonical, exc)
return False
soulseek = self._plugins.get('soulseek')
if soulseek is not None:
try:
return await soulseek.cancel_download(download_id, source_hint, remove)
except Exception as exc:
logger.debug("soulseek cancel_download failed: %s", exc)
return False
# No hint → ask every plugin until one cancels successfully.
for source_name, plugin in self._plugins.items():
if plugin is None:
continue
try:
if await plugin.cancel_download(download_id, source_hint, remove):
return True
except Exception as exc:
logger.debug("%s cancel_download failed: %s", source_name, exc)
return False
async def clear_all_completed_downloads(self) -> bool:
"""Best-effort cleanup of every plugin's completed-downloads
list. Skips plugins that report not-configured (saves API
calls + log noise)."""
results = []
for source_name, plugin in self._plugins.items():
if plugin is None:
continue
if hasattr(plugin, 'is_configured') and not plugin.is_configured():
logger.debug("Skipping %s clear_all_completed_downloads (not configured)", source_name)
continue
try:
results.append(await plugin.clear_all_completed_downloads())
except Exception as exc:
logger.warning("%s clear_all_completed_downloads failed: %s", source_name, exc)
results.append(False)
return all(results) if results else True
# ------------------------------------------------------------------
# Hybrid fallback — Phase F surface
# ------------------------------------------------------------------
async def search_with_fallback(self, query: str, source_chain,
timeout=None, progress_callback=None):
"""Try each source in ``source_chain`` until one returns
tracks. Skips unconfigured / unregistered sources, swallows
per-source exceptions. Returns the first non-empty
(tracks, albums) tuple, or ``([], [])`` when every source
in the chain is exhausted.
Replaces orchestrator's hand-rolled hybrid search loop. The
chain is ordered (most-preferred first).
"""
for i, source_name in enumerate(source_chain):
plugin = self._plugins.get(source_name)
if plugin is None:
logger.info(f"Skipping {source_name} (not available)")
continue
if hasattr(plugin, 'is_configured') and not plugin.is_configured():
logger.info(f"Skipping {source_name} (not configured)")
continue
try:
logger.info(f"Trying {source_name} (priority {i+1}): {query}")
tracks, albums = await plugin.search(query, timeout, progress_callback)
if tracks:
logger.info(f"{source_name} found {len(tracks)} tracks")
return (tracks, albums)
except Exception as e:
logger.warning(f"{source_name} search failed: {e}")
logger.warning(
"Hybrid search: all sources (%s) found nothing for: %s",
', '.join(source_chain), query,
)
return ([], [])
async def download_with_fallback(self, username: str, filename: str,
file_size: int, source_chain) -> Optional[str]:
"""Try each source in ``source_chain`` until one accepts the
download (returns a non-None download_id). Fixes the legacy
bug where hybrid mode silently routed to a single source via
the username hint with no retry on failure.
``username`` is treated as a hint when it matches a source
name in the chain that source is tried FIRST regardless of
chain order. Anything else (e.g. a real Soulseek peer name)
routes through the chain in declared order.
"""
# Promote a matching source-name hint to the head of the chain.
ordered_chain = list(source_chain)
if username and username in ordered_chain:
ordered_chain.remove(username)
ordered_chain.insert(0, username)
for source_name in ordered_chain:
plugin = self._plugins.get(source_name)
if plugin is None:
continue
if hasattr(plugin, 'is_configured') and not plugin.is_configured():
continue
try:
download_id = await plugin.download(username, filename, file_size)
if download_id is not None:
return download_id
logger.info(f"{source_name} declined download — trying next in chain")
except Exception as e:
logger.warning(f"{source_name} download raised — trying next in chain: {e}")
logger.warning(
"Hybrid download: every source in chain (%s) refused %r",
', '.join(ordered_chain), filename,
)
return None

View file

@ -0,0 +1,73 @@
"""Per-source rate-limit policy declarations.
Today's per-source download throttling is scattered:
- YouTube: ``self._download_delay = config_manager.get('youtube.download_delay', 3)``
set in ``__init__``, applied in ``set_engine`` via worker.set_delay.
- Qobuz: module-level ``_qobuz_api_lock`` + ``_QOBUZ_MIN_INTERVAL`` for
search-side throttling, no download-side throttle.
- Other sources: no explicit declarations default to 0s delay /
concurrency=1, which works because the streaming APIs have their
own gateway-level rate limits.
Phase E centralizes this into one place: each plugin declares a
``RateLimitPolicy`` (either as a class attribute or returned from a
``rate_limit_policy()`` method), and the engine reads + applies the
policy to ``engine.worker`` at ``register_plugin`` time.
Adding a new source = declaring its policy alongside the rest of
the source's auth/config — no longer a hidden line in __init__ or a
module-level constant in the client file.
"""
from __future__ import annotations
from dataclasses import dataclass
@dataclass(frozen=True)
class RateLimitPolicy:
"""Per-source download throttling policy.
Attributes:
download_concurrency: Max number of concurrent downloads
from this source. Default 1 (serial). Most streaming
APIs prefer serial transfers because parallel just
trades rate-limit errors for thread overhead.
download_delay_seconds: Minimum gap between successive
downloads from this source. YouTube uses 3s today
(legacy ``_download_delay`` config key) to avoid
yt-dlp 429s. Most other sources use 0.
"""
download_concurrency: int = 1
download_delay_seconds: float = 0.0
# Sentinel default — most plugins want this. Plugins that need
# tighter throttling override by exposing ``RATE_LIMIT_POLICY`` as
# a class attribute or returning a custom one from
# ``rate_limit_policy()``.
DEFAULT_POLICY = RateLimitPolicy()
def resolve_policy(plugin) -> RateLimitPolicy:
"""Read a plugin's declared rate-limit policy. Checks (in order):
1. ``plugin.rate_limit_policy()`` method (returns a RateLimitPolicy)
2. ``plugin.RATE_LIMIT_POLICY`` class attribute
3. ``DEFAULT_POLICY``
"""
method = getattr(plugin, 'rate_limit_policy', None)
if callable(method):
try:
policy = method()
if isinstance(policy, RateLimitPolicy):
return policy
except Exception:
pass
declared = getattr(plugin, 'RATE_LIMIT_POLICY', None)
if isinstance(declared, RateLimitPolicy):
return declared
return DEFAULT_POLICY

View file

@ -0,0 +1,315 @@
"""BackgroundDownloadWorker — engine-owned thread spawning + state
lifecycle for downloads.
Today every streaming download client (YouTube, Tidal, Qobuz, HiFi,
Deezer, SoundCloud) hand-rolls the same thread-spawn pattern:
```python
async def download(self, ...):
download_id = str(uuid.uuid4())
with self._download_lock:
self.active_downloads[download_id] = {...initial state...}
threading.Thread(
target=self._download_thread_worker,
args=(download_id, target_id, display_name, ...),
daemon=True,
).start()
return download_id
def _download_thread_worker(self, download_id, target_id, display_name, ...):
with self._download_semaphore:
# rate-limit sleep
# update state to 'InProgress, Downloading'
file_path = self._download_sync(...) # the source-specific atomic op
# update state to 'Completed, Succeeded' / 'Errored'
```
That pattern is duplicated 6+ times across the codebase (~70 LOC
each, ~490 total). The worker class lifts it into the engine each
plugin only has to provide the atomic op (``impl_callable``) and
declare its rate-limit policy. Adding a new download source becomes
a much smaller patch.
Phase C1 scope: introduce the worker. No client migrated yet the
worker just exists for C2C7 to migrate sources one at a time, each
under a passing pinning test.
"""
from __future__ import annotations
import threading
import time
import uuid
from typing import Any, Callable, Dict, Optional
from utils.logging_config import get_logger
logger = get_logger("download_engine.worker")
# Type aliases for clarity. ``ImplCallable`` is the per-plugin
# atomic download operation — synchronous, returns a file path on
# success or raises (or returns None) on failure.
ImplCallable = Callable[[str, Any, str], Optional[str]]
class BackgroundDownloadWorker:
"""Engine-owned thread spawner for per-source downloads.
State-machine semantics (preserved verbatim from the legacy
per-client workers so consumers reading these fields keep
working):
- ``Initializing`` set on dispatch, before the thread starts.
- ``InProgress, Downloading`` set when the worker thread
acquires the semaphore and is about to call the impl.
- ``Completed, Succeeded`` set when impl returns a non-None
file path. ``progress=100.0`` and ``file_path=<the path>``
also written.
- ``Errored`` set when impl returns None OR raises. The
record is left in place so downstream consumers can inspect
what failed.
Per-source serialization: each source gets a ``threading.Semaphore``
(default size 1, configurable per-source via ``set_concurrency``).
Same shape the existing clients use today (each source defines
its own semaphore). Engine owning them centrally lets a future
Phase E rate-limiter swap the semaphore for a smarter pool.
Per-source delay-between-downloads: default 0 seconds (most
sources don't need it). YouTube currently uses 3s, Qobuz uses
1s the legacy values get configured in via ``set_delay``
when the source registers.
"""
def __init__(self, engine: Any) -> None:
self._engine = engine
# Per-source semaphores + delay state. The first dispatch
# for a source auto-creates a semaphore with concurrency=1
# if the source hasn't been configured explicitly.
self._semaphores: Dict[str, threading.Semaphore] = {}
self._delays: Dict[str, float] = {}
self._last_download_at: Dict[str, float] = {}
self._config_lock = threading.Lock()
# ------------------------------------------------------------------
# Per-source rate-limit configuration
# ------------------------------------------------------------------
def set_concurrency(self, source_name: str, max_concurrent: int) -> None:
"""Set the max number of concurrent downloads for a source.
Default is 1 (serial). Most sources will keep the default
the streaming APIs all rate-limit at the API gateway level
anyway, parallel downloads just trade rate-limit errors for
thread overhead."""
with self._config_lock:
self._semaphores[source_name] = threading.Semaphore(max_concurrent)
def set_delay(self, source_name: str, seconds: float) -> None:
"""Set a minimum delay between successive downloads from the
same source. YouTube uses 3s today (avoid yt-dlp 429s),
Qobuz uses 1s. Other sources use 0 (no delay)."""
with self._config_lock:
self._delays[source_name] = float(seconds)
def _get_semaphore(self, source_name: str) -> threading.Semaphore:
with self._config_lock:
sem = self._semaphores.get(source_name)
if sem is None:
sem = threading.Semaphore(1)
self._semaphores[source_name] = sem
return sem
def _get_delay(self, source_name: str) -> float:
with self._config_lock:
return self._delays.get(source_name, 0.0)
# ------------------------------------------------------------------
# Dispatch — public API
# ------------------------------------------------------------------
def dispatch(
self,
source_name: str,
target_id: Any,
display_name: str,
original_filename: str,
impl_callable: ImplCallable,
extra_record_fields: Optional[Dict[str, Any]] = None,
username_override: Optional[str] = None,
thread_name: Optional[str] = None,
) -> str:
"""Kick off a background download.
Args:
source_name: Canonical source name (e.g. 'youtube',
'tidal'). Used as the engine state key + the
username slot in the record (unless overridden).
target_id: Source-specific identifier (track_id, video_id,
permalink_url, album_foreign_id, etc.). Passed
verbatim to ``impl_callable``.
display_name: Human-readable label for logs / UI.
original_filename: The encoded filename the orchestrator
received (e.g. ``'12345||Song Title'``). Stored in
the record's ``filename`` slot for context-key lookups.
impl_callable: Synchronous function that performs the
actual download. Signature:
``impl_callable(download_id, target_id, display_name) -> Optional[str]``.
Returns the final file path on success or None /
raises on failure.
extra_record_fields: Per-source extras to merge into the
initial record (e.g. ``{'video_id': '...', 'url':
'...', 'title': '...'}`` for YouTube). Used to
preserve source-specific slots that downstream
consumers + status APIs read.
username_override: Use this instead of ``source_name``
in the record's ``username`` slot. Required for
Deezer (legacy ``'deezer_dl'``) every other source
uses the canonical name.
thread_name: Optional thread name for diagnostics. Deezer
uses ``'deezer-dl-<track_id>'`` Phase A pinning
tests catch any drift in this convention.
Returns:
download_id (UUID4 string). The orchestrator polls via
``engine.get_download_status(download_id)`` for progress.
"""
download_id = str(uuid.uuid4())
record: Dict[str, Any] = {
'id': download_id,
'filename': original_filename,
'username': username_override or source_name,
'state': 'Initializing',
'progress': 0.0,
'size': 0,
'transferred': 0,
'speed': 0,
'time_remaining': None,
'file_path': None,
}
if extra_record_fields:
record.update(extra_record_fields)
self._engine.add_record(source_name, download_id, record)
thread = threading.Thread(
target=self._worker_loop,
args=(source_name, download_id, target_id, display_name, impl_callable),
daemon=True,
name=thread_name,
)
thread.start()
return download_id
# ------------------------------------------------------------------
# Worker thread — the lifted boilerplate
# ------------------------------------------------------------------
def _worker_loop(
self,
source_name: str,
download_id: str,
target_id: Any,
display_name: str,
impl_callable: ImplCallable,
) -> None:
"""Runs on the spawned daemon thread. Handles semaphore
acquisition, rate-limit sleep, state lifecycle, exception
capture. The plugin-specific work happens entirely inside
``impl_callable``."""
try:
with self._get_semaphore(source_name):
# Rate-limit delay against the LAST download from
# this source (not just this worker — semaphore
# ensures serial access while delay is configured).
delay = self._get_delay(source_name)
if delay > 0:
last_at = self._last_download_at.get(source_name, 0.0)
elapsed = time.time() - last_at
if last_at > 0 and elapsed < delay:
wait_time = delay - elapsed
logger.info(
"Rate-limit delay for %s: waiting %.1fs before next download",
source_name, wait_time,
)
time.sleep(wait_time)
self._engine.update_record(source_name, download_id, {
'state': 'InProgress, Downloading',
})
try:
file_path = impl_callable(download_id, target_id, display_name)
except Exception as exc:
logger.error(
"%s download %s failed (impl raised): %s",
source_name, download_id, exc,
)
self._mark_terminal(
source_name, download_id,
success=False, error=str(exc),
)
return
self._last_download_at[source_name] = time.time()
if file_path:
# Atomic write — preserve Cancelled if user cancelled
# between impl returning and this write. Same guard
# _mark_terminal uses; Cin flagged both split sites.
self._engine.update_record_unless_state(
source_name, download_id,
{
'state': 'Completed, Succeeded',
'progress': 100.0,
'file_path': file_path,
},
skip_if_state_in=('Cancelled',),
)
logger.info(
"%s download %s completed: %s",
source_name, download_id, file_path,
)
else:
self._mark_terminal(source_name, download_id, success=False)
logger.error(
"%s download %s failed (impl returned None)",
source_name, download_id,
)
except Exception as exc:
# Defensive — semaphore / sleep shouldn't blow up the
# thread, but if they do the record needs SOME terminal
# state or it sits at 'Initializing' forever.
logger.exception(
"%s worker_loop crashed for download %s: %s",
source_name, download_id, exc,
)
self._mark_terminal(
source_name, download_id,
success=False, error=f'worker crash: {exc}',
)
def _mark_terminal(self, source_name: str, download_id: str,
success: bool, error: Optional[str] = None) -> None:
"""Write a terminal state, but DON'T clobber an explicit
'Cancelled' state set by the user via cancel_download.
Mirrors the legacy per-client guard
(``if state != 'Cancelled': state = 'Errored'``) every
client used to hand-roll inside its thread worker.
Uses ``update_record_unless_state`` so the check + write are
atomic under the engine's per-source lock. Cin caught a race
where a cancel landing between the read-snapshot + write
could overwrite Cancelled back to Errored / Completed.
"""
patch: Dict[str, Any] = {
'state': 'Completed, Succeeded' if success else 'Errored',
}
if error is not None:
patch['error'] = error
self._engine.update_record_unless_state(
source_name, download_id, patch, skip_if_state_in=('Cancelled',),
)

View file

@ -11,6 +11,12 @@ Supports eight modes:
- Deezer Only: Deezer downloads via ARL authentication
- SoundCloud Only: Anonymous SoundCloud downloads (DJ mixes, removed/exclusive tracks)
- Hybrid: Try primary source first, fallback to others
The orchestrator dispatches through ``core.download_plugins.registry``
instead of hardcoded per-source ``[self.soulseek, self.youtube, ...]``
lists. External callers reach individual clients via the generic
``orchestrator.client('<name>')`` accessor (alias-aware), not direct
attribute access.
"""
import asyncio
@ -19,14 +25,9 @@ from pathlib import Path
from utils.logging_config import get_logger
from config.settings import config_manager
from core.soulseek_client import SoulseekClient, TrackResult, AlbumResult, DownloadStatus
from core.youtube_client import YouTubeClient
from core.tidal_download_client import TidalDownloadClient
from core.qobuz_client import QobuzClient
from core.hifi_client import HiFiClient
from core.deezer_download_client import DeezerDownloadClient
from core.lidarr_download_client import LidarrDownloadClient
from core.soundcloud_client import SoundcloudClient
from core.download_engine import DownloadEngine
from core.download_plugins.registry import DownloadPluginRegistry, build_default_registry
from core.download_plugins.types import TrackResult, AlbumResult, DownloadStatus
logger = get_logger("download_orchestrator")
@ -39,19 +40,31 @@ class DownloadOrchestrator:
Routes requests to the appropriate client(s) based on configured mode.
"""
def __init__(self):
"""Initialize orchestrator with all clients.
Each client is initialized independently one failing client doesn't prevent others from working."""
self._init_failures = []
def __init__(self, registry: Optional[DownloadPluginRegistry] = None,
engine: Optional[DownloadEngine] = None):
"""Initialize orchestrator with a plugin registry. Each plugin
is built and registered independently one failing plugin
doesn't prevent others from working. The ``registry`` arg
exists so tests can inject a registry with mock plugins; in
production callers leave it None and get the default.
self.soulseek = self._safe_init('Soulseek', SoulseekClient)
self.youtube = self._safe_init('YouTube', YouTubeClient)
self.tidal = self._safe_init('Tidal', TidalDownloadClient)
self.qobuz = self._safe_init('Qobuz', QobuzClient)
self.hifi = self._safe_init('HiFi', HiFiClient)
self.deezer_dl = self._safe_init('Deezer', DeezerDownloadClient)
self.lidarr = self._safe_init('Lidarr', LidarrDownloadClient)
self.soundcloud = self._safe_init('SoundCloud', SoundcloudClient)
``engine`` is the cross-source state owner. Phase B introduces
it as a held reference; it isn't on any code path yet — Phase
C/D/E/F migrate behavior into it incrementally.
"""
self.registry = registry if registry is not None else build_default_registry()
self.registry.initialize()
self._init_failures = self.registry.init_failures
# Engine — owns cross-source state, threading, search retry,
# rate-limits, fallback. Built in subsequent phases. For Phase
# B it's just an empty registry of plugins so future phases
# can route through it without further orchestrator changes.
self.engine = engine if engine is not None else DownloadEngine()
for source_name, plugin in self.registry.all_plugins():
spec = self.registry.get_spec(source_name)
aliases = spec.aliases if spec else ()
self.engine.register_plugin(source_name, plugin, aliases=aliases)
if self._init_failures:
logger.warning(f"Download clients failed to initialize: {', '.join(self._init_failures)}")
@ -71,15 +84,6 @@ class DownloadOrchestrator:
self.hybrid_secondary,
)
def _safe_init(self, name, cls):
"""Initialize a download client, returning None on failure instead of crashing."""
try:
return cls()
except Exception as e:
logger.error(f"{name} download client failed to initialize: {e}")
self._init_failures.append(name)
return None
def reload_settings(self):
"""Reload settings from config (call after settings change)"""
self.mode = config_manager.get('download_source.mode', 'soulseek')
@ -88,20 +92,28 @@ class DownloadOrchestrator:
self.hybrid_order = config_manager.get('download_source.hybrid_order', ['hifi', 'youtube', 'soulseek'])
# Reload underlying client configs (SLSKD URL, API key, etc.)
if self.soulseek:
self.soulseek._setup_client()
soulseek = self.client('soulseek')
if soulseek:
soulseek._setup_client()
logger.info("Soulseek client config reloaded")
# Reconnect Deezer if ARL changed
deezer_arl = config_manager.get('deezer_download.arl', '')
if deezer_arl and self.deezer_dl:
self.deezer_dl.reconnect(deezer_arl)
self.deezer_dl._quality = config_manager.get('deezer_download.quality', 'flac')
deezer_dl = self.client('deezer_dl')
if deezer_arl and deezer_dl:
deezer_dl.reconnect(deezer_arl)
deezer_dl._quality = config_manager.get('deezer_download.quality', 'flac')
# Reload download path for all clients that cache it
# Reload download path for all clients that cache it.
# Soulseek owns the path config and is reloaded above; every
# other source mirrors that path so files all land in one
# tree. Sources without a `download_path` attribute (e.g.
# Lidarr — pulls into Lidarr's own tree) silently skip.
new_path = Path(config_manager.get('soulseek.download_path', './downloads'))
for client in [self.youtube, self.tidal, self.qobuz, self.hifi, self.deezer_dl, self.soundcloud]:
if client and hasattr(client, 'download_path') and client.download_path != new_path:
for name, client in self.registry.all_plugins():
if name == 'soulseek':
continue
if hasattr(client, 'download_path') and client.download_path != new_path:
client.download_path = new_path
client.download_path.mkdir(parents=True, exist_ok=True)
# YouTube also caches path in yt-dlp opts
@ -111,11 +123,61 @@ class DownloadOrchestrator:
logger.info(f"Download Orchestrator settings reloaded - Mode: {self.mode}")
def _client(self, name):
"""Get a client by name, returning None if not initialized."""
return {'soulseek': self.soulseek, 'youtube': self.youtube, 'tidal': self.tidal,
'qobuz': self.qobuz, 'hifi': self.hifi, 'deezer_dl': self.deezer_dl,
'lidarr': self.lidarr, 'soundcloud': self.soundcloud}.get(name)
def client(self, name):
"""Generic accessor for a download source client by name.
Cin's review feedback: external callers should reach into
per-source clients via this method (``orch.client('hifi')``)
instead of attribute access (``orch.hifi``). Resolves both
canonical names (``deezer``) and legacy aliases (``deezer_dl``)
via the registry. Returns None if the source isn't registered
or failed to initialize.
"""
return self.registry.get(name)
# Internal alias kept for legacy callers inside this file.
_client = client
def configured_clients(self) -> dict:
"""Return ``{source_name: client}`` for every download source
that's both initialized AND reports is_configured() == True.
Replaces the legacy per-source iteration pattern Cin called
out `if hasattr(orch, 'soulseek') and orch.soulseek and
orch.soulseek.is_configured(): download_clients['soulseek']
= orch.soulseek` repeated for each source.
"""
result = {}
for name, client in self.registry.all_plugins():
try:
if not hasattr(client, 'is_configured') or client.is_configured():
result[name] = client
except Exception as exc:
logger.debug("%s is_configured raised: %s", name, exc)
return result
def reload_instances(self, source: str = None) -> bool:
"""Reload a source's instance config (e.g. HiFi instance list,
Qobuz session restore). Generic dispatch caller passes the
source name instead of reaching for ``orch.hifi.reload_instances()``.
When ``source`` is None, reloads every source that has a
``reload_instances`` method.
"""
sources = [source] if source else list(self.registry.names())
ok = True
for name in sources:
client = self.client(name)
if client is None:
continue
if not hasattr(client, 'reload_instances'):
continue
try:
client.reload_instances()
except Exception as exc:
logger.warning("%s reload_instances failed: %s", name, exc)
ok = False
return ok
def is_configured(self) -> bool:
"""
@ -132,12 +194,18 @@ class DownloadOrchestrator:
return False
def get_source_status(self) -> dict:
"""Return configured status for each download source."""
return {name: (c.is_configured() if c else False)
for name, c in [('soulseek', self.soulseek), ('youtube', self.youtube),
('tidal', self.tidal), ('qobuz', self.qobuz),
('hifi', self.hifi), ('deezer_dl', self.deezer_dl),
('lidarr', self.lidarr), ('soundcloud', self.soundcloud)]}
"""Return configured status for each download source.
Keys preserve the legacy ``deezer_dl`` alias used by the
frontend status indicators and per-source dispatch strings,
so callers reading specific keys keep working unchanged.
"""
status = {}
for name in self.registry.names():
client = self.registry.get(name)
key = 'deezer_dl' if name == 'deezer' else name
status[key] = client.is_configured() if client else False
return status
async def check_connection(self) -> bool:
"""
@ -149,7 +217,7 @@ class DownloadOrchestrator:
if client and self.mode != 'hybrid':
return await client.check_connection()
elif self.mode == 'hybrid':
sources_to_check = self.hybrid_order if self.hybrid_order else ['soulseek', 'youtube', 'tidal', 'qobuz', 'hifi', 'deezer_dl', 'lidarr', 'soundcloud']
sources_to_check = self.hybrid_order if self.hybrid_order else self.registry.names()
results = {}
for source in sources_to_check:
client = self._client(source)
@ -168,77 +236,62 @@ class DownloadOrchestrator:
return False
def _normalize_source_name(self, name: str) -> Optional[str]:
"""Convert a possibly-aliased source name (e.g. legacy
``'deezer_dl'``) to the canonical registry name (``'deezer'``).
Returns None if the input matches neither a canonical name
nor an alias.
Cin's review caught a bug where legacy alias values from
config (hybrid_order containing ``'deezer_dl'``) silently
dropped Deezer from hybrid mode because the canonical-name
membership check rejected the alias.
"""
spec = self.registry.get_spec(name) if name else None
return spec.name if spec else None
def _resolve_source_chain(self) -> List[str]:
"""Order the configured sources for hybrid mode. Prefers
``hybrid_order`` config; falls back to legacy
primary/secondary pair when no order set. Normalizes alias
names through the registry so legacy ``deezer_dl`` config
values resolve correctly to the canonical ``deezer`` plugin."""
if self.hybrid_order:
chain = []
seen = set()
for raw in self.hybrid_order:
canonical = self._normalize_source_name(raw)
if canonical and canonical not in seen:
chain.append(canonical)
seen.add(canonical)
return chain
primary = self._normalize_source_name(self.hybrid_primary) or 'soulseek'
secondary = self._normalize_source_name(self.hybrid_secondary) or 'soulseek'
if secondary == primary:
secondary = next(
(name for name in self.registry.names() if name != primary),
'soulseek',
)
chain = [primary, secondary]
if not chain:
chain = ['soulseek']
return chain
async def search(self, query: str, timeout: int = None, progress_callback=None) -> Tuple[List[TrackResult], List[AlbumResult]]:
"""
Search for tracks using configured source(s).
Args:
query: Search query
timeout: Search timeout (for Soulseek)
progress_callback: Progress callback (for Soulseek)
Returns:
Tuple of (track_results, album_results)
"""
source_names = {'soulseek': 'Soulseek', 'youtube': 'YouTube', 'tidal': 'Tidal',
'qobuz': 'Qobuz', 'hifi': 'HiFi', 'deezer_dl': 'Deezer', 'lidarr': 'Lidarr',
'soundcloud': 'SoundCloud'}
"""Search for tracks using configured source(s). Single-source
modes route directly; hybrid mode delegates to
``engine.search_with_fallback`` which tries the chain in order."""
if self.mode != 'hybrid':
client = self._client(self.mode)
if not client:
logger.error(f"{source_names.get(self.mode, self.mode)} client not available (failed to initialize)")
logger.error(f"{self.registry.display_name(self.mode)} client not available (failed to initialize)")
return [], []
logger.info(f"Searching {source_names.get(self.mode, self.mode)}: {query}")
logger.info(f"Searching {self.registry.display_name(self.mode)}: {query}")
return await client.search(query, timeout, progress_callback)
elif self.mode == 'hybrid':
clients = {name: self._client(name) for name in source_names}
# Build ordered source list: prefer hybrid_order, fall back to legacy primary/secondary
if self.hybrid_order:
source_order = [s for s in self.hybrid_order if s in clients]
else:
primary = self.hybrid_primary if self.hybrid_primary in clients else 'soulseek'
secondary = self.hybrid_secondary if self.hybrid_secondary in clients else 'soulseek'
if secondary == primary:
secondary = next((name for name in clients if name != primary), 'soulseek')
source_order = [primary, secondary]
if not source_order:
source_order = ['soulseek']
logger.info(f"Hybrid search ({''.join(source_order)}): {query}")
# Try each source in priority order (skip unconfigured/unavailable ones)
for i, source_name in enumerate(source_order):
client = clients.get(source_name)
if not client:
logger.info(f"Skipping {source_name} (not available)")
continue
if hasattr(client, 'is_configured') and not client.is_configured():
logger.info(f"Skipping {source_name} (not configured)")
continue
try:
if i == 0:
logger.info(f"Trying {source_name} (priority {i+1}): {query}")
else:
logger.info(f"Trying {source_name} (priority {i+1}): {query}")
tracks, albums = await client.search(query, timeout, progress_callback)
if tracks:
logger.info(f"{source_name} found {len(tracks)} tracks")
return (tracks, albums)
except Exception as e:
logger.warning(f"{source_name} search failed: {e}")
# Nothing found from any source
logger.warning(f"Hybrid search: all sources ({', '.join(source_order)}) found nothing for: {query}")
return ([], [])
# Fallback: empty results
return ([], [])
chain = self._resolve_source_chain()
logger.info(f"Hybrid search ({''.join(chain)}): {query}")
return await self.engine.search_with_fallback(query, chain, timeout, progress_callback)
async def search_and_download_best(self, query: str, expected_track=None) -> Optional[str]:
"""
@ -316,7 +369,8 @@ class DownloadOrchestrator:
elif is_streaming:
filtered_results = tracks
else:
filtered_results = self.soulseek.filter_results_by_quality_preference(tracks) if self.soulseek else tracks
soulseek = self.client('soulseek')
filtered_results = soulseek.filter_results_by_quality_preference(tracks) if soulseek else tracks
if not filtered_results:
logger.warning(f"No suitable quality results found for: {query}")
@ -339,105 +393,47 @@ class DownloadOrchestrator:
Download a track using the appropriate client.
Args:
username: Username (or "youtube" for YouTube)
filename: Filename or YouTube video ID
username: Source-name string for streaming sources
(e.g. ``'youtube'``, ``'tidal'``, ``'deezer_dl'``)
OR the actual slskd peer username for Soulseek.
filename: Filename / video ID / track ID encoding (source-specific)
file_size: File size estimate
Returns:
download_id: Unique download ID for tracking
"""
# Detect which client to use based on username
source_map = {'youtube': self.youtube, 'tidal': self.tidal, 'qobuz': self.qobuz,
'hifi': self.hifi, 'deezer_dl': self.deezer_dl, 'lidarr': self.lidarr,
'soundcloud': self.soundcloud}
source_names = {'youtube': 'YouTube', 'tidal': 'Tidal', 'qobuz': 'Qobuz',
'hifi': 'HiFi', 'deezer_dl': 'Deezer', 'lidarr': 'Lidarr',
'soundcloud': 'SoundCloud'}
if username in source_map:
client = source_map[username]
# Streaming sources are dispatched by name match; anything
# unrecognized falls through to Soulseek (peer username case).
spec = self.registry.get_spec(username) if username else None
if spec is not None and spec.name != 'soulseek':
client = self.registry.get(spec.name)
if not client:
raise RuntimeError(f"{source_names[username]} download client not available (failed to initialize)")
logger.info(f"Downloading from {source_names[username]}: {filename}")
raise RuntimeError(f"{spec.display_name} download client not available (failed to initialize)")
logger.info(f"Downloading from {spec.display_name}: {filename}")
return await client.download(username, filename, file_size)
else:
if not self.soulseek:
raise RuntimeError("Soulseek client not available (failed to initialize)")
logger.info(f"Downloading from Soulseek: {filename}")
return await self.soulseek.download(username, filename, file_size)
soulseek = self.registry.get('soulseek')
if not soulseek:
raise RuntimeError("Soulseek client not available (failed to initialize)")
logger.info(f"Downloading from Soulseek: {filename}")
return await soulseek.download(username, filename, file_size)
async def get_all_downloads(self) -> List[DownloadStatus]:
"""
Get all active downloads from all sources.
Returns:
List of DownloadStatus objects
"""
# Get downloads from all available sources
all_downloads = []
for client in [self.soulseek, self.youtube, self.tidal, self.qobuz, self.hifi, self.deezer_dl, self.lidarr, self.soundcloud]:
if client:
try:
all_downloads.extend(await client.get_all_downloads())
except Exception:
pass
return all_downloads
"""Aggregated view across every source. Delegates to the
engine, which iterates registered plugins."""
return await self.engine.get_all_downloads()
async def get_download_status(self, download_id: str) -> Optional[DownloadStatus]:
"""
Get status of a specific download.
Args:
download_id: Download ID to query
Returns:
DownloadStatus object or None if not found
"""
# Try each source until we find the download
for client in [self.soulseek, self.youtube, self.tidal, self.qobuz, self.hifi, self.deezer_dl, self.lidarr, self.soundcloud]:
if not client:
continue
try:
status = await client.get_download_status(download_id)
if status:
return status
except Exception:
pass
return None
"""Find a download by id across every source. Delegates to
the engine."""
return await self.engine.get_download_status(download_id)
async def cancel_download(self, download_id: str, username: str = None, remove: bool = False) -> bool:
"""
Cancel an active download.
Args:
download_id: Download ID to cancel
username: Username hint (optional)
remove: Whether to remove from active downloads
Returns:
True if cancelled successfully
"""
# If username is provided, route directly to that source
source_map = {'youtube': self.youtube, 'tidal': self.tidal, 'qobuz': self.qobuz,
'hifi': self.hifi, 'deezer_dl': self.deezer_dl, 'lidarr': self.lidarr,
'soundcloud': self.soundcloud}
if username in source_map:
client = source_map[username]
return await client.cancel_download(download_id, username, remove) if client else False
elif username:
return await self.soulseek.cancel_download(download_id, username, remove) if self.soulseek else False
# Otherwise, try all available sources
for client in [self.soulseek, self.youtube, self.tidal, self.qobuz, self.hifi, self.deezer_dl, self.lidarr, self.soundcloud]:
if not client:
continue
try:
if await client.cancel_download(download_id, username, remove):
return True
except Exception:
pass
return False
"""Cancel an active download. Delegates to the engine, which
handles source-hint routing (streaming source name direct
plugin, unknown name Soulseek as peer username, no hint
try every plugin)."""
return await self.engine.cancel_download(download_id, username, remove)
async def signal_download_completion(self, download_id: str, username: str, remove: bool = True) -> bool:
"""
@ -452,40 +448,16 @@ class DownloadOrchestrator:
True if successful
"""
# This is Soulseek-specific, so only call on Soulseek client
if not self.soulseek:
soulseek = self.client('soulseek')
if not soulseek:
return False
return await self.soulseek.signal_download_completion(download_id, username, remove)
return await soulseek.signal_download_completion(download_id, username, remove)
async def clear_all_completed_downloads(self) -> bool:
"""
Clear all completed downloads from both sources.
Returns:
True if successful
"""
results = []
for name, client in [
("soulseek", self.soulseek),
("youtube", self.youtube),
("tidal", self.tidal),
("qobuz", self.qobuz),
("hifi", self.hifi),
("deezer_dl", self.deezer_dl),
("lidarr", self.lidarr),
("soundcloud", self.soundcloud),
]:
if not client:
continue
if hasattr(client, "is_configured") and not client.is_configured():
logger.debug("Skipping %s clear_all_completed_downloads (not configured)", name)
continue
try:
results.append(await client.clear_all_completed_downloads())
except Exception as exc:
logger.warning("%s clear_all_completed_downloads failed: %s", name, exc)
results.append(False)
return all(results) if results else True
"""Clear completed downloads from every source. Delegates
to the engine, which skips unconfigured plugins and treats
per-plugin failures as False (not an exception)."""
return await self.engine.clear_all_completed_downloads()
# ===== Soulseek-specific methods (for backwards compatibility) =====
# These are internal methods that some parts of the codebase use directly
@ -503,9 +475,10 @@ class DownloadOrchestrator:
Returns:
API response
"""
if not self.soulseek:
soulseek = self.client('soulseek')
if not soulseek:
raise RuntimeError("Soulseek client not available (failed to initialize)")
return await self.soulseek._make_request(method, endpoint, **kwargs)
return await soulseek._make_request(method, endpoint, **kwargs)
async def _make_direct_request(self, method: str, endpoint: str, **kwargs):
"""
@ -520,9 +493,10 @@ class DownloadOrchestrator:
Returns:
API response
"""
if not self.soulseek:
soulseek = self.client('soulseek')
if not soulseek:
raise RuntimeError("Soulseek client not available (failed to initialize)")
return await self.soulseek._make_direct_request(method, endpoint, **kwargs)
return await soulseek._make_direct_request(method, endpoint, **kwargs)
async def clear_all_searches(self) -> bool:
"""
@ -531,7 +505,8 @@ class DownloadOrchestrator:
Returns:
True if successful
"""
return await self.soulseek.clear_all_searches() if self.soulseek else True
soulseek = self.client('soulseek')
return await soulseek.clear_all_searches() if soulseek else True
async def maintain_search_history_with_buffer(self, keep_searches: int = 50, trigger_threshold: int = 200) -> bool:
"""
@ -544,15 +519,54 @@ class DownloadOrchestrator:
Returns:
True if successful
"""
return await self.soulseek.maintain_search_history_with_buffer(keep_searches, trigger_threshold) if self.soulseek else True
soulseek = self.client('soulseek')
return await soulseek.maintain_search_history_with_buffer(keep_searches, trigger_threshold) if soulseek else True
async def cancel_all_downloads(self) -> bool:
"""Cancel and remove all downloads from all sources."""
"""Cancel and remove all downloads from all sources.
Note: YouTube is intentionally excluded from this loop in the
legacy implementation preserved here. (yt-dlp downloads
run as detached subprocesses and don't share the
``cancel_all_downloads`` semantics the streaming sources
use.) Sources without ``cancel_all_downloads`` fall back to
``clear_all_completed_downloads``.
"""
ok = True
for client in [self.soulseek, self.tidal, self.qobuz, self.hifi, self.deezer_dl, self.lidarr, self.soundcloud]:
if client:
try:
await client.cancel_all_downloads() if hasattr(client, 'cancel_all_downloads') else await client.clear_all_completed_downloads()
except Exception:
ok = False
for name, client in self.registry.all_plugins():
if name == 'youtube':
continue
try:
await client.cancel_all_downloads() if hasattr(client, 'cancel_all_downloads') else await client.clear_all_completed_downloads()
except Exception:
ok = False
return ok
# ---------------------------------------------------------------------------
# Singleton accessor — mirrors Cin's metadata engine pattern
# (``get_metadata_engine()``). Callers that don't need a custom
# registry use this instead of instantiating DownloadOrchestrator
# directly. web_server.py constructs the singleton at startup and
# exposes it via the ``download_orchestrator`` global.
# ---------------------------------------------------------------------------
_default_orchestrator: Optional['DownloadOrchestrator'] = None
def get_download_orchestrator() -> 'DownloadOrchestrator':
"""Return (lazily creating) the process-wide DownloadOrchestrator
singleton. Mirrors the ``get_metadata_engine()`` pattern Cin used
for the metadata engine refactor."""
global _default_orchestrator
if _default_orchestrator is None:
_default_orchestrator = DownloadOrchestrator()
return _default_orchestrator
def set_download_orchestrator(orchestrator: 'DownloadOrchestrator') -> None:
"""Set the process-wide singleton. Used by web_server.py at boot
to install the orchestrator it constructs as the default for
callers that grab via ``get_download_orchestrator()``."""
global _default_orchestrator
_default_orchestrator = orchestrator

View file

@ -0,0 +1,34 @@
"""Download source plugin contract + registry.
This package defines the canonical interface every download source
(Soulseek, YouTube, Tidal, Qobuz, HiFi, Deezer, Lidarr, SoundCloud,
and future additions like Usenet) must satisfy. The orchestrator
dispatches through this contract instead of hardcoded
`if self.youtube ... elif self.tidal ...` chains.
This is the foundation step of a multi-commit refactor. Subsequent
commits extract shared logic (background download worker, search
query normalization, post-processing context building) into the
contract so adding a new source becomes a one-class plugin instead
of a 700+ LOC copy-paste loop.
See `core/download_plugins/base.py` for the protocol contract and
`core/download_plugins/registry.py` for the dispatch entry point.
"""
from core.download_plugins.base import DownloadSourcePlugin
# NOTE: DownloadPluginRegistry is intentionally NOT re-exported here.
# Importing the registry triggers eager imports of every client class
# (see registry.py for why eager — test fixtures inject mock modules
# at collection time and we need real bindings before that). Clients
# inherit from DownloadSourcePlugin (Cin's review feedback — visible
# contract conformance), so importing the package via ``from
# core.download_plugins import DownloadSourcePlugin`` from a client
# file would create a circular import if registry came along for the
# ride. Callers that need the registry import it directly:
# from core.download_plugins.registry import DownloadPluginRegistry
__all__ = [
"DownloadSourcePlugin",
]

View file

@ -0,0 +1,133 @@
"""Canonical contract every download source plugin must satisfy.
`DownloadSourcePlugin` is a structural Protocol any class that
implements these methods with matching signatures is automatically
treated as a download source. No inheritance required, no manual
registration required beyond the registry entry.
The protocol is intentionally narrow only the methods the
orchestrator dispatches generically across all sources. Source-
specific extras (Soulseek's slskd internals, Lidarr's album-only
flow, etc.) stay on the underlying client and are accessed through
the registry's typed accessor.
This file is the FOUNDATION step. Existing client classes
(SoulseekClient, YouTubeClient, TidalDownloadClient, etc.) already
conform structurally they grew the same shape independently
because every consumer site needed the same calls. This file just
makes the implicit contract explicit so:
- Type checkers can flag drift if a new source forgets a method.
- The orchestrator can iterate plugins generically instead of
hardcoding `[self.soulseek, self.youtube, ...]` everywhere.
- Future PRs can move shared logic INTO the contract (a base
class with default implementations) without changing the
signature surface every consumer already depends on.
"""
from __future__ import annotations
from typing import TYPE_CHECKING, List, Optional, Protocol, Tuple, runtime_checkable
# Soulseek client owns the canonical TrackResult / AlbumResult /
# DownloadStatus dataclasses — every other source already imports
# from there. We only need them for type annotations on this
# protocol; using TYPE_CHECKING avoids a circular import once the
# clients themselves inherit from DownloadSourcePlugin (Cin's
# review feedback — clients explicitly declare conformance instead
# of relying on structural typing).
if TYPE_CHECKING:
from core.download_plugins.types import AlbumResult, DownloadStatus, TrackResult
@runtime_checkable
class DownloadSourcePlugin(Protocol):
"""Structural contract for a download source.
`runtime_checkable` lets `isinstance(client, DownloadSourcePlugin)`
work for the conformance test, but it ONLY checks method names
not signatures or async-ness. The conformance test in
``tests/test_download_plugin_conformance.py`` does the deeper
signature check.
"""
# ------------------------------------------------------------------
# Configuration / lifecycle
# ------------------------------------------------------------------
def is_configured(self) -> bool:
"""Return True iff this source has the credentials / settings
it needs to function. Used by the orchestrator to skip
unconfigured sources during hybrid fallback."""
...
async def check_connection(self) -> bool:
"""Probe the source's API / endpoint. Return True if the
source is reachable. May make a live network call."""
...
# ------------------------------------------------------------------
# Search
# ------------------------------------------------------------------
async def search(
self,
query: str,
timeout: Optional[int] = None,
progress_callback=None,
) -> Tuple[List[TrackResult], List[AlbumResult]]:
"""Search the source for tracks (and albums where supported).
Returns a tuple of (track_results, album_results). Either
list may be empty. Sources that don't expose album-level
search return ``[]`` as the second element.
"""
...
# ------------------------------------------------------------------
# Download
# ------------------------------------------------------------------
async def download(
self,
username: str,
filename: str,
file_size: int = 0,
) -> Optional[str]:
"""Kick off a download. Returns a download_id string the
caller can poll via ``get_download_status``. Returns ``None``
if the source can't / won't handle this download.
``username`` is the source-name string for streaming sources
(e.g. ``'youtube'``, ``'tidal'``) and the actual slskd peer
username for Soulseek. ``filename`` is source-specific
Soulseek file path, YouTube ``video_id||title``, Tidal /
Qobuz ``track_id||display``, etc.
"""
...
async def get_all_downloads(self) -> List[DownloadStatus]:
"""Return live status of all downloads currently tracked by
this source. The orchestrator concatenates results from
every plugin to build the global download list."""
...
async def get_download_status(self, download_id: str) -> Optional[DownloadStatus]:
"""Return status for a single download or ``None`` if this
source doesn't know about that download_id."""
...
async def cancel_download(
self,
download_id: str,
username: Optional[str] = None,
remove: bool = False,
) -> bool:
"""Cancel an active download. ``remove=True`` also drops
the row from the source's active-downloads tracking."""
...
async def clear_all_completed_downloads(self) -> bool:
"""Drop completed downloads from active tracking. Sources
that don't keep completed history return True with no-op."""
...

View file

@ -0,0 +1,192 @@
"""Plugin registry — single source of truth for which download
sources exist, what their canonical names are, and which client
class implements each.
Replaces the orchestrator's hardcoded ``[self.soulseek,
self.youtube, self.tidal, ...]`` lists and ``source_map`` dicts
that historically had to be touched in 6+ places to add a source.
With the registry:
- One ``register()`` call adds a source to every dispatch path.
- Iteration helpers replace hand-maintained lists.
- The orchestrator stays oblivious to source-specific quirks.
- Adding Usenet (planned) becomes a one-line registry entry plus
the new client class no orchestrator changes.
This is the foundation step. Subsequent commits move shared logic
(thread workers, search query normalization, post-processing
context building) out of the orchestrator and the per-source
clients into helpers the registry exposes.
"""
from __future__ import annotations
from dataclasses import dataclass, field
from typing import Callable, Dict, Iterator, List, Optional, Tuple
from utils.logging_config import get_logger
from core.download_plugins.base import DownloadSourcePlugin
# Eager client imports — keep the import-order behavior the orchestrator
# had before this refactor. Some integration tests inject mock modules
# into ``sys.modules`` at collection time (see
# ``tests/test_tidal_search_shortening.py``); making the registry's
# factories lazy-import would cause tidalapi-dependent code to bind to
# whichever ``tidalapi`` object happens to be in ``sys.modules`` at the
# moment ``DownloadOrchestrator()`` is constructed — which is later
# than the legacy module-top imports here. Importing everything at
# registry-load time pins the bindings the same way the legacy
# orchestrator did.
from core.deezer_download_client import DeezerDownloadClient
from core.hifi_client import HiFiClient
from core.lidarr_download_client import LidarrDownloadClient
from core.qobuz_client import QobuzClient
from core.soulseek_client import SoulseekClient
from core.soundcloud_client import SoundcloudClient
from core.tidal_download_client import TidalDownloadClient
from core.youtube_client import YouTubeClient
logger = get_logger("download_plugins.registry")
@dataclass(frozen=True)
class PluginSpec:
"""Static descriptor for a download source. The ``factory`` is
a zero-arg callable that builds the client instance kept as a
callable rather than a class so each source can do its own
setup (e.g. SoulseekClient calls ``_setup_client`` after init,
Deezer reads ARL from config). ``aliases`` lets the registry
accept multiple historical names (e.g. ``deezer_dl`` is the
legacy alias for ``deezer``)."""
name: str
factory: Callable[[], DownloadSourcePlugin]
display_name: str
aliases: Tuple[str, ...] = field(default_factory=tuple)
class DownloadPluginRegistry:
"""Holds the live plugin instances + name → instance lookup.
Construction is two-phase:
1. Specs are registered (cheap just stores callable refs).
2. ``initialize()`` calls each factory once and stores the
resulting client. Failures are caught and logged so one
broken source doesn't take down the orchestrator (mirrors
the existing ``_safe_init`` behavior).
Iteration helpers (``all_plugins``, ``configured_plugins``)
replace the hand-maintained lists scattered across the
orchestrator's ``get_all_downloads``, ``cancel_all_downloads``,
etc. so adding a source touches the registry alone.
"""
def __init__(self) -> None:
self._specs: Dict[str, PluginSpec] = {}
self._instances: Dict[str, Optional[DownloadSourcePlugin]] = {}
self._init_failures: List[str] = []
def register(self, spec: PluginSpec) -> None:
"""Register a plugin spec under its canonical name + each alias.
Aliases all resolve to the same instance after ``initialize``."""
if spec.name in self._specs:
raise ValueError(f"Plugin already registered: {spec.name}")
self._specs[spec.name] = spec
def initialize(self) -> None:
"""Build every registered plugin's instance. Failures captured
in ``init_failures`` and the slot is set to None so the
orchestrator can skip unavailable sources without crashing."""
for spec in self._specs.values():
try:
instance = spec.factory()
self._instances[spec.name] = instance
except Exception as exc:
logger.error("%s download client failed to initialize: %s", spec.display_name, exc)
self._init_failures.append(spec.display_name)
self._instances[spec.name] = None
@property
def init_failures(self) -> List[str]:
return list(self._init_failures)
def get(self, name: str) -> Optional[DownloadSourcePlugin]:
"""Resolve a name (or alias) to its plugin instance, or
None if the source failed to initialize / isn't registered."""
if not name:
return None
# Direct hit
if name in self._instances:
return self._instances[name]
# Alias lookup
for spec in self._specs.values():
if name in spec.aliases:
return self._instances.get(spec.name)
return None
def get_spec(self, name: str) -> Optional[PluginSpec]:
if name in self._specs:
return self._specs[name]
for spec in self._specs.values():
if name in spec.aliases:
return spec
return None
def display_name(self, name: str) -> str:
spec = self.get_spec(name)
return spec.display_name if spec else name
def names(self) -> List[str]:
"""Canonical names of every registered source (regardless of
whether it initialized successfully)."""
return list(self._specs.keys())
def all_plugins(self) -> Iterator[Tuple[str, DownloadSourcePlugin]]:
"""Yield (name, plugin) for every successfully-initialized
plugin. Replaces the orchestrator's hand-maintained client
lists in get_all_downloads / cancel_all_downloads / etc."""
for name, instance in self._instances.items():
if instance is not None:
yield name, instance
def configured_plugins(self) -> Iterator[Tuple[str, DownloadSourcePlugin]]:
"""Yield (name, plugin) for every initialized AND configured
plugin. Useful for hybrid mode and any operation that should
skip sources the user hasn't set up."""
for name, instance in self.all_plugins():
try:
if instance.is_configured():
yield name, instance
except Exception:
continue
def build_default_registry() -> DownloadPluginRegistry:
"""Construct the registry with SoulSync's eight built-in download
sources. Called once during orchestrator init.
Adding a new source (Usenet, etc.) means adding one ``register``
call here no orchestrator changes required.
The factory itself is just the class constructor clients are
imported eagerly at the top of this module so they bind to the
real third-party libs (tidalapi, etc.) at import time, not at
factory-call time. See the import-block comment above for why.
"""
registry = DownloadPluginRegistry()
registry.register(PluginSpec(name='soulseek', factory=SoulseekClient, display_name='Soulseek'))
registry.register(PluginSpec(name='youtube', factory=YouTubeClient, display_name='YouTube'))
registry.register(PluginSpec(name='tidal', factory=TidalDownloadClient, display_name='Tidal'))
registry.register(PluginSpec(name='qobuz', factory=QobuzClient, display_name='Qobuz'))
registry.register(PluginSpec(name='hifi', factory=HiFiClient, display_name='HiFi'))
# 'deezer_dl' is the legacy name used in config + per-source dispatch
# strings (e.g. orchestrator's ``source_map``). Canonical name is
# ``deezer`` so future-facing code reads naturally.
registry.register(PluginSpec(name='deezer', factory=DeezerDownloadClient, display_name='Deezer',
aliases=('deezer_dl',)))
registry.register(PluginSpec(name='lidarr', factory=LidarrDownloadClient, display_name='Lidarr'))
registry.register(PluginSpec(name='soundcloud',factory=SoundcloudClient, display_name='SoundCloud'))
return registry

View file

@ -0,0 +1,199 @@
"""Shared dataclasses for the download-plugin contract.
Every download source returns the same shape for search hits and
download status the four classes here are the canonical types
that the ``DownloadSourcePlugin`` Protocol exchanges. Living in
this neutral module (rather than ``core/soulseek_client.py`` where
they grew up by accident) means a new plugin doesn't have to import
from a sibling source just to satisfy the contract.
Move history: extracted from ``core.soulseek_client`` so plugins
import from a neutral package per Cin's contract-first standard.
"""
from __future__ import annotations
from dataclasses import dataclass
from typing import Any, Dict, List, Optional
from core.imports.filename import parse_filename_metadata
@dataclass
class SearchResult:
"""Base class for search results"""
username: str
filename: str
size: int
bitrate: Optional[int]
duration: Optional[int] # Duration in milliseconds (converted from slskd's seconds)
quality: str
free_upload_slots: int
upload_speed: int
queue_length: int
result_type: str = "track" # "track" or "album"
@property
def quality_score(self) -> float:
quality_weights = {
'flac': 1.0,
'mp3': 0.8,
'ogg': 0.7,
'aac': 0.6,
'wma': 0.5
}
base_score = quality_weights.get(self.quality.lower(), 0.3)
if self.bitrate:
if self.bitrate >= 320:
base_score += 0.2
elif self.bitrate >= 256:
base_score += 0.1
elif self.bitrate < 128:
base_score -= 0.2
# Free upload slots
if self.free_upload_slots == 0:
base_score -= 0.15
elif self.free_upload_slots > 0:
base_score += 0.05
# Upload speed in bytes/sec (tiered)
if self.upload_speed >= 5_000_000: # ~5 MB/s / 40 Mbps
base_score += 0.15
elif self.upload_speed >= 1_000_000: # ~1 MB/s / 8 Mbps
base_score += 0.10
elif self.upload_speed >= 500_000: # ~500 KB/s / 4 Mbps
base_score += 0.05
elif self.upload_speed < 100_000: # ~100 KB/s / 800 kbps
base_score -= 0.05
# Queue length (graduated penalty)
if self.queue_length > 50:
base_score -= 0.25
elif self.queue_length > 20:
base_score -= 0.15
elif self.queue_length > 10:
base_score -= 0.10
return min(base_score, 1.0)
@dataclass
class TrackResult(SearchResult):
"""Individual track search result"""
artist: Optional[str] = None
title: Optional[str] = None
album: Optional[str] = None
track_number: Optional[int] = None
_source_metadata: Optional[Dict[str, Any]] = None
def __post_init__(self):
self.result_type = "track"
# Try to extract metadata from filename if not provided
if not self.title or not self.artist:
self._parse_filename_metadata()
def _parse_filename_metadata(self):
"""Extract artist, title, album from filename patterns"""
parsed = parse_filename_metadata(self.filename)
if not self.artist and parsed.get("artist"):
self.artist = parsed["artist"]
if not self.title and parsed.get("title"):
self.title = parsed["title"]
if not self.album and parsed.get("album"):
self.album = parsed["album"]
if self.track_number is None:
track_number = parsed.get("track_number")
if track_number is not None:
self.track_number = track_number
@dataclass
class AlbumResult:
"""Album/folder search result containing multiple tracks"""
username: str
album_path: str # Directory path
album_title: str
artist: Optional[str]
track_count: int
total_size: int
tracks: List[TrackResult]
dominant_quality: str # Most common quality in album
year: Optional[str] = None
free_upload_slots: int = 0
upload_speed: int = 0
queue_length: int = 0
result_type: str = "album"
@property
def quality_score(self) -> float:
"""Calculate album quality score based on dominant quality and track count"""
quality_weights = {
'flac': 1.0,
'mp3': 0.8,
'ogg': 0.7,
'aac': 0.6,
'wma': 0.5
}
base_score = quality_weights.get(self.dominant_quality.lower(), 0.3)
# Bonus for complete albums (typically 8-15 tracks)
if 8 <= self.track_count <= 20:
base_score += 0.1
elif self.track_count > 20:
base_score += 0.05
# Free upload slots
if self.free_upload_slots == 0:
base_score -= 0.15
elif self.free_upload_slots > 0:
base_score += 0.05
# Upload speed in bytes/sec (tiered)
if self.upload_speed >= 5_000_000: # ~5 MB/s / 40 Mbps
base_score += 0.15
elif self.upload_speed >= 1_000_000: # ~1 MB/s / 8 Mbps
base_score += 0.10
elif self.upload_speed >= 500_000: # ~500 KB/s / 4 Mbps
base_score += 0.05
elif self.upload_speed < 100_000: # ~100 KB/s / 800 kbps
base_score -= 0.05
# Queue length (graduated penalty)
if self.queue_length > 50:
base_score -= 0.25
elif self.queue_length > 20:
base_score -= 0.15
elif self.queue_length > 10:
base_score -= 0.10
return min(base_score, 1.0)
@property
def size_mb(self) -> int:
"""Album size in MB"""
return self.total_size // (1024 * 1024)
@property
def average_track_size_mb(self) -> float:
"""Average track size in MB"""
if self.track_count > 0:
return self.size_mb / self.track_count
return 0
@dataclass
class DownloadStatus:
id: str
filename: str
username: str
state: str
progress: float
size: int
transferred: int
speed: int
time_remaining: Optional[int] = None
file_path: Optional[str] = None

View file

@ -42,31 +42,31 @@ _TERMINAL_STATUSES = {
}
def cancel_single_download(soulseek_client, run_async: Callable,
def cancel_single_download(download_orchestrator, run_async: Callable,
download_id: str, username: str) -> bool:
"""Cancel one specific slskd download (with `remove=True`)."""
return run_async(soulseek_client.cancel_download(download_id, username, remove=True))
return run_async(download_orchestrator.cancel_download(download_id, username, remove=True))
def cancel_all_active(soulseek_client, run_async: Callable,
def cancel_all_active(download_orchestrator, run_async: Callable,
sweep_callback: Callable[[], None]) -> tuple[bool, str]:
"""Cancel every active slskd download, clear the resulting ones, sweep dirs.
Returns `(success, message)` so the route can map to the right HTTP shape.
"""
cancel_success = run_async(soulseek_client.cancel_all_downloads())
cancel_success = run_async(download_orchestrator.cancel_all_downloads())
if not cancel_success:
return False, "Failed to cancel active downloads."
run_async(soulseek_client.clear_all_completed_downloads())
run_async(download_orchestrator.clear_all_completed_downloads())
sweep_callback()
return True, "All downloads cancelled and cleared."
def clear_finished_active(soulseek_client, run_async: Callable,
def clear_finished_active(download_orchestrator, run_async: Callable,
sweep_callback: Callable[[], None]) -> bool:
"""Clear all terminal transfers from slskd, sweep dirs on success."""
success = run_async(soulseek_client.clear_all_completed_downloads())
success = run_async(download_orchestrator.clear_all_completed_downloads())
if success:
sweep_callback()
return success

View file

@ -25,7 +25,7 @@ and notifies the lifecycle via `on_download_completed(success=False)` so the
worker slot frees up.
Lifted verbatim from web_server.py. Wide dependency surface
(soulseek_client, spotify_client, lifecycle callback, context-key helper,
(download_orchestrator, spotify_client, lifecycle callback, context-key helper,
status updater, DB) all injected via `CandidatesDeps`.
"""
@ -49,7 +49,7 @@ logger = logging.getLogger(__name__)
@dataclass
class CandidatesDeps:
"""Bundle of cross-cutting deps the candidate-fallback logic needs."""
soulseek_client: Any
download_orchestrator: Any
spotify_client: Any
run_async: Callable[..., Any]
get_database: Callable[[], Any]
@ -209,7 +209,7 @@ def attempt_download_with_candidates(task_id, candidates, track, batch_id=None,
# Initiate download
logger.info(f"[Modal Worker] Starting download: {username} / {os.path.basename(filename)}")
download_id = deps.run_async(deps.soulseek_client.download(username, filename, size))
download_id = deps.run_async(deps.download_orchestrator.download(username, filename, size))
if download_id:
# Store context for post-processing with complete Spotify metadata (GUI PARITY)
@ -330,7 +330,7 @@ def attempt_download_with_candidates(task_id, candidates, track, batch_id=None,
logger.warning(f"[Modal Worker] Task {task_id} cancelled after download {download_id} started - attempting to cancel download")
# Try to cancel the download immediately
try:
deps.run_async(deps.soulseek_client.cancel_download(download_id, username, remove=True))
deps.run_async(deps.download_orchestrator.cancel_download(download_id, username, remove=True))
logger.warning(f"Successfully cancelled active download {download_id}")
except Exception as cancel_error:
logger.error(f"Failed to cancel active download {download_id}: {cancel_error}")
@ -340,8 +340,8 @@ def attempt_download_with_candidates(task_id, candidates, track, batch_id=None,
deps.on_download_completed(batch_id, task_id, success=False)
return False
# Store download information - use real download ID from soulseek_client
# CRITICAL FIX: Trust the download ID returned by soulseek_client.download()
# Store download information - use real download ID from download_orchestrator
# CRITICAL FIX: Trust the download ID returned by download_orchestrator.download()
download_tasks[task_id]['download_id'] = download_id
download_tasks[task_id]['username'] = username

View file

@ -42,7 +42,7 @@ logger = logging.getLogger(__name__)
class MasterDeps:
"""Bundle of cross-cutting deps the master worker needs."""
config_manager: Any
soulseek_client: Any
download_orchestrator: Any
run_async: Callable[..., Any]
mb_worker: Any
mb_release_cache: dict
@ -372,7 +372,7 @@ def run_full_missing_tracks_process(batch_id, playlist_id, tracks_json, deps: Ma
_sr.info(f"[Album Pre-flight] Searching for '{artist_name} {album_name}'")
logger.info(f"[Album Pre-flight] Searching Soulseek for complete album: '{artist_name} - {album_name}'")
slsk = deps.soulseek_client.soulseek if hasattr(deps.soulseek_client, 'soulseek') else deps.soulseek_client
slsk = deps.download_orchestrator.client('soulseek') if hasattr(deps.download_orchestrator, 'client') else deps.download_orchestrator
# Try multiple query variations (banned keywords in artist/album name can return 0 results)
album_queries = [f"{artist_name} {album_name}"]

View file

@ -33,7 +33,7 @@ _run_post_processing_worker = None
_start_next_batch_of_downloads = None
_orphaned_download_keys = None
missing_download_executor = None
soulseek_client = None
download_orchestrator = None
def init(
@ -44,12 +44,12 @@ def init(
start_next_batch_of_downloads,
orphaned_download_keys,
missing_download_executor_obj,
soulseek_client_obj,
download_orchestrator_obj,
):
"""Bind web_server-side helpers/globals so the class body can resolve them."""
global _make_context_key, _on_download_completed, _download_track_worker
global _run_post_processing_worker, _start_next_batch_of_downloads
global _orphaned_download_keys, missing_download_executor, soulseek_client
global _orphaned_download_keys, missing_download_executor, download_orchestrator
_make_context_key = make_context_key
_on_download_completed = on_download_completed
_download_track_worker = download_track_worker
@ -57,7 +57,7 @@ def init(
_start_next_batch_of_downloads = start_next_batch_of_downloads
_orphaned_download_keys = orphaned_download_keys
missing_download_executor = missing_download_executor_obj
soulseek_client = soulseek_client_obj
download_orchestrator = download_orchestrator_obj
class WebUIDownloadMonitor:
@ -199,7 +199,7 @@ class WebUIDownloadMonitor:
if op[0] == 'cancel_download':
_, download_id, username = op
logger.debug(f"[Deferred] Cancelling download: {download_id} from {username}")
run_async(soulseek_client.cancel_download(download_id, username, remove=True))
run_async(download_orchestrator.cancel_download(download_id, username, remove=True))
logger.debug(f"[Deferred] Successfully cancelled download {download_id}")
elif op[0] == 'cleanup_orphan':
_, context_key = op
@ -254,8 +254,9 @@ class WebUIDownloadMonitor:
# Get Soulseek downloads from API
transfers_data = None
if soulseek_active and soulseek_client and getattr(soulseek_client, 'soulseek', None) and soulseek_client.soulseek.base_url:
transfers_data = run_async(soulseek_client._make_request('GET', 'transfers/downloads'))
_slsk = download_orchestrator.client('soulseek') if download_orchestrator and hasattr(download_orchestrator, 'client') else None
if soulseek_active and _slsk and _slsk.base_url:
transfers_data = run_async(download_orchestrator._make_request('GET', 'transfers/downloads'))
if transfers_data:
for user_data in transfers_data:
username = user_data.get('username', 'Unknown')
@ -266,31 +267,34 @@ class WebUIDownloadMonitor:
key = _make_context_key(username, file_info.get('filename', ''))
live_transfers[key] = file_info
# Also get non-Soulseek downloads (YouTube/Tidal/Qobuz/HiFi/Deezer/Lidarr)
# Call each client directly to avoid redundant slskd API call through orchestrator
# Also get non-Soulseek downloads via the engine — single
# cross-source aggregation, no per-source iteration.
try:
all_downloads = []
for _dl_client in [soulseek_client.youtube, soulseek_client.tidal, soulseek_client.qobuz,
soulseek_client.hifi, soulseek_client.deezer_dl, soulseek_client.lidarr,
getattr(soulseek_client, 'soundcloud', None)]:
if _dl_client:
try:
all_downloads.extend(run_async(_dl_client.get_all_downloads()))
except Exception:
pass
if download_orchestrator and hasattr(download_orchestrator, 'engine'):
try:
# Exclude soulseek — slskd transfers were already
# pulled via the transfers/downloads endpoint above.
# Without the exclude both fetch paths run, doubling
# the per-tick slskd API hit.
all_downloads = run_async(
download_orchestrator.engine.get_all_downloads(exclude=('soulseek',))
)
except Exception:
pass
for download in all_downloads:
key = _make_context_key(download.username, download.filename)
# Convert DownloadStatus to transfer dict format for monitor compatibility
live_transfers[key] = {
'id': download.id,
'filename': download.filename,
'username': download.username,
'state': download.state,
'percentComplete': download.progress,
'size': download.size,
'bytesTransferred': download.transferred,
'averageSpeed': download.speed,
}
key = _make_context_key(download.username, download.filename)
# Convert DownloadStatus to transfer dict format for monitor compatibility
live_transfers[key] = {
'id': download.id,
'filename': download.filename,
'username': download.username,
'state': download.state,
'percentComplete': download.progress,
'size': download.size,
'bytesTransferred': download.transferred,
'averageSpeed': download.speed,
}
except Exception as yt_error:
logger.error(f"Monitor: Could not fetch streaming source downloads: {yt_error}")

View file

@ -54,7 +54,7 @@ class PostProcessDeps:
always live (no caching of pre-init Spotify clients etc).
"""
config_manager: Any
soulseek_client: Any
download_orchestrator: Any
run_async: Callable
docker_resolve_path: Callable[[str], str]
extract_filename: Callable[[str], str]
@ -196,7 +196,7 @@ def run_post_processing_worker(task_id: str, batch_id: str, deps: PostProcessDep
# Query the download orchestrator for the status which contains the real file path
# CRITICAL FIX: Use the actual download_id designated by the client, not the internal task_id
actual_download_id = task.get('download_id') or task_id
status = deps.run_async(deps.soulseek_client.get_download_status(actual_download_id))
status = deps.run_async(deps.download_orchestrator.get_download_status(actual_download_id))
if status and status.file_path:
real_path = status.file_path
if os.path.exists(real_path):

View file

@ -34,7 +34,7 @@ logger = logging.getLogger(__name__)
@dataclass
class TaskWorkerDeps:
"""Bundle of cross-cutting deps the per-task download worker needs."""
soulseek_client: Any
download_orchestrator: Any
matching_engine: Any
run_async: Callable
try_source_reuse: Callable # (task_id, batch_id, track) -> bool
@ -210,7 +210,7 @@ def download_track_worker(task_id: str, batch_id: Optional[str], deps: TaskWorke
try:
# Perform search with timeout
tracks_result, _ = deps.run_async(deps.soulseek_client.search(query, timeout=30))
tracks_result, _ = deps.run_async(deps.download_orchestrator.search(query, timeout=30))
logger.debug(f"Search completed for task {task_id}, got {len(tracks_result) if tracks_result else 0} results")
# CRITICAL: Check cancellation immediately after search returns
@ -260,7 +260,13 @@ def download_track_worker(task_id: str, batch_id: Optional[str], deps: TaskWorke
search_diagnostics.append(f'"{query}": {result_count} results, {len(candidates)} passed filters but download failed to start')
else:
search_diagnostics.append(f'"{query}": {result_count} results but none passed quality/artist filters')
all_raw_results.extend(tracks_result[:20]) # Keep top results for review
# Strip SoundCloud preview snippets before caching for the
# review modal — the user can't pick something useful from
# a 30s preview clip, and clicking one bypasses validation
# and downloads it anyway.
from core.downloads.validation import filter_soundcloud_previews
_filtered_raw = filter_soundcloud_previews(tracks_result[:20], track)
all_raw_results.extend(_filtered_raw)
else:
search_diagnostics.append(f'"{query}": no results found')
@ -272,24 +278,23 @@ def download_track_worker(task_id: str, batch_id: Optional[str], deps: TaskWorke
# === HYBRID FALLBACK: If primary source failed, try remaining sources directly ===
# The orchestrator's hybrid search stops at the first source with results, even if
# those results all fail quality filtering. Try remaining sources individually.
if getattr(deps.soulseek_client, 'mode', '') == 'hybrid':
if getattr(deps.download_orchestrator, 'mode', '') == 'hybrid':
try:
orch = deps.soulseek_client
orch = deps.download_orchestrator
hybrid_order = getattr(orch, 'hybrid_order', None) or []
if not hybrid_order:
primary = getattr(orch, 'hybrid_primary', 'soulseek')
secondary = getattr(orch, 'hybrid_secondary', '')
hybrid_order = [primary, secondary] if secondary and secondary != primary else [primary]
# Resolve via the orchestrator's generic accessor — the
# legacy per-source attrs were dropped in the registry
# refactor, so getattr(orch, 'soulseek', None) etc. all
# silently returned None and the fallback never fired.
source_clients = {
'soulseek': getattr(orch, 'soulseek', None),
'youtube': getattr(orch, 'youtube', None),
'tidal': getattr(orch, 'tidal', None),
'qobuz': getattr(orch, 'qobuz', None),
'hifi': getattr(orch, 'hifi', None),
'deezer_dl': getattr(orch, 'deezer_dl', None),
'lidarr': getattr(orch, 'lidarr', None),
'soundcloud': getattr(orch, 'soundcloud', None),
name: orch.client(name)
for name in ('soulseek', 'youtube', 'tidal', 'qobuz',
'hifi', 'deezer_dl', 'lidarr', 'soundcloud')
}
# The orchestrator tried sources in order but stopped at the first with results.

View file

@ -1,7 +1,7 @@
"""Soulseek/streaming candidate validation — lifted from web_server.py.
Body is byte-identical to the original. ``matching_engine`` and
``soulseek_client`` are injected via init() because both are
``download_orchestrator`` are injected via init() because both are
constructed in web_server.py and referenced by name throughout
the body.
"""
@ -14,14 +14,51 @@ logger = logging.getLogger(__name__)
# Injected at runtime via init().
matching_engine = None
soulseek_client = None
download_orchestrator = None
def init(matching_engine_obj, soulseek_client_obj):
def init(matching_engine_obj, download_orchestrator_obj):
"""Bind the matching engine and download orchestrator from web_server."""
global matching_engine, soulseek_client
global matching_engine, download_orchestrator
matching_engine = matching_engine_obj
soulseek_client = soulseek_client_obj
download_orchestrator = download_orchestrator_obj
def filter_soundcloud_previews(results, expected_track):
"""Drop SoundCloud preview snippets so they never reach the cache,
the modal, or the auto-download attempt.
SoundCloud serves a ~30s preview clip for tracks gated behind Go+ /
login. yt-dlp accepts the preview as the download payload, the
integrity check catches the truncated file, but the user just sees
"all candidates failed" with previews still listed in the modal
(and clickable for manual retry, which downloads another preview).
Filter at every spot raw search results enter the task: validation
scoring, modal-cache fallback when validation drops everything,
and the not-found raw-results cache. Keep candidates that genuinely
are short (intros, sound effects) when the expected track is also
short.
"""
if not results or not expected_track:
return results
expected_ms = getattr(expected_track, 'duration_ms', 0) or 0
if expected_ms <= 0:
return results
expected_secs = expected_ms / 1000.0
if expected_secs <= 60:
return results
def _is_preview(r):
if getattr(r, 'username', None) != 'soundcloud':
return False
cand_ms = getattr(r, 'duration', None) or 0
if cand_ms <= 0:
return False
cand_secs = cand_ms / 1000.0
return cand_secs < 35 or cand_secs < expected_secs * 0.5
return [r for r in results if not _is_preview(r)]
def get_valid_candidates(results, spotify_track, query):
@ -33,6 +70,13 @@ def get_valid_candidates(results, spotify_track, query):
if not results:
return []
# Pre-filter: drop SoundCloud preview snippets when expected
# duration is non-trivially long. Same helper is also applied at
# the modal-cache fallback path so previews never reach the UI.
results = filter_soundcloud_previews(results, spotify_track)
if not results:
return []
# Streaming sources (YouTube, Tidal, Qobuz, HiFi, Deezer, SoundCloud) return structured API results
# with proper artist/title metadata — score using the same matching engine as Soulseek
_streaming_sources = ("youtube", "tidal", "qobuz", "hifi", "deezer_dl", "soundcloud")
@ -45,7 +89,12 @@ def get_valid_candidates(results, spotify_track, query):
# Detect if the expected track is a specific version (live, remix, acoustic, etc.)
expected_title_lower = (expected_title or '').lower()
_version_keywords = ['remix', 'live', 'acoustic', 'instrumental', 'radio edit',
'extended', 'slowed', 'sped up', 'reverb', 'karaoke']
'extended', 'slowed', 'sped up', 'reverb', 'karaoke',
# Producer-tag noise common on SoundCloud — "type
# beat" is an instrumental track produced in
# someone's style, tagged with the artist name to
# game search. NEVER the real song.
'type beat']
expected_is_version = any(kw in expected_title_lower for kw in _version_keywords)
scored = []
@ -151,8 +200,8 @@ def get_valid_candidates(results, spotify_track, query):
quality_filtered_candidates = initial_candidates
else:
# Filter by user's quality profile before artist verification (Soulseek only)
# Use existing soulseek_client to avoid re-initializing (which accesses download_path filesystem)
quality_filtered_candidates = soulseek_client.soulseek.filter_results_by_quality_preference(initial_candidates)
# Use existing download_orchestrator to avoid re-initializing (which accesses download_path filesystem)
quality_filtered_candidates = download_orchestrator.client('soulseek').filter_results_by_quality_preference(initial_candidates)
# IMPORTANT: Respect empty results from quality filter
# If user has strict quality requirements (e.g., FLAC-only with fallback disabled),

View file

@ -2,7 +2,7 @@
Body is byte-identical to the original. Wishlist helpers are
direct imports from core.wishlist.*; runtime state comes from
core.runtime_state; automation_engine, soulseek_client, and the
core.runtime_state; automation_engine, download_orchestrator, and the
sweep helper are injected via init() because they are constructed
in web_server.py.
"""
@ -29,15 +29,15 @@ logger = logging.getLogger(__name__)
# Injected at runtime via init().
automation_engine = None
soulseek_client = None
download_orchestrator = None
_sweep_empty_download_directories = None
def init(engine, soulseek_client_obj, sweep_fn):
def init(engine, download_orchestrator_obj, sweep_fn):
"""Bind shared singletons + the sweep helper from web_server."""
global automation_engine, soulseek_client, _sweep_empty_download_directories
global automation_engine, download_orchestrator, _sweep_empty_download_directories
automation_engine = engine
soulseek_client = soulseek_client_obj
download_orchestrator = download_orchestrator_obj
_sweep_empty_download_directories = sweep_fn
@ -185,7 +185,7 @@ def _process_failed_tracks_to_wishlist_exact(batch_id):
# Auto-cleanup: Clear completed downloads from slskd
try:
logger.info(f"[Auto-Cleanup] Clearing completed downloads from slskd after batch {batch_id}")
run_async(soulseek_client.clear_all_completed_downloads())
run_async(download_orchestrator.clear_all_completed_downloads())
logger.info("[Auto-Cleanup] Completed downloads cleared from slskd")
except Exception as cleanup_error:
logger.warning(f"[Auto-Cleanup] Failed to clear completed downloads: {cleanup_error}")

View file

@ -33,7 +33,7 @@ import requests as http_requests
from utils.logging_config import get_logger
from config.settings import config_manager
from core.soulseek_client import TrackResult, AlbumResult, DownloadStatus
from core.download_plugins.types import TrackResult, AlbumResult, DownloadStatus
logger = get_logger("hifi_client")
@ -86,7 +86,10 @@ DEFAULT_INSTANCES = [
]
class HiFiClient:
from core.download_plugins.base import DownloadSourcePlugin
class HiFiClient(DownloadSourcePlugin):
"""
HiFi API client for searching and downloading lossless music.
Uses public hifi-api instances (Tidal backend) no auth required.
@ -110,9 +113,7 @@ class HiFiClient:
'Accept': 'application/json',
})
self.active_downloads: Dict[str, Dict[str, Any]] = {}
self._download_lock = threading.Lock()
self._engine = None
self.shutdown_check = None
self._last_api_call = 0
@ -125,6 +126,9 @@ class HiFiClient:
def set_shutdown_check(self, check_callable):
self.shutdown_check = check_callable
def set_engine(self, engine):
self._engine = engine
def _load_instances_from_db(self):
try:
from database.music_database import get_database
@ -571,71 +575,34 @@ class HiFiClient:
)
async def download(self, username: str, filename: str, file_size: int = 0) -> Optional[str]:
if '||' not in filename:
logger.error(f"Invalid filename format: {filename}")
return None
if self._engine is None:
# Raise rather than return None so the orchestrator's
# download_with_fallback surfaces a real warning + tries
# the next source. Returning None silently dropped the
# download with no user feedback (per JohnBaumb).
raise RuntimeError("HiFi client has no engine reference — cannot dispatch download")
track_id_str, display_name = filename.split('||', 1)
try:
if '||' not in filename:
logger.error(f"Invalid filename format: {filename}")
return None
track_id_str, display_name = filename.split('||', 1)
try:
track_id = int(track_id_str)
except ValueError:
logger.error(f"Invalid track ID: {track_id_str}")
return None
download_id = str(uuid.uuid4())
with self._download_lock:
self.active_downloads[download_id] = {
'id': download_id,
'filename': filename,
'username': 'hifi',
'state': 'Initializing',
'progress': 0.0,
'size': 0,
'transferred': 0,
'speed': 0,
'time_remaining': None,
'track_id': track_id,
'display_name': display_name,
'file_path': None,
}
thread = threading.Thread(
target=self._download_worker,
args=(download_id, track_id, display_name),
daemon=True,
)
thread.start()
return download_id
except Exception as e:
logger.error(f"Failed to start HiFi download: {e}")
track_id = int(track_id_str)
except ValueError:
logger.error(f"Invalid track ID: {track_id_str}")
return None
def _download_worker(self, download_id: str, track_id: int, display_name: str):
try:
with self._download_lock:
if download_id in self.active_downloads:
self.active_downloads[download_id]['state'] = 'InProgress, Downloading'
file_path = self._download_sync(download_id, track_id, display_name)
with self._download_lock:
if download_id in self.active_downloads:
if file_path:
self.active_downloads[download_id]['state'] = 'Completed, Succeeded'
self.active_downloads[download_id]['progress'] = 100.0
self.active_downloads[download_id]['file_path'] = file_path
else:
self.active_downloads[download_id]['state'] = 'Errored'
except Exception as e:
logger.error(f"HiFi download worker failed for {download_id}: {e}")
with self._download_lock:
if download_id in self.active_downloads:
self.active_downloads[download_id]['state'] = 'Errored'
return self._engine.worker.dispatch(
source_name='hifi',
target_id=track_id,
display_name=display_name,
original_filename=filename,
impl_callable=self._download_sync,
extra_record_fields={
'track_id': track_id,
'display_name': display_name,
},
)
def _download_sync(self, download_id: str, track_id: int, display_name: str) -> Optional[str]:
quality_key = config_manager.get('hifi_download.quality', 'lossless')
@ -676,9 +643,8 @@ class HiFiClient:
speed_start = time.time()
segments_completed = 0
with self._download_lock:
if download_id in self.active_downloads:
self.active_downloads[download_id]['size'] = 0
if self._engine is not None:
self._engine.update_record('hifi', download_id, {'size': 0})
with intermediate_path.open('wb') as output_file:
if init_uri:
@ -777,80 +743,77 @@ class HiFiClient:
def _update_download_progress(self, download_id: str, downloaded: int,
segments_completed: int, total_segments: int,
speed_start: float):
with self._download_lock:
if download_id not in self.active_downloads:
return
info = self.active_downloads[download_id]
info['transferred'] = downloaded
if self._engine is None:
return
record = self._engine.get_record('hifi', download_id)
if record is None:
return
now = time.time()
elapsed_total = now - speed_start
speed = int(downloaded / elapsed_total) if elapsed_total > 0 else 0
info['speed'] = speed
now = time.time()
elapsed_total = now - speed_start
speed = int(downloaded / elapsed_total) if elapsed_total > 0 else 0
if total_segments > 0:
progress = (segments_completed / total_segments) * 100
info['progress'] = round(min(progress, 99.9), 1)
progress = record.get('progress', 0.0)
if total_segments > 0:
progress = round(min((segments_completed / total_segments) * 100, 99.9), 1)
time_remaining = None
if speed > 0:
remaining_bytes = downloaded * (total_segments / max(segments_completed, 1)) - downloaded
if remaining_bytes > 0:
time_remaining = int(remaining_bytes / speed)
info['time_remaining'] = time_remaining
time_remaining = None
if speed > 0:
remaining_bytes = downloaded * (total_segments / max(segments_completed, 1)) - downloaded
if remaining_bytes > 0:
time_remaining = int(remaining_bytes / speed)
self._engine.update_record('hifi', download_id, {
'transferred': downloaded,
'speed': speed,
'progress': progress,
'time_remaining': time_remaining,
})
def _record_to_status(self, record):
return DownloadStatus(
id=record['id'],
filename=record['filename'],
username=record['username'],
state=record['state'],
progress=record['progress'],
size=record.get('size', 0),
transferred=record.get('transferred', 0),
speed=record.get('speed', 0),
time_remaining=record.get('time_remaining'),
file_path=record.get('file_path'),
)
async def get_all_downloads(self) -> List[DownloadStatus]:
statuses = []
with self._download_lock:
for _dl_id, info in self.active_downloads.items():
statuses.append(DownloadStatus(
id=info['id'],
filename=info['filename'],
username=info['username'],
state=info['state'],
progress=info['progress'],
size=info['size'],
transferred=info['transferred'],
speed=info['speed'],
time_remaining=info.get('time_remaining'),
file_path=info.get('file_path'),
))
return statuses
if self._engine is None:
return []
return [
self._record_to_status(record)
for record in self._engine.iter_records_for_source('hifi')
]
async def get_download_status(self, download_id: str) -> Optional[DownloadStatus]:
with self._download_lock:
info = self.active_downloads.get(download_id)
if not info:
return None
return DownloadStatus(
id=info['id'],
filename=info['filename'],
username=info['username'],
state=info['state'],
progress=info['progress'],
size=info['size'],
transferred=info['transferred'],
speed=info['speed'],
time_remaining=info.get('time_remaining'),
file_path=info.get('file_path'),
)
if self._engine is None:
return None
record = self._engine.get_record('hifi', download_id)
return self._record_to_status(record) if record is not None else None
async def cancel_download(self, download_id: str, username: str = None,
remove: bool = False) -> bool:
with self._download_lock:
if download_id not in self.active_downloads:
return False
self.active_downloads[download_id]['state'] = 'Cancelled'
if remove:
del self.active_downloads[download_id]
if self._engine is None:
return False
if self._engine.get_record('hifi', download_id) is None:
return False
self._engine.update_record('hifi', download_id, {'state': 'Cancelled'})
if remove:
self._engine.remove_record('hifi', download_id)
return True
async def clear_all_completed_downloads(self) -> bool:
with self._download_lock:
to_remove = [
did for did, info in self.active_downloads.items()
if info.get('state', '') in ('Completed, Succeeded', 'Cancelled', 'Errored', 'Aborted')
]
for did in to_remove:
del self.active_downloads[did]
if self._engine is None:
return True
terminal = {'Completed, Succeeded', 'Cancelled', 'Errored', 'Aborted'}
for record in list(self._engine.iter_records_for_source('hifi')):
if record.get('state') in terminal:
self._engine.remove_record('hifi', record['id'])
return True

View file

@ -208,7 +208,7 @@ def redownload_start(track_id):
# Build a TrackResult-like candidate and submit to download
def _run_redownload():
try:
from core.soulseek_client import TrackResult
from core.download_plugins.types import TrackResult
from core.itunes_client import Track as MetaTrack
tr = TrackResult(
username=candidate['username'],

View file

@ -28,12 +28,15 @@ from utils.logging_config import get_logger
from config.settings import config_manager
# Import Soulseek data structures for drop-in replacement compatibility
from core.soulseek_client import TrackResult, AlbumResult, DownloadStatus
from core.download_plugins.types import TrackResult, AlbumResult, DownloadStatus
logger = get_logger("lidarr_client")
class LidarrDownloadClient:
from core.download_plugins.base import DownloadSourcePlugin
class LidarrDownloadClient(DownloadSourcePlugin):
"""Lidarr download client — uses Lidarr as a download source for Usenet/torrent content.
Implements the same interface as SoulseekClient, QobuzClient, TidalDownloadClient

View file

@ -8,7 +8,7 @@ from config.settings import config_manager
from core.spotify_client import Track as SpotifyTrack
from core.plex_client import PlexTrackInfo
from core.soulseek_client import TrackResult, AlbumResult
from core.download_plugins.types import TrackResult, AlbumResult
logger = get_logger("matching_engine")

View file

@ -28,7 +28,7 @@ from utils.logging_config import get_logger
from config.settings import config_manager
# Import Soulseek data structures for drop-in replacement compatibility
from core.soulseek_client import TrackResult, AlbumResult, DownloadStatus
from core.download_plugins.types import TrackResult, AlbumResult, DownloadStatus
logger = get_logger("qobuz_client")
@ -104,7 +104,10 @@ QOBUZ_QUALITY_MAP = {
}
class QobuzClient:
from core.download_plugins.base import DownloadSourcePlugin
class QobuzClient(DownloadSourcePlugin):
"""
Qobuz download client using Qobuz REST API.
Provides search, matching, and download capabilities as a drop-in alternative to Soulseek/YouTube/Tidal.
@ -136,13 +139,19 @@ class QobuzClient:
self.user_info: Optional[Dict] = None
self._auth_error: Optional[str] = None
# Download queue management
self.active_downloads: Dict[str, Dict[str, Any]] = {}
self._download_lock = threading.Lock()
# Engine reference is populated by set_engine() at registration
# time. None until orchestrator wires the registry.
self._engine = None
# Try to restore saved session
self._restore_session()
def set_engine(self, engine):
"""Engine callback — gives the client access to the central
thread worker + state store. Engine calls this during
``register_plugin`` if the plugin defines it."""
self._engine = engine
def set_shutdown_check(self, check_callable):
"""Set a callback function to check for system shutdown"""
self.shutdown_check = check_callable
@ -520,7 +529,7 @@ class QobuzClient:
"""Pull session state from config without making a network probe.
SoulSync runs two ``QobuzClient`` instances side by side one wired
through ``soulseek_client.qobuz`` for the auth-flow endpoints, and a
through ``soulseek_client.client('qobuz')`` for the auth-flow endpoints, and a
second owned by the enrichment worker for thread safety. When the user
logs in via ``/api/qobuz/auth/login`` or ``/api/qobuz/auth/token`` only
the auth-flow instance's in-memory state is updated; the worker's
@ -884,97 +893,38 @@ class QobuzClient:
return None
async def download(self, username: str, filename: str, file_size: int = 0) -> Optional[str]:
"""
Download a Qobuz track (async, Soulseek-compatible interface).
"""Download a Qobuz track. Delegates to engine.worker which
spawns the background thread + manages state."""
if '||' not in filename:
logger.error(f"Invalid filename format: {filename}")
return None
if self._engine is None:
# Raise rather than return None so the orchestrator's
# download_with_fallback surfaces a real warning + tries
# the next source. Returning None silently dropped the
# download with no user feedback (per JohnBaumb).
raise RuntimeError("Qobuz client has no engine reference — cannot dispatch download")
Returns download_id immediately and runs download in background thread.
Args:
username: Ignored for Qobuz (always "qobuz")
filename: Encoded as "track_id||display_name"
file_size: Ignored
"""
track_id_str, display_name = filename.split('||', 1)
try:
if '||' not in filename:
logger.error(f"Invalid filename format: {filename}")
return None
track_id_str, display_name = filename.split('||', 1)
try:
track_id = int(track_id_str)
except ValueError:
logger.error(f"Invalid Qobuz track ID: {track_id_str}")
return None
logger.info(f"Starting Qobuz download: {display_name}")
download_id = str(uuid.uuid4())
with self._download_lock:
self.active_downloads[download_id] = {
'id': download_id,
'filename': filename,
'username': 'qobuz',
'state': 'Initializing',
'progress': 0.0,
'size': 0,
'transferred': 0,
'speed': 0,
'time_remaining': None,
'track_id': track_id,
'display_name': display_name,
'file_path': None,
}
# Start download in background thread
download_thread = threading.Thread(
target=self._download_thread_worker,
args=(download_id, track_id, display_name, filename),
daemon=True,
)
download_thread.start()
logger.info(f"Qobuz download {download_id} started in background")
return download_id
except Exception as e:
logger.error(f"Failed to start Qobuz download: {e}")
import traceback
traceback.print_exc()
track_id = int(track_id_str)
except ValueError:
logger.error(f"Invalid Qobuz track ID: {track_id_str}")
return None
def _download_thread_worker(self, download_id: str, track_id: int, display_name: str, original_filename: str):
"""Background thread worker for downloading Qobuz tracks."""
try:
with self._download_lock:
if download_id in self.active_downloads:
self.active_downloads[download_id]['state'] = 'InProgress, Downloading'
logger.info(f"Starting Qobuz download: {display_name}")
file_path = self._download_sync(download_id, track_id, display_name)
if file_path:
with self._download_lock:
if download_id in self.active_downloads:
self.active_downloads[download_id]['state'] = 'Completed, Succeeded'
self.active_downloads[download_id]['progress'] = 100.0
self.active_downloads[download_id]['file_path'] = file_path
logger.info(f"Qobuz download {download_id} completed: {file_path}")
else:
with self._download_lock:
if download_id in self.active_downloads:
self.active_downloads[download_id]['state'] = 'Errored'
logger.error(f"Qobuz download {download_id} failed")
except Exception as e:
logger.error(f"Qobuz download thread failed for {download_id}: {e}")
import traceback
traceback.print_exc()
with self._download_lock:
if download_id in self.active_downloads:
self.active_downloads[download_id]['state'] = 'Errored'
return self._engine.worker.dispatch(
source_name='qobuz',
target_id=track_id,
display_name=display_name,
original_filename=filename,
impl_callable=self._download_sync,
extra_record_fields={
'track_id': track_id,
'display_name': display_name,
},
)
def _download_sync(self, download_id: str, track_id: int, display_name: str) -> Optional[str]:
"""
@ -1050,9 +1000,8 @@ class QobuzClient:
chunk_size = 64 * 1024 # 64KB chunks
start_time = time.time()
with self._download_lock:
if download_id in self.active_downloads:
self.active_downloads[download_id]['size'] = total_size
if self._engine is not None:
self._engine.update_record('qobuz', download_id, {'size': total_size})
with open(out_path, 'wb') as f:
for chunk in response.iter_content(chunk_size=chunk_size):
@ -1061,9 +1010,10 @@ class QobuzClient:
# Check for shutdown or cancellation
cancelled = False
with self._download_lock:
if download_id in self.active_downloads:
cancelled = self.active_downloads[download_id].get('state') == 'Cancelled'
if self._engine is not None:
rec = self._engine.get_record('qobuz', download_id)
if rec is not None:
cancelled = rec.get('state') == 'Cancelled'
if cancelled or (self.shutdown_check and self.shutdown_check()):
reason = "cancelled" if cancelled else "server shutting down"
logger.info(f"Aborting Qobuz download mid-stream: {reason}")
@ -1084,18 +1034,20 @@ class QobuzClient:
progress = 0
time_remaining = None
with self._download_lock:
if download_id in self.active_downloads:
self.active_downloads[download_id]['transferred'] = downloaded
self.active_downloads[download_id]['progress'] = round(progress, 1)
self.active_downloads[download_id]['speed'] = int(speed)
self.active_downloads[download_id]['time_remaining'] = time_remaining
if self._engine is not None:
self._engine.update_record('qobuz', download_id, {
'transferred': downloaded,
'progress': round(progress, 1),
'speed': int(speed),
'time_remaining': time_remaining,
})
# If download was aborted (shutdown/cancel), clean up partial file
abort_check = False
with self._download_lock:
if download_id in self.active_downloads:
abort_check = self.active_downloads[download_id].get('state') == 'Cancelled'
if self._engine is not None:
rec = self._engine.get_record('qobuz', download_id)
if rec is not None:
abort_check = rec.get('state') == 'Cancelled'
if abort_check or (self.shutdown_check and self.shutdown_check()):
out_path.unlink(missing_ok=True)
return None
@ -1139,79 +1091,55 @@ class QobuzClient:
# ===================== Status / Cancel / Clear =====================
def _record_to_status(self, record):
return DownloadStatus(
id=record['id'],
filename=record['filename'],
username=record['username'],
state=record['state'],
progress=record['progress'],
size=record.get('size', 0),
transferred=record.get('transferred', 0),
speed=record.get('speed', 0),
time_remaining=record.get('time_remaining'),
file_path=record.get('file_path'),
)
async def get_all_downloads(self) -> List[DownloadStatus]:
"""Get all active downloads (matches Soulseek interface)."""
download_statuses = []
with self._download_lock:
for _download_id, info in self.active_downloads.items():
status = DownloadStatus(
id=info['id'],
filename=info['filename'],
username=info['username'],
state=info['state'],
progress=info['progress'],
size=info['size'],
transferred=info['transferred'],
speed=info['speed'],
time_remaining=info.get('time_remaining'),
file_path=info.get('file_path'),
)
download_statuses.append(status)
return download_statuses
if self._engine is None:
return []
return [
self._record_to_status(record)
for record in self._engine.iter_records_for_source('qobuz')
]
async def get_download_status(self, download_id: str) -> Optional[DownloadStatus]:
"""Get status of a specific download (matches Soulseek interface)."""
with self._download_lock:
if download_id not in self.active_downloads:
return None
info = self.active_downloads[download_id]
return DownloadStatus(
id=info['id'],
filename=info['filename'],
username=info['username'],
state=info['state'],
progress=info['progress'],
size=info['size'],
transferred=info['transferred'],
speed=info['speed'],
time_remaining=info.get('time_remaining'),
file_path=info.get('file_path'),
)
if self._engine is None:
return None
record = self._engine.get_record('qobuz', download_id)
return self._record_to_status(record) if record is not None else None
async def cancel_download(self, download_id: str, username: str = None, remove: bool = False) -> bool:
"""Cancel an active download (matches Soulseek interface)."""
try:
with self._download_lock:
if download_id not in self.active_downloads:
logger.warning(f"Download {download_id} not found")
return False
self.active_downloads[download_id]['state'] = 'Cancelled'
logger.info(f"Marked Qobuz download {download_id} as cancelled")
if remove:
del self.active_downloads[download_id]
logger.info(f"Removed Qobuz download {download_id} from queue")
return True
except Exception as e:
logger.error(f"Failed to cancel download {download_id}: {e}")
if self._engine is None:
return False
if self._engine.get_record('qobuz', download_id) is None:
logger.warning(f"Qobuz download {download_id} not found")
return False
self._engine.update_record('qobuz', download_id, {'state': 'Cancelled'})
logger.info(f"Marked Qobuz download {download_id} as cancelled")
if remove:
self._engine.remove_record('qobuz', download_id)
logger.info(f"Removed Qobuz download {download_id} from queue")
return True
async def clear_all_completed_downloads(self) -> bool:
"""Clear all terminal downloads from the list (matches Soulseek interface)."""
if self._engine is None:
return True
try:
with self._download_lock:
ids_to_remove = [
did for did, info in self.active_downloads.items()
if info.get('state', '') in ('Completed, Succeeded', 'Cancelled', 'Errored', 'Aborted')
]
for did in ids_to_remove:
del self.active_downloads[did]
terminal = {'Completed, Succeeded', 'Cancelled', 'Errored', 'Aborted'}
for record in list(self._engine.iter_records_for_source('qobuz')):
if record.get('state') in terminal:
self._engine.remove_record('qobuz', record['id'])
return True
except Exception as e:
logger.error(f"Error clearing downloads: {e}")

View file

@ -14,7 +14,7 @@ logger = logging.getLogger(__name__)
def run_basic_soulseek_search(
query: str,
soulseek_client,
download_orchestrator,
run_async: Callable,
) -> list[dict]:
"""Search Soulseek for `query`, normalize albums + tracks to one sorted list.
@ -22,7 +22,7 @@ def run_basic_soulseek_search(
Returns dicts with `result_type` set to "album" or "track" and sorted by
`quality_score` descending. Empty list on any failure (caller logs).
"""
tracks, albums = run_async(soulseek_client.search(query))
tracks, albums = run_async(download_orchestrator.search(query))
processed_albums = []
for album in albums:

View file

@ -49,7 +49,7 @@ class SearchDeps:
spotify_client: Any
hydrabase_client: Any
hydrabase_worker: Any
soulseek_client: Any
download_orchestrator: Any
fix_artist_image_url: Callable[[Optional[str]], Optional[str]]
is_hydrabase_active: Callable[[], bool]
get_metadata_fallback_source: Callable[[], str]
@ -289,10 +289,11 @@ def run_enhanced_search(query: str, requested_source: str, deps: SearchDeps) ->
# ---------------------------------------------------------------------------
def resolve_youtube_videos_client(deps: SearchDeps):
"""Return the soulseek_client.youtube subclient or None when unavailable."""
if not deps.soulseek_client:
"""Return the YouTube download client (used for music-video search)
via the orchestrator's generic accessor, or None when unavailable."""
if not deps.download_orchestrator or not hasattr(deps.download_orchestrator, 'client'):
return None
return getattr(deps.soulseek_client, 'youtube', None)
return deps.download_orchestrator.client('youtube')
def stream_youtube_videos(query: str, youtube_client, run_async: Callable) -> Iterator[str]:

View file

@ -96,7 +96,7 @@ def stream_search_track(
album_name: Optional[str],
duration_ms: int,
config_manager,
soulseek_client,
download_orchestrator,
matching_engine,
run_async: Callable,
) -> Optional[dict]:
@ -118,15 +118,9 @@ def stream_search_track(
queries = _build_stream_queries(track_name, artist_name, effective_mode)
stream_clients = {
'youtube': soulseek_client.youtube,
'tidal': soulseek_client.tidal,
'qobuz': soulseek_client.qobuz,
'hifi': soulseek_client.hifi,
'deezer_dl': soulseek_client.deezer_dl,
'lidarr': soulseek_client.lidarr,
}
stream_client = stream_clients.get(effective_mode)
# Map mode name to canonical registry name (legacy 'deezer_dl'
# alias resolves to 'deezer' via the orchestrator's registry).
stream_client = download_orchestrator.client(effective_mode)
use_direct_client = stream_client is not None
max_peer_queue = config_manager.get('soulseek.max_peer_queue', 0) or 0
@ -137,7 +131,7 @@ def stream_search_track(
if use_direct_client:
tracks_result, _ = run_async(stream_client.search(query, timeout=15))
else:
tracks_result, _ = run_async(soulseek_client.search(query, timeout=15))
tracks_result, _ = run_async(download_orchestrator.search(query, timeout=15))
if not tracks_result:
logger.info(f"No results for query '{query}', trying next...")

View file

@ -9,186 +9,21 @@ from pathlib import Path
from utils.logging_config import get_logger
from config.settings import config_manager
from core.imports.filename import parse_filename_metadata
# Shared download-result dataclasses + plugin contract live in the
# neutral plugin package — every source uses the same types, so they
# belong there rather than this soulseek-specific module.
from core.download_plugins.types import (
AlbumResult,
DownloadStatus,
SearchResult,
TrackResult,
)
from core.download_plugins.base import DownloadSourcePlugin
logger = get_logger("soulseek_client")
@dataclass
class SearchResult:
"""Base class for search results"""
username: str
filename: str
size: int
bitrate: Optional[int]
duration: Optional[int] # Duration in milliseconds (converted from slskd's seconds)
quality: str
free_upload_slots: int
upload_speed: int
queue_length: int
result_type: str = "track" # "track" or "album"
@property
def quality_score(self) -> float:
quality_weights = {
'flac': 1.0,
'mp3': 0.8,
'ogg': 0.7,
'aac': 0.6,
'wma': 0.5
}
base_score = quality_weights.get(self.quality.lower(), 0.3)
if self.bitrate:
if self.bitrate >= 320:
base_score += 0.2
elif self.bitrate >= 256:
base_score += 0.1
elif self.bitrate < 128:
base_score -= 0.2
# Free upload slots
if self.free_upload_slots == 0:
base_score -= 0.15
elif self.free_upload_slots > 0:
base_score += 0.05
# Upload speed in bytes/sec (tiered)
if self.upload_speed >= 5_000_000: # ~5 MB/s / 40 Mbps
base_score += 0.15
elif self.upload_speed >= 1_000_000: # ~1 MB/s / 8 Mbps
base_score += 0.10
elif self.upload_speed >= 500_000: # ~500 KB/s / 4 Mbps
base_score += 0.05
elif self.upload_speed < 100_000: # ~100 KB/s / 800 kbps
base_score -= 0.05
# Queue length (graduated penalty)
if self.queue_length > 50:
base_score -= 0.25
elif self.queue_length > 20:
base_score -= 0.15
elif self.queue_length > 10:
base_score -= 0.10
return min(base_score, 1.0)
@dataclass
class TrackResult(SearchResult):
"""Individual track search result"""
artist: Optional[str] = None
title: Optional[str] = None
album: Optional[str] = None
track_number: Optional[int] = None
_source_metadata: Optional[Dict[str, Any]] = None
def __post_init__(self):
self.result_type = "track"
# Try to extract metadata from filename if not provided
if not self.title or not self.artist:
self._parse_filename_metadata()
def _parse_filename_metadata(self):
"""Extract artist, title, album from filename patterns"""
parsed = parse_filename_metadata(self.filename)
if not self.artist and parsed.get("artist"):
self.artist = parsed["artist"]
if not self.title and parsed.get("title"):
self.title = parsed["title"]
if not self.album and parsed.get("album"):
self.album = parsed["album"]
if self.track_number is None:
track_number = parsed.get("track_number")
if track_number is not None:
self.track_number = track_number
@dataclass
class AlbumResult:
"""Album/folder search result containing multiple tracks"""
username: str
album_path: str # Directory path
album_title: str
artist: Optional[str]
track_count: int
total_size: int
tracks: List[TrackResult]
dominant_quality: str # Most common quality in album
year: Optional[str] = None
free_upload_slots: int = 0
upload_speed: int = 0
queue_length: int = 0
result_type: str = "album"
@property
def quality_score(self) -> float:
"""Calculate album quality score based on dominant quality and track count"""
quality_weights = {
'flac': 1.0,
'mp3': 0.8,
'ogg': 0.7,
'aac': 0.6,
'wma': 0.5
}
base_score = quality_weights.get(self.dominant_quality.lower(), 0.3)
# Bonus for complete albums (typically 8-15 tracks)
if 8 <= self.track_count <= 20:
base_score += 0.1
elif self.track_count > 20:
base_score += 0.05
# Free upload slots
if self.free_upload_slots == 0:
base_score -= 0.15
elif self.free_upload_slots > 0:
base_score += 0.05
# Upload speed in bytes/sec (tiered)
if self.upload_speed >= 5_000_000: # ~5 MB/s / 40 Mbps
base_score += 0.15
elif self.upload_speed >= 1_000_000: # ~1 MB/s / 8 Mbps
base_score += 0.10
elif self.upload_speed >= 500_000: # ~500 KB/s / 4 Mbps
base_score += 0.05
elif self.upload_speed < 100_000: # ~100 KB/s / 800 kbps
base_score -= 0.05
# Queue length (graduated penalty)
if self.queue_length > 50:
base_score -= 0.25
elif self.queue_length > 20:
base_score -= 0.15
elif self.queue_length > 10:
base_score -= 0.10
return min(base_score, 1.0)
@property
def size_mb(self) -> int:
"""Album size in MB"""
return self.total_size // (1024 * 1024)
@property
def average_track_size_mb(self) -> float:
"""Average track size in MB"""
if self.track_count > 0:
return self.size_mb / self.track_count
return 0
@dataclass
class DownloadStatus:
id: str
filename: str
username: str
state: str
progress: float
size: int
transferred: int
speed: int
time_remaining: Optional[int] = None
file_path: Optional[str] = None
class SoulseekClient:
class SoulseekClient(DownloadSourcePlugin):
def __init__(self):
self.base_url: Optional[str] = None
self.api_key: Optional[str] = None

View file

@ -27,7 +27,6 @@ import re
import asyncio
import uuid
import time
import threading
from typing import List, Optional, Dict, Any, Tuple, Callable
from pathlib import Path
@ -41,7 +40,7 @@ from config.settings import config_manager
# Standard data structures shared across all download clients so downstream
# matching/post-processing stays source-agnostic.
from core.soulseek_client import TrackResult, AlbumResult, DownloadStatus
from core.download_plugins.types import TrackResult, AlbumResult, DownloadStatus
logger = get_logger("soundcloud_client")
@ -83,7 +82,10 @@ def _sanitize_filename(name: str) -> str:
return cleaned or 'soundcloud_track'
class SoundcloudClient:
from core.download_plugins.base import DownloadSourcePlugin
class SoundcloudClient(DownloadSourcePlugin):
"""SoundCloud download client built on yt-dlp's SoundCloud extractor.
Mirrors the public surface of TidalDownloadClient / QobuzClient so the
@ -106,13 +108,16 @@ class SoundcloudClient:
# in-flight downloads when the worker is shutting down.
self.shutdown_check: Optional[Callable[[], bool]] = None
self.active_downloads: Dict[str, Dict[str, Any]] = {}
self._download_lock = threading.Lock()
self._engine = None
# ------------------------------------------------------------------
# Lifecycle / availability
# ------------------------------------------------------------------
def set_engine(self, engine):
"""Engine callback — wires the central thread worker + state store."""
self._engine = engine
def set_shutdown_check(self, check_callable: Optional[Callable[[], bool]]) -> None:
self.shutdown_check = check_callable
@ -316,98 +321,46 @@ class SoundcloudClient:
# ------------------------------------------------------------------
async def download(self, username: str, filename: str, file_size: int = 0) -> Optional[str]:
"""Kick off a SoundCloud download in a background thread.
Returns the internal download_id used by status/cancel calls,
matching the contract of every other download client.
"""
try:
parts = filename.split('||', 2)
if len(parts) < 2:
logger.error(f"Invalid SoundCloud filename format: {filename}")
return None
sc_track_id = parts[0]
permalink_url = parts[1]
display_name = parts[2] if len(parts) > 2 else sc_track_id
if not sc_track_id or not permalink_url:
logger.error(f"Missing SoundCloud track id or url in: {filename}")
return None
logger.info(f"Starting SoundCloud download: {display_name}")
download_id = str(uuid.uuid4())
with self._download_lock:
self.active_downloads[download_id] = {
'id': download_id,
'filename': filename,
'username': 'soundcloud',
'state': 'Initializing',
'progress': 0.0,
'size': 0,
'transferred': 0,
'speed': 0,
'time_remaining': None,
'track_id': sc_track_id,
'permalink_url': permalink_url,
'display_name': display_name,
'file_path': None,
}
download_thread = threading.Thread(
target=self._download_thread_worker,
args=(download_id, permalink_url, display_name, filename),
daemon=True,
)
download_thread.start()
logger.info(f"SoundCloud download {download_id} started in background")
return download_id
except Exception as exc:
logger.error(f"Failed to start SoundCloud download: {exc}")
import traceback
traceback.print_exc()
"""Kick off a SoundCloud download via engine.worker."""
parts = filename.split('||', 2)
if len(parts) < 2:
logger.error(f"Invalid SoundCloud filename format: {filename}")
return None
def _download_thread_worker(self, download_id: str, permalink_url: str,
display_name: str, original_filename: str) -> None:
"""Background-thread wrapper around `_download_sync`.
sc_track_id = parts[0]
permalink_url = parts[1]
display_name = parts[2] if len(parts) > 2 else sc_track_id
Owns the state transitions on `self.active_downloads[...]` so the
sync impl can just return a path / None and not worry about state.
"""
try:
with self._download_lock:
if download_id in self.active_downloads:
self.active_downloads[download_id]['state'] = 'InProgress, Downloading'
if not sc_track_id or not permalink_url:
logger.error(f"Missing SoundCloud track id or url in: {filename}")
return None
if self._engine is None:
# Raise rather than return None so the orchestrator's
# download_with_fallback surfaces a real warning + tries
# the next source. Returning None silently dropped the
# download with no user feedback (per JohnBaumb).
raise RuntimeError("SoundCloud client has no engine reference — cannot dispatch download")
file_path = self._download_sync(download_id, permalink_url, display_name)
logger.info(f"Starting SoundCloud download: {display_name}")
if file_path:
with self._download_lock:
if download_id in self.active_downloads:
self.active_downloads[download_id]['state'] = 'Completed, Succeeded'
self.active_downloads[download_id]['progress'] = 100.0
self.active_downloads[download_id]['file_path'] = file_path
logger.info(f"SoundCloud download {download_id} completed: {file_path}")
else:
with self._download_lock:
if download_id in self.active_downloads:
# Don't clobber an explicit Cancelled state with Errored.
if self.active_downloads[download_id]['state'] != 'Cancelled':
self.active_downloads[download_id]['state'] = 'Errored'
logger.error(f"SoundCloud download {download_id} failed")
# Worker passes (download_id, target_id, display_name) to impl;
# SoundCloud's _download_sync wants permalink_url (not track_id),
# so adapt by closing over permalink_url here.
def _impl(download_id, _target_id, _display_name):
return self._download_sync(download_id, permalink_url, display_name)
except Exception as exc:
logger.error(f"SoundCloud download thread failed for {download_id}: {exc}")
import traceback
traceback.print_exc()
with self._download_lock:
if download_id in self.active_downloads:
self.active_downloads[download_id]['state'] = 'Errored'
return self._engine.worker.dispatch(
source_name='soundcloud',
target_id=sc_track_id,
display_name=display_name,
original_filename=filename,
impl_callable=_impl,
extra_record_fields={
'track_id': sc_track_id,
'permalink_url': permalink_url,
'display_name': display_name,
},
)
def _download_sync(self, download_id: str, permalink_url: str,
display_name: str) -> Optional[str]:
@ -524,162 +477,128 @@ class SoundcloudClient:
def _update_download_progress_fragmented(self, download_id: str, downloaded: int,
fragment_index: int, fragment_count: int,
speed_start: float) -> None:
"""HLS-aware progress update.
"""HLS-aware progress update — fragment_index / fragment_count
gives an accurate signal even when each fragment's
``total_bytes`` only describes the current fragment."""
if self._engine is None:
return
record = self._engine.get_record('soundcloud', download_id)
if record is None:
return
SoundCloud's HLS streams arrive as N small fragments. yt-dlp can't
compute a true byte-based percentage because each fragment's
``total_bytes`` only describes the current fragment, not the
whole download. fragment_index / fragment_count gives an accurate
progress signal N fragments downloaded out of M known fragments
is the real ratio.
"""
with self._download_lock:
if download_id not in self.active_downloads:
return
info = self.active_downloads[download_id]
info['transferred'] = downloaded
now = time.time()
elapsed = now - speed_start
speed = int(downloaded / elapsed) if elapsed > 0 else 0
now = time.time()
elapsed = now - speed_start
info['speed'] = int(downloaded / elapsed) if elapsed > 0 else 0
progress = round(min((fragment_index / fragment_count) * 100, 99.9), 1) if fragment_count > 0 else 0.0
# `fragment_index` is 1-based when yt-dlp finishes a fragment;
# use it directly. Cap below 100% so the worker thread owns the
# final flip.
progress = (fragment_index / fragment_count) * 100
info['progress'] = round(min(progress, 99.9), 1)
# Estimate total size from per-fragment average.
if fragment_index > 0 and downloaded > 0:
est_total = int(downloaded * (fragment_count / fragment_index))
else:
est_total = downloaded
# Estimate total size so the UI's bytes-remaining reads match
# roughly what users expect — extrapolate from per-fragment
# average. Defensive against fragment_index being 0 on the
# very first callback.
if fragment_index > 0 and downloaded > 0:
est_total = int(downloaded * (fragment_count / fragment_index))
info['size'] = est_total
else:
info['size'] = downloaded
time_remaining: Optional[int] = None
remaining_fragments = max(0, fragment_count - fragment_index)
if speed > 0 and remaining_fragments > 0 and fragment_index > 0:
seconds_per_fragment = elapsed / fragment_index if fragment_index > 0 else 0
time_remaining = int(remaining_fragments * seconds_per_fragment)
time_remaining: Optional[int] = None
remaining_fragments = max(0, fragment_count - fragment_index)
if info['speed'] > 0 and remaining_fragments > 0 and fragment_index > 0:
# Extrapolate seconds from per-fragment average download time.
seconds_per_fragment = elapsed / fragment_index if fragment_index > 0 else 0
time_remaining = int(remaining_fragments * seconds_per_fragment)
info['time_remaining'] = time_remaining
self._engine.update_record('soundcloud', download_id, {
'transferred': downloaded,
'speed': speed,
'progress': progress,
'size': est_total,
'time_remaining': time_remaining,
})
def _update_download_progress(self, download_id: str, downloaded: int,
total: int, speed_start: float) -> None:
"""Push a progress update into the active_downloads ledger.
"""Byte-based progress update for non-HLS streams."""
if self._engine is None:
return
record = self._engine.get_record('soundcloud', download_id)
if record is None:
return
Mirrors the structure other download clients populate so the
existing /api/downloads endpoint can serialize it without caring
about the source.
"""
with self._download_lock:
if download_id not in self.active_downloads:
return
info = self.active_downloads[download_id]
info['transferred'] = downloaded
info['size'] = total
now = time.time()
elapsed = now - speed_start
speed = int(downloaded / elapsed) if elapsed > 0 else 0
now = time.time()
elapsed = now - speed_start
info['speed'] = int(downloaded / elapsed) if elapsed > 0 else 0
progress = record.get('progress', 0.0)
if total > 0:
progress = round(min((downloaded / total) * 100, 99.9), 1)
if total > 0:
progress = (downloaded / total) * 100
# Cap pre-completion progress at 99.9% so the worker thread
# owns the final flip to 100% / Completed.
info['progress'] = round(min(progress, 99.9), 1)
time_remaining: Optional[int] = None
if speed > 0 and total > 0:
remaining = total - downloaded
if remaining > 0:
time_remaining = int(remaining / speed)
time_remaining: Optional[int] = None
if info['speed'] > 0 and total > 0:
remaining = total - downloaded
if remaining > 0:
time_remaining = int(remaining / info['speed'])
info['time_remaining'] = time_remaining
self._engine.update_record('soundcloud', download_id, {
'transferred': downloaded,
'size': total,
'speed': speed,
'progress': progress,
'time_remaining': time_remaining,
})
# ------------------------------------------------------------------
# Status / cancellation
# ------------------------------------------------------------------
def _record_to_status(self, record: dict) -> DownloadStatus:
return DownloadStatus(
id=record['id'],
filename=record['filename'],
username=record['username'],
state=record['state'],
progress=record['progress'],
size=record.get('size', 0),
transferred=record.get('transferred', 0),
speed=record.get('speed', 0),
time_remaining=record.get('time_remaining'),
file_path=record.get('file_path'),
)
async def get_all_downloads(self) -> List[DownloadStatus]:
"""Snapshot every tracked download as DownloadStatus objects."""
out: List[DownloadStatus] = []
with self._download_lock:
for _download_id, info in self.active_downloads.items():
out.append(DownloadStatus(
id=info['id'],
filename=info['filename'],
username=info['username'],
state=info['state'],
progress=info['progress'],
size=info['size'],
transferred=info['transferred'],
speed=info['speed'],
time_remaining=info.get('time_remaining'),
file_path=info.get('file_path'),
))
return out
if self._engine is None:
return []
return [
self._record_to_status(record)
for record in self._engine.iter_records_for_source('soundcloud')
]
async def get_download_status(self, download_id: str) -> Optional[DownloadStatus]:
with self._download_lock:
info = self.active_downloads.get(download_id)
if info is None:
return None
return DownloadStatus(
id=info['id'],
filename=info['filename'],
username=info['username'],
state=info['state'],
progress=info['progress'],
size=info['size'],
transferred=info['transferred'],
speed=info['speed'],
time_remaining=info.get('time_remaining'),
file_path=info.get('file_path'),
)
if self._engine is None:
return None
record = self._engine.get_record('soundcloud', download_id)
return self._record_to_status(record) if record is not None else None
async def cancel_download(self, download_id: str, username: Optional[str] = None,
remove: bool = False) -> bool:
"""Mark a download as cancelled.
Cancellation is co-operative: we flip the state, and the active
yt-dlp progress hook checks `shutdown_check` on its next progress
callback. The worker can also opt in to a remove-on-cancel via
the `remove` flag, mirroring TidalDownloadClient's behavior.
"""
try:
with self._download_lock:
info = self.active_downloads.get(download_id)
if info is None:
logger.warning(f"SoundCloud download {download_id} not found")
return False
info['state'] = 'Cancelled'
logger.info(f"Marked SoundCloud download {download_id} as cancelled")
if remove:
del self.active_downloads[download_id]
logger.info(f"Removed SoundCloud download {download_id} from queue")
return True
except Exception as exc:
logger.error(f"Failed to cancel SoundCloud download {download_id}: {exc}")
"""Mark a download as cancelled. Co-operative — yt-dlp's
progress hook checks shutdown_check on next callback."""
if self._engine is None:
return False
if self._engine.get_record('soundcloud', download_id) is None:
logger.warning(f"SoundCloud download {download_id} not found")
return False
self._engine.update_record('soundcloud', download_id, {'state': 'Cancelled'})
logger.info(f"Marked SoundCloud download {download_id} as cancelled")
if remove:
self._engine.remove_record('soundcloud', download_id)
logger.info(f"Removed SoundCloud download {download_id} from queue")
return True
async def clear_all_completed_downloads(self) -> bool:
"""Drop terminal-state entries from the active_downloads ledger."""
try:
with self._download_lock:
terminal_states = {'Completed, Succeeded', 'Cancelled', 'Errored', 'Aborted'}
ids_to_remove = [
did for did, info in self.active_downloads.items()
if info.get('state', '') in terminal_states
]
for did in ids_to_remove:
del self.active_downloads[did]
logger.info(f"Cleared {len(ids_to_remove)} completed SoundCloud downloads")
if self._engine is None:
return True
except Exception as exc:
logger.error(f"Failed to clear SoundCloud downloads: {exc}")
return False
terminal = {'Completed, Succeeded', 'Cancelled', 'Errored', 'Aborted'}
cleared = 0
for record in list(self._engine.iter_records_for_source('soundcloud')):
if record.get('state') in terminal:
self._engine.remove_record('soundcloud', record['id'])
cleared += 1
logger.info(f"Cleared {cleared} completed SoundCloud downloads")
return True

View file

@ -7,9 +7,9 @@ it in the local Stream/ folder for the browser audio player.
1. Reset stream state to 'loading' with the new track info.
2. Clear any prior file from the Stream/ folder (only one stream lives
there at a time).
3. Spin up a fresh asyncio event loop and `soulseek_client.download()`
3. Spin up a fresh asyncio event loop and `download_orchestrator.download()`
the track.
4. Poll `soulseek_client.get_all_downloads()` every 1.5 s to track
4. Poll `download_orchestrator.get_all_downloads()` every 1.5 s to track
progress, with separate handling for queued vs actively downloading
states. Queue timeout = 15 s; overall timeout = 60 s.
5. On completion (state ~ 'succeeded' or progress >= 100% AND bytes
@ -45,7 +45,7 @@ logger = logging.getLogger(__name__)
class PrepareStreamDeps:
"""Bundle of cross-cutting deps the stream-prep worker needs."""
config_manager: Any
soulseek_client: Any
download_orchestrator: Any
stream_lock: Any # threading.Lock
project_root: str # absolute path to web_server.py's directory
docker_resolve_path: Callable[[str], str]
@ -112,7 +112,7 @@ def prepare_stream_task(track_data, deps: PrepareStreamDeps):
asyncio.set_event_loop(loop)
try:
download_result = loop.run_until_complete(deps.soulseek_client.download(
download_result = loop.run_until_complete(deps.download_orchestrator.download(
track_data.get('username'),
track_data.get('filename'),
track_data.get('size', 0)
@ -144,7 +144,7 @@ def prepare_stream_task(track_data, deps: PrepareStreamDeps):
try:
# Use orchestrator's get_all_downloads() which works for both sources
all_downloads = loop.run_until_complete(deps.soulseek_client.get_all_downloads())
all_downloads = loop.run_until_complete(deps.download_orchestrator.get_all_downloads())
download_status = deps.find_streaming_download_in_all_downloads(all_downloads, track_data)
if download_status:
@ -252,7 +252,7 @@ def prepare_stream_task(track_data, deps: PrepareStreamDeps):
download_id = download_status.get('id', '')
if download_id and track_data.get('username'):
success = loop.run_until_complete(
deps.soulseek_client.signal_download_completion(
deps.download_orchestrator.signal_download_completion(
download_id, track_data.get('username'), remove=True)
)
if success:

View file

@ -14,7 +14,6 @@ import re
import asyncio
import uuid
import time
import threading
import shutil
import subprocess
from typing import List, Optional, Dict, Any, Tuple
@ -33,7 +32,7 @@ from utils.logging_config import get_logger
from config.settings import config_manager
# Import Soulseek data structures for drop-in replacement compatibility
from core.soulseek_client import TrackResult, AlbumResult, DownloadStatus
from core.download_plugins.types import TrackResult, AlbumResult, DownloadStatus
logger = get_logger("tidal_download_client")
@ -93,7 +92,10 @@ HLS_QUALITY_MAP = {
HLS_MAP_TAG_RE = re.compile(r'#EXT-X-MAP:.*URI="([^"]+)"')
class TidalDownloadClient:
from core.download_plugins.base import DownloadSourcePlugin
class TidalDownloadClient(DownloadSourcePlugin):
"""
Tidal download client using tidalapi.
Provides search, matching, and download capabilities as a drop-in alternative to YouTube/Soulseek.
@ -116,12 +118,21 @@ class TidalDownloadClient:
self.session: Optional['tidalapi.Session'] = None
self._init_session()
self.active_downloads: Dict[str, Dict[str, Any]] = {}
self._download_lock = threading.Lock()
self._device_auth_future = None
self._device_auth_link = None
# Engine reference is populated by set_engine() at registration
# time. Until then dispatch returns None — orchestrator wires
# this immediately so the only None case is tests that bypass
# the orchestrator.
self._engine = None
def set_engine(self, engine):
"""Engine callback — gives the client access to the central
thread worker + state store. Engine calls this during
``register_plugin`` if the plugin defines it."""
self._engine = engine
def set_shutdown_check(self, check_callable):
self.shutdown_check = check_callable
@ -604,85 +615,36 @@ class TidalDownloadClient:
) from exc
async def download(self, username: str, filename: str, file_size: int = 0) -> Optional[str]:
if '||' not in filename:
logger.error(f"Invalid filename format: {filename}")
return None
if self._engine is None:
# Raise rather than return None so the orchestrator's
# download_with_fallback surfaces a real warning + tries
# the next source. Returning None silently dropped the
# download with no user feedback (per JohnBaumb).
raise RuntimeError("Tidal client has no engine reference — cannot dispatch download")
track_id_str, display_name = filename.split('||', 1)
try:
if '||' not in filename:
logger.error(f"Invalid filename format: {filename}")
return None
track_id_str, display_name = filename.split('||', 1)
try:
track_id = int(track_id_str)
except ValueError:
logger.error(f"Invalid Tidal track ID: {track_id_str}")
return None
logger.info(f"Starting Tidal download: {display_name}")
download_id = str(uuid.uuid4())
with self._download_lock:
self.active_downloads[download_id] = {
'id': download_id,
'filename': filename,
'username': 'tidal',
'state': 'Initializing',
'progress': 0.0,
'size': 0,
'transferred': 0,
'speed': 0,
'time_remaining': None,
'track_id': track_id,
'display_name': display_name,
'file_path': None,
}
download_thread = threading.Thread(
target=self._download_thread_worker,
args=(download_id, track_id, display_name, filename),
daemon=True,
)
download_thread.start()
logger.info(f"Tidal download {download_id} started in background")
return download_id
except Exception as e:
logger.error(f"Failed to start Tidal download: {e}")
import traceback
traceback.print_exc()
track_id = int(track_id_str)
except ValueError:
logger.error(f"Invalid Tidal track ID: {track_id_str}")
return None
def _download_thread_worker(self, download_id: str, track_id: int, display_name: str, original_filename: str):
try:
with self._download_lock:
if download_id in self.active_downloads:
self.active_downloads[download_id]['state'] = 'InProgress, Downloading'
logger.info(f"Starting Tidal download: {display_name}")
file_path = self._download_sync(download_id, track_id, display_name)
if file_path:
with self._download_lock:
if download_id in self.active_downloads:
self.active_downloads[download_id]['state'] = 'Completed, Succeeded'
self.active_downloads[download_id]['progress'] = 100.0
self.active_downloads[download_id]['file_path'] = file_path
logger.info(f"Tidal download {download_id} completed: {file_path}")
else:
with self._download_lock:
if download_id in self.active_downloads:
self.active_downloads[download_id]['state'] = 'Errored'
logger.error(f"Tidal download {download_id} failed")
except Exception as e:
logger.error(f"Tidal download thread failed for {download_id}: {e}")
import traceback
traceback.print_exc()
with self._download_lock:
if download_id in self.active_downloads:
self.active_downloads[download_id]['state'] = 'Errored'
return self._engine.worker.dispatch(
source_name='tidal',
target_id=track_id,
display_name=display_name,
original_filename=filename,
impl_callable=self._download_sync,
extra_record_fields={
'track_id': track_id,
'display_name': display_name,
},
)
def _download_sync(self, download_id: str, track_id: int, display_name: str) -> Optional[str]:
if not self.session or not self.session.check_login():
@ -727,9 +689,8 @@ class TidalDownloadClient:
speed_start = time.time()
segments_completed = 0
with self._download_lock:
if download_id in self.active_downloads:
self.active_downloads[download_id]['size'] = 0
if self._engine is not None:
self._engine.update_record('tidal', download_id, {'size': 0})
with intermediate_path.open('wb') as output_file:
if init_uri:
@ -828,97 +789,82 @@ class TidalDownloadClient:
def _update_download_progress(self, download_id: str, downloaded: int,
segments_completed: int, total_segments: int,
speed_start: float):
with self._download_lock:
if download_id not in self.active_downloads:
return
info = self.active_downloads[download_id]
info['transferred'] = downloaded
if self._engine is None:
return
record = self._engine.get_record('tidal', download_id)
if record is None:
return
now = time.time()
elapsed_total = now - speed_start
speed = int(downloaded / elapsed_total) if elapsed_total > 0 else 0
info['speed'] = speed
now = time.time()
elapsed_total = now - speed_start
speed = int(downloaded / elapsed_total) if elapsed_total > 0 else 0
if total_segments > 0:
progress = (segments_completed / total_segments) * 100
info['progress'] = round(min(progress, 99.9), 1)
progress = record.get('progress', 0.0)
if total_segments > 0:
progress = round(min((segments_completed / total_segments) * 100, 99.9), 1)
time_remaining = None
if speed > 0:
remaining_bytes = downloaded * (total_segments / max(segments_completed, 1)) - downloaded
if remaining_bytes > 0:
time_remaining = int(remaining_bytes / speed)
info['time_remaining'] = time_remaining
time_remaining = None
if speed > 0:
remaining_bytes = downloaded * (total_segments / max(segments_completed, 1)) - downloaded
if remaining_bytes > 0:
time_remaining = int(remaining_bytes / speed)
self._engine.update_record('tidal', download_id, {
'transferred': downloaded,
'speed': speed,
'progress': progress,
'time_remaining': time_remaining,
})
def _record_to_status(self, record):
return DownloadStatus(
id=record['id'],
filename=record['filename'],
username=record['username'],
state=record['state'],
progress=record['progress'],
size=record.get('size', 0),
transferred=record.get('transferred', 0),
speed=record.get('speed', 0),
time_remaining=record.get('time_remaining'),
file_path=record.get('file_path'),
)
async def get_all_downloads(self) -> List[DownloadStatus]:
download_statuses = []
with self._download_lock:
for _download_id, info in self.active_downloads.items():
status = DownloadStatus(
id=info['id'],
filename=info['filename'],
username=info['username'],
state=info['state'],
progress=info['progress'],
size=info['size'],
transferred=info['transferred'],
speed=info['speed'],
time_remaining=info.get('time_remaining'),
file_path=info.get('file_path'),
)
download_statuses.append(status)
return download_statuses
if self._engine is None:
return []
return [
self._record_to_status(record)
for record in self._engine.iter_records_for_source('tidal')
]
async def get_download_status(self, download_id: str) -> Optional[DownloadStatus]:
with self._download_lock:
if download_id not in self.active_downloads:
return None
info = self.active_downloads[download_id]
return DownloadStatus(
id=info['id'],
filename=info['filename'],
username=info['username'],
state=info['state'],
progress=info['progress'],
size=info['size'],
transferred=info['transferred'],
speed=info['speed'],
time_remaining=info.get('time_remaining'),
file_path=info.get('file_path'),
)
if self._engine is None:
return None
record = self._engine.get_record('tidal', download_id)
return self._record_to_status(record) if record is not None else None
async def cancel_download(self, download_id: str, username: str = None, remove: bool = False) -> bool:
try:
with self._download_lock:
if download_id not in self.active_downloads:
logger.warning(f"Download {download_id} not found")
return False
self.active_downloads[download_id]['state'] = 'Cancelled'
logger.info(f"Marked Tidal download {download_id} as cancelled")
if remove:
del self.active_downloads[download_id]
logger.info(f"Removed Tidal download {download_id} from queue")
return True
except Exception as e:
logger.error(f"Failed to cancel download {download_id}: {e}")
if self._engine is None:
return False
if self._engine.get_record('tidal', download_id) is None:
logger.warning(f"Tidal download {download_id} not found")
return False
self._engine.update_record('tidal', download_id, {'state': 'Cancelled'})
logger.info(f"Marked Tidal download {download_id} as cancelled")
if remove:
self._engine.remove_record('tidal', download_id)
logger.info(f"Removed Tidal download {download_id} from queue")
return True
async def clear_all_completed_downloads(self) -> bool:
if self._engine is None:
return True
try:
with self._download_lock:
ids_to_remove = [
did for did, info in self.active_downloads.items()
if info.get('state', '') in ('Completed, Succeeded', 'Cancelled', 'Errored', 'Aborted')
]
for did in ids_to_remove:
del self.active_downloads[did]
terminal = {'Completed, Succeeded', 'Cancelled', 'Errored', 'Aborted'}
for record in list(self._engine.iter_records_for_source('tidal')):
if record.get('state') in terminal:
self._engine.remove_record('tidal', record['id'])
return True
except Exception as e:
logger.error(f"Error clearing downloads: {e}")

View file

@ -17,7 +17,6 @@ import time
import platform
import asyncio
import uuid
import threading
from typing import List, Optional, Dict, Any, Tuple
from dataclasses import dataclass
from pathlib import Path
@ -34,7 +33,7 @@ from core.matching_engine import MusicMatchingEngine
from core.spotify_client import Track as SpotifyTrack
# Import Soulseek data structures for drop-in replacement compatibility
from core.soulseek_client import SearchResult, TrackResult, AlbumResult, DownloadStatus
from core.download_plugins.types import SearchResult, TrackResult, AlbumResult, DownloadStatus
logger = get_logger("youtube_client")
@ -92,7 +91,10 @@ class YouTubeSearchResult:
self.parsed_artist = self.channel
class YouTubeClient:
from core.download_plugins.base import DownloadSourcePlugin
class YouTubeClient(DownloadSourcePlugin):
"""
YouTube download client using yt-dlp.
Provides search, matching, and download capabilities with full Spotify metadata integration.
@ -112,10 +114,39 @@ class YouTubeClient:
# Callback for shutdown check (avoids circular imports)
self.shutdown_check = None
# Rate limiting — serialize YouTube downloads with delay
self._download_semaphore = threading.Semaphore(1)
# Rate-limit policy — applied to engine.worker once the engine
# is wired in via set_engine(). Kept as an attribute for
# backward-compat external readers + so settings reload can
# update it without touching the engine.
self._download_delay = config_manager.get('youtube.download_delay', 3)
self._last_download_time = 0
# Engine reference is populated by set_engine() at registration
# time. Until then the client can't dispatch downloads — but
# in production the orchestrator wires the engine immediately
# after constructing the registry, so this is only None in
# tests that bypass the orchestrator.
self._engine = None
def rate_limit_policy(self):
"""YouTube reads its download delay from user-tunable config
(``youtube.download_delay``, default 3s). Engine reads this
at ``register_plugin`` time, then ``set_engine`` runs and
re-applies if the config changed since instance construction."""
from core.download_engine import RateLimitPolicy
return RateLimitPolicy(
download_concurrency=1,
download_delay_seconds=float(self._download_delay),
)
def set_engine(self, engine):
"""Engine callback — gives the client access to the central
thread worker + state store. Engine calls this during
``register_plugin`` if the plugin defines it. Worker delay
was already set from rate_limit_policy() re-apply here so
runtime ``reload_settings`` updates take effect via the
same pathway."""
self._engine = engine
engine.worker.set_delay('youtube', float(self._download_delay))
def set_shutdown_check(self, check_callable):
"""Set a callback function to check for system shutdown"""
@ -130,11 +161,6 @@ class YouTubeClient:
logger.error("ffmpeg is required but not found")
logger.error("The client will attempt to auto-download ffmpeg on first use")
# Download queue management (mirrors Soulseek's download tracking)
# Maps download_id -> download_info dict
self.active_downloads: Dict[str, Dict[str, Any]] = {}
self._download_lock = threading.Lock() # Use threading.Lock for thread safety
# Configure yt-dlp options with bot detection bypass
self.download_opts = {
'format': 'bestaudio/best',
@ -272,15 +298,14 @@ class YouTubeClient:
self.progress_callback = callback
def _progress_hook(self, d):
"""
yt-dlp progress hook - called during download to report progress.
Updates the active_downloads dictionary for the current download.
Mirrors Soulseek's transfer status updates.
"""
"""yt-dlp progress hook — called during download to report
progress. Writes to the engine record (Phase C2 lifted state
out of the per-client dict; this hook follows suit)."""
try:
# Only update if we have a current download ID
if not self.current_download_id:
return
if self._engine is None:
return
status = d.get('status', 'unknown')
@ -289,24 +314,18 @@ class YouTubeClient:
total = d.get('total_bytes') or d.get('total_bytes_estimate', 0)
speed = d.get('speed', 0) or 0
eta = d.get('eta', 0) or 0
percent = (downloaded / total) * 100 if total > 0 else 0
if total > 0:
percent = (downloaded / total) * 100
else:
percent = 0
self._engine.update_record('youtube', self.current_download_id, {
'state': 'InProgress, Downloading',
'progress': round(percent, 1),
'transferred': downloaded,
'size': total,
'speed': int(speed),
'time_remaining': int(eta) if eta > 0 else None,
})
# Update active downloads dictionary (thread-safe update with lock)
with self._download_lock:
if self.current_download_id in self.active_downloads:
download_info = self.active_downloads[self.current_download_id]
download_info['state'] = 'InProgress, Downloading' # Match Soulseek state format
download_info['progress'] = round(percent, 1)
download_info['transferred'] = downloaded
download_info['size'] = total
download_info['speed'] = int(speed)
download_info['time_remaining'] = int(eta) if eta > 0 else None
# Also update current_download_progress for legacy compatibility
# Legacy progress dict for any external listeners.
self.current_download_progress = {
'status': 'downloading',
'percent': round(percent, 1),
@ -316,30 +335,27 @@ class YouTubeClient:
'eta': int(eta),
'filename': d.get('filename', '')
}
# Call progress callback if set (for UI updates)
if self.progress_callback:
self.progress_callback(self.current_download_progress)
elif status == 'finished':
# Download finished, ffmpeg is converting to MP3
# Keep state as 'InProgress, Downloading' - the download thread will set final state
with self._download_lock:
if self.current_download_id in self.active_downloads:
self.active_downloads[self.current_download_id]['progress'] = 95.0 # Almost done (converting)
# Download finished — ffmpeg now converts to MP3. The
# engine.worker thread flips to 'Completed, Succeeded'
# once _download_sync returns; this just bumps progress
# to 95% so the UI doesn't sit at 99.9% during the
# ffmpeg post-process.
self._engine.update_record('youtube', self.current_download_id, {
'progress': 95.0,
})
self.current_download_progress['status'] = 'postprocessing'
self.current_download_progress['percent'] = 95.0
if self.progress_callback:
self.progress_callback(self.current_download_progress)
elif status == 'error':
# Mark as error (thread-safe)
with self._download_lock:
if self.current_download_id in self.active_downloads:
self.active_downloads[self.current_download_id]['state'] = 'Errored'
self._engine.update_record('youtube', self.current_download_id, {
'state': 'Errored',
})
self.current_download_progress['status'] = 'error'
if self.progress_callback:
self.progress_callback(self.current_download_progress)
@ -859,137 +875,57 @@ class YouTubeClient:
return matches
async def download(self, username: str, filename: str, file_size: int = 0) -> Optional[str]:
"""
Download YouTube video as audio (async, Soulseek-compatible interface).
"""Download YouTube video as audio.
Returns download_id immediately and runs download in background thread.
Monitor via get_download_status() or get_all_downloads().
Returns download_id immediately; the actual download runs in
a background thread spawned by ``engine.worker``. Monitor
via ``orchestrator.get_download_status(download_id)``.
Args:
username: Ignored for YouTube (always "youtube")
filename: Encoded as "video_id||title" from search results
file_size: Ignored for YouTube (kept for interface compatibility)
Returns:
download_id: Unique ID for tracking this download
"""
try:
# Parse filename to extract video_id
if '||' not in filename:
logger.error(f"Invalid filename format: {filename}")
return None
video_id, title = filename.split('||', 1)
youtube_url = f"https://www.youtube.com/watch?v={video_id}"
logger.info(f"Starting YouTube download: {title}")
logger.info(f" URL: {youtube_url}")
# Create unique download ID
download_id = str(uuid.uuid4())
# Initialize download info in active downloads
with self._download_lock:
self.active_downloads[download_id] = {
'id': download_id,
'filename': filename, # Keep original encoded format for context matching!
'username': 'youtube',
'state': 'Initializing', # Soulseek-style states
'progress': 0.0,
'size': file_size or 0,
'transferred': 0,
'speed': 0,
'time_remaining': None,
'video_id': video_id,
'url': youtube_url,
'title': title,
'file_path': None, # Will be set when download completes
}
# Start download in background thread (returns immediately)
download_thread = threading.Thread(
target=self._download_thread_worker,
args=(download_id, youtube_url, title, filename),
daemon=True
)
download_thread.start()
logger.info(f"YouTube download {download_id} started in background")
return download_id
except Exception as e:
logger.error(f"Failed to start YouTube download: {e}")
import traceback
traceback.print_exc()
if '||' not in filename:
logger.error(f"Invalid filename format: {filename}")
return None
if self._engine is None:
# Raise rather than return None so the orchestrator's
# download_with_fallback surfaces a real warning + tries
# the next source. Returning None silently dropped the
# download with no user feedback (per JohnBaumb).
raise RuntimeError("YouTube client has no engine reference — cannot dispatch download")
def _download_thread_worker(self, download_id: str, youtube_url: str, title: str, original_filename: str):
"""
Background thread worker for downloading YouTube videos.
Updates active_downloads dict with progress.
Serialized via semaphore with configurable delay between downloads.
"""
try:
with self._download_semaphore:
# Enforce delay since last download completed
elapsed = time.time() - self._last_download_time
if self._last_download_time > 0 and elapsed < self._download_delay:
wait_time = self._download_delay - elapsed
logger.info(f"Rate limiting: waiting {wait_time:.1f}s before next YouTube download")
time.sleep(wait_time)
video_id, title = filename.split('||', 1)
youtube_url = f"https://www.youtube.com/watch?v={video_id}"
logger.info("Starting YouTube download: %s (%s)", title, youtube_url)
# Update state to downloading
with self._download_lock:
if download_id in self.active_downloads:
self.active_downloads[download_id]['state'] = 'InProgress, Downloading' # Match Soulseek state
# Set current download ID for progress hook
self.current_download_id = download_id
# Perform actual download
file_path = self._download_sync(youtube_url, title)
# Clear current download ID
def _impl(download_id, _target_id, display_name):
# The progress hook reads ``current_download_id`` to know
# which download to update. Set it before the call, clear
# after, even on exception.
self.current_download_id = download_id
try:
return self._download_sync(youtube_url, title)
finally:
self.current_download_id = None
# Record completion time for rate limiting
self._last_download_time = time.time()
if file_path:
# Mark as completed/succeeded (match Soulseek state)
with self._download_lock:
if download_id in self.active_downloads:
# IMPORTANT: Keep original filename for context lookup!
# The filename must match what was used to create the context entry
# We store the actual file path separately
self.active_downloads[download_id]['state'] = 'Completed, Succeeded' # Match Soulseek
self.active_downloads[download_id]['progress'] = 100.0
self.active_downloads[download_id]['file_path'] = file_path
# DO NOT update filename - keep original_filename for context matching
logger.info(f"YouTube download {download_id} completed: {file_path}")
else:
# Mark as errored
with self._download_lock:
if download_id in self.active_downloads:
self.active_downloads[download_id]['state'] = 'Errored'
logger.error(f"YouTube download {download_id} failed")
except Exception as e:
logger.error(f"YouTube download thread failed for {download_id}: {e}")
import traceback
traceback.print_exc()
# Mark as errored
with self._download_lock:
if download_id in self.active_downloads:
self.active_downloads[download_id]['state'] = 'Errored'
# Clear current download ID
if self.current_download_id == download_id:
self.current_download_id = None
return self._engine.worker.dispatch(
source_name='youtube',
target_id=video_id,
display_name=title,
original_filename=filename,
impl_callable=_impl,
extra_record_fields={
'video_id': video_id,
'url': youtube_url,
'title': title,
},
)
# Legacy worker stub kept temporarily for legacy comment context —
# see _download_sync below for the actual yt-dlp invocation that
# the engine's BackgroundDownloadWorker now drives.
def _download_sync(self, youtube_url: str, title: str) -> Optional[str]:
"""
Synchronous download method (runs in thread pool executor).
@ -1138,123 +1074,76 @@ class YouTubeClient:
traceback.print_exc()
return None
def _record_to_status(self, record):
"""Translate an engine record dict into the DownloadStatus
dataclass shape consumers expect."""
return DownloadStatus(
id=record['id'],
filename=record['filename'],
username=record['username'],
state=record['state'],
progress=record['progress'],
size=record.get('size', 0),
transferred=record.get('transferred', 0),
speed=record.get('speed', 0),
time_remaining=record.get('time_remaining'),
file_path=record.get('file_path'),
)
async def get_all_downloads(self) -> List[DownloadStatus]:
"""
Get all active downloads (matches Soulseek interface).
Returns:
List of DownloadStatus objects for all active downloads
"""
download_statuses = []
with self._download_lock:
for _download_id, download_info in self.active_downloads.items():
status = DownloadStatus(
id=download_info['id'],
filename=download_info['filename'],
username=download_info['username'],
state=download_info['state'],
progress=download_info['progress'],
size=download_info['size'],
transferred=download_info['transferred'],
speed=download_info['speed'],
time_remaining=download_info.get('time_remaining')
)
download_statuses.append(status)
return download_statuses
"""Active downloads owned by the YouTube source — read from
engine state."""
if self._engine is None:
return []
return [
self._record_to_status(record)
for record in self._engine.iter_records_for_source('youtube')
]
async def get_download_status(self, download_id: str) -> Optional[DownloadStatus]:
"""
Get status of a specific download (matches Soulseek interface).
Args:
download_id: Download ID to query
Returns:
DownloadStatus object or None if not found
"""
with self._download_lock:
if download_id not in self.active_downloads:
return None
download_info = self.active_downloads[download_id]
return DownloadStatus(
id=download_info['id'],
filename=download_info['filename'],
username=download_info['username'],
state=download_info['state'],
progress=download_info['progress'],
size=download_info['size'],
transferred=download_info['transferred'],
speed=download_info['speed'],
time_remaining=download_info.get('time_remaining'),
file_path=download_info.get('file_path')
)
"""Single download status — read from engine state. Returns
None if this id isn't owned by YouTube (or not found)."""
if self._engine is None:
return None
record = self._engine.get_record('youtube', download_id)
if record is None:
return None
return self._record_to_status(record)
async def clear_all_completed_downloads(self) -> bool:
"""
Clear all terminal (completed, cancelled, errored) downloads from the list.
Matches Soulseek interface.
"""
"""Clear terminal-state downloads (Completed / Cancelled /
Errored / Aborted) from engine state."""
if self._engine is None:
return True
try:
with self._download_lock:
# Identify IDs to remove
ids_to_remove = []
for download_id, info in self.active_downloads.items():
state = info.get('state', '')
# Check for terminal states
# Note: We check exact strings used in _download_thread_worker and cancel_download
if state in ['Completed, Succeeded', 'Cancelled', 'Errored', 'Aborted']:
ids_to_remove.append(download_id)
# Remove them
for download_id in ids_to_remove:
del self.active_downloads[download_id]
logger.debug(f"Cleared finished download {download_id}")
terminal_states = {'Completed, Succeeded', 'Cancelled', 'Errored', 'Aborted'}
for record in list(self._engine.iter_records_for_source('youtube')):
if record.get('state') in terminal_states:
self._engine.remove_record('youtube', record['id'])
logger.debug("Cleared finished YouTube download %s", record['id'])
return True
except Exception as e:
logger.error(f"Error clearing downloads: {e}")
return False
async def cancel_download(self, download_id: str, username: str = None, remove: bool = False) -> bool:
"""
Cancel an active download (matches Soulseek interface).
NOTE: YouTube downloads cannot be truly cancelled mid-download,
but we mark them as cancelled for UI consistency.
Args:
download_id: Download ID to cancel
username: Ignored for YouTube (kept for interface compatibility)
remove: If True, remove from active downloads after cancelling
Returns:
True if cancelled successfully, False otherwise
"""
try:
with self._download_lock:
if download_id not in self.active_downloads:
logger.warning(f"Download {download_id} not found")
return False
# Update state to cancelled
self.active_downloads[download_id]['state'] = 'Cancelled'
logger.info(f"Marked YouTube download {download_id} as cancelled")
# Remove from active downloads if requested
if remove:
del self.active_downloads[download_id]
logger.info(f"Removed YouTube download {download_id} from queue")
return True
except Exception as e:
logger.error(f"Failed to cancel download {download_id}: {e}")
"""Mark a YouTube download as cancelled. yt-dlp downloads
can't be truly interrupted mid-stream — this only flips
the state for UI consistency. ``remove=True`` also drops
the engine record."""
if self._engine is None:
return False
record = self._engine.get_record('youtube', download_id)
if record is None:
logger.warning(f"YouTube download {download_id} not found")
return False
self._engine.update_record('youtube', download_id, {'state': 'Cancelled'})
logger.info(f"Marked YouTube download {download_id} as cancelled")
if remove:
self._engine.remove_record('youtube', download_id)
logger.info(f"Removed YouTube download {download_id} from queue")
return True
def _enhance_metadata(self, filepath: str, spotify_track: Optional[SpotifyTrack], yt_result: YouTubeSearchResult, track_number: int = 1, disc_number: int = 1, release_year: str = None, artist_genres: list = None):
"""

View file

@ -0,0 +1,206 @@
# Download Engine Refactor Plan
## Goal
Mirror Cin's "metadata engine" architecture for the download dispatcher. Move shared logic OUT of the per-source clients (currently 1600+ LOC of duplicated thread workers, search retry ladders, rate-limiters, state machines) and INTO a central `DownloadEngine`. Clients become dumb: make raw API requests + manage their own auth state. Everything else is the engine.
This is the SAME architectural smell Cin flagged on the metadata layer, applied to downloads. If we keep adding sources (usenet planned + likely more), the only honest fix is to stop reinventing the wheel per client.
## Architecture target
```
┌─────────────┐ ┌──────────────────┐
│ feature │ ── search/download ──▶ ┌────────────────────┐ ─▶│ Soulseek (raw) │
│ │ ◀── normalized ──────── │ DownloadEngine │ ─▶│ YouTube (raw) │
└─────────────┘ │ │ ─▶│ Tidal (raw) │
│ ◆ thread workers │ ─▶│ Qobuz (raw) │
│ ◆ rate-limit pool │ ─▶│ HiFi (raw) │
│ ◆ search retry │ ─▶│ Deezer (raw) │
│ ◆ quality filter │ ─▶│ SoundCloud (raw) │
│ ◆ state tracking │ ─▶│ Lidarr (album) │
│ ◆ fallback chain │ └──────────────────┘
│ ◆ cache │ clients only do:
└────────────────────┘ - raw API request
- auth/token state
```
## What clients keep (legitimately per-source)
- Auth flow + token refresh (Tidal OAuth, Qobuz session, Deezer ARL, slskd API key, etc.)
- Source-specific protocol (slskd events vs HTTP REST vs HLS demux vs Blowfish decrypt vs yt-dlp subprocess)
- Source-specific search query shape (free text vs keyword filters vs MusicBrainz ID lookup)
- Source-specific "download a thing" atomic operation (`_download_impl(target_id) → file_path`)
## What moves into the engine
| Today (per-client, duplicated) | Tomorrow (engine, single source of truth) |
|---|---|
| `self.active_downloads = {}` per client | `engine.active_downloads = {}` |
| `self._download_lock = Lock()` per client | `engine.state_lock = Lock()` |
| `self._download_semaphore = Semaphore(...)` per client | `engine.download_pool` (per-source semaphore from registry) |
| `self._last_download_time / _download_delay` per client | `engine.rate_limiter.acquire(source)` |
| `_download_thread_worker` × 7 (~70 LOC each) | `engine.dispatch_download(plugin, target_id)` |
| Search retry ladder × 7 | `engine.search(query)` with shared retry policy |
| Quality filter × 7 | `engine.filter_by_quality(results, prefs)` |
| Result dedup × 7 | `engine.dedup(results)` |
| Hybrid fallback (search only) | `engine.fallback_chain(operation)` (search AND download) |
## New plugin contract (much smaller)
```python
class DownloadSourcePlugin(Protocol):
# Identity
name: str
# Lifecycle
def is_configured(self) -> bool: ...
async def check_connection(self) -> bool: ...
def reload_settings(self) -> None: ...
# Search — DUMB. Just hit the API.
async def search_raw(self, query: str) -> List[RawSearchResult]: ...
# Download — DUMB. Just download the bytes to a file.
# The engine handles thread spawning, state tracking, rate limits.
# Plugin returns the final file path on success or raises.
async def download_raw(self, target_id: str, dest_dir: Path) -> Path: ...
# Cancel — best-effort. Engine handles state cleanup.
async def cancel_raw(self, target_id: str) -> bool: ...
```
Compare to today's plugin protocol (which my Phase 0 PR introduced) — that one was wrapped around fat clients. This one is dumber. Clients shrink dramatically (estimated 40-60% LOC reduction per file).
## Special cases
- **Soulseek** — slskd is event-driven, NOT thread-based. Engine's BackgroundDownloadWorker doesn't apply. Keep Soulseek's path special: `download_raw` returns immediately; engine subscribes to slskd events for state updates instead of running a thread.
- **YouTube/SoundCloud** — yt-dlp is a subprocess. The "thread" is really `subprocess.run(['yt-dlp', ...]).wait()`. Engine handles thread; plugin's `download_raw` just runs subprocess and returns file path.
- **Lidarr** — album-grabber, not track-grabber. Different contract. Either separate `AlbumOnlyPlugin` interface OR plugin declares `supports_track_search: bool = False`. Decide during migration.
## Phased commit plan
Each phase is one or more commits. Each commit independently revertable. Tests stay green between commits — never ship a half-broken state.
### Phase A — Behavior pinning tests (BEFORE any code moves)
**Goal:** Baseline tests for what each source's download path currently does. Catches regressions during extraction.
**Commit A1:** `tests/downloads/test_soulseek_download_path.py` — pin Soulseek's download lifecycle (search → download → completion → file path returned).
**Commit A2:** Same for YouTube. Mock yt-dlp subprocess.
**Commit A3:** Same for Tidal. Mock tidalapi.Session.
**Commit A4:** Same for Qobuz. Mock Qobuz REST API.
**Commit A5:** Same for HiFi. Mock hifi-api instance.
**Commit A6:** Same for Deezer. Mock Deezer GW API + Blowfish stream.
**Commit A7:** Same for SoundCloud. Mock yt-dlp scsearch.
**Commit A8:** Same for Lidarr. Mock Lidarr REST API.
After Phase A: ~50 new tests pinning current behavior. We can refactor with confidence.
### Phase B — Engine skeleton + state lift
**Commit B1:** Create `core/download_engine/` package with `DownloadEngine` class. Engine starts EMPTY — just exposes `register_plugin(plugin)`, `active_downloads` dict, `state_lock`. Orchestrator gets a `self.engine` reference but doesn't use it yet.
**Commit B2:** Move `active_downloads` state out of every client into `engine.active_downloads`. Each client's `download()` now updates engine state via callback instead of `self.active_downloads[id] = ...`. Backward compat: each client's `self.active_downloads` becomes a property that delegates to `engine.active_downloads.filter(source=self.name)`.
**Commit B3:** Move `get_all_downloads` / `get_download_status` / `cancel_download` dispatch from orchestrator (which iterates plugins) into engine (which queries unified state). Orchestrator's methods become thin pass-throughs.
### Phase C — Background download worker lift
**Commit C1:** New `core/download_engine/worker.py``BackgroundDownloadWorker` class. Owns semaphore, rate-limit sleep, state-update lock pattern. Provides `dispatch(plugin, target_id, display_name) → download_id`.
**Commit C2:** Migrate YouTube to use BackgroundDownloadWorker. Strip `_download_thread_worker` from `youtube_client.py`. Add `download_raw(video_id, dest) → Path`. Tests stay green (Phase A pinned them).
**Commit C3:** Same for Tidal.
**Commit C4:** Same for Qobuz.
**Commit C5:** Same for HiFi.
**Commit C6:** Same for Deezer.
**Commit C7:** Same for SoundCloud.
After Phase C: ~490 LOC of duplicated thread management deleted. Each affected client shrinks.
### Phase D — SKIPPED
**Original intent:** Lift search retry / query normalization / quality filter into engine. **Dropped after surveying actual per-source search code.** Search is 90% source-specific (slskd event subscription vs yt-dlp subprocess vs HTTP REST vs HLS quality map), not 60% like the original plan estimated. Lifting would be either lossy (force per-source quirks through a uniform interface) or bloated (adapter code bigger than the original). The shared portion is ~10 LOC per source — not worth a SearchOrchestrator. Per-source search stays per-source.
### Phase D (original — kept for reference, NOT executed)
**Commit D1:** New `core/download_engine/search.py``SearchOrchestrator`. Owns: query normalization, shortened-query retry ladder, quality filter, dedup. Calls `plugin.search_raw(query)` for the actual API hit.
**Commit D2:** Migrate Tidal's search. Strip `_generate_shortened_queries`, quality filter, dedup from client. Add `search_raw(query) → List[RawResult]`.
**Commit D3:** Same for Qobuz.
**Commit D4:** Same for HiFi.
**Commit D5:** Same for YouTube.
**Commit D6:** Same for Deezer.
**Commit D7:** Same for SoundCloud.
**Commit D8:** Same for Soulseek (keep slskd event-driven specifics, but the post-search filter/dedup moves out).
**Commit D9:** Same for Lidarr.
### Phase E — Rate-limit pool
**Commit E1:** New `core/download_engine/rate_limit.py` — per-source rate limiter registry. Spotify limit, Qobuz 1/sec, etc. Each plugin declares its limits in its registry spec.
**Commit E2:** Strip per-client rate-limit state. Replace with `await engine.rate_limit.acquire(self.name)` at the top of `search_raw` / `download_raw`.
### Phase F — Fallback chain into engine
**Commit F1:** Engine owns fallback: `engine.search_with_fallback(query, source_chain)` and `engine.download_with_fallback(target_id, source_chain)`. Search hybrid behavior preserved; download hybrid newly works (today it silently routes to one source).
**Commit F2:** Orchestrator's `search` and `download` methods delegate to engine's fallback methods. Hybrid mode logic moves out of orchestrator.
### Phase G — Plugin contract narrows
**Commit G1:** Update `DownloadSourcePlugin` Protocol — narrow to the small surface (`search_raw`, `download_raw`, `cancel_raw`, `is_configured`, `check_connection`, `reload_settings`). Conformance tests updated.
**Commit G2:** Remove dead methods from clients that the engine now owns (`_download_thread_worker`, `_filter_results_by_quality`, etc.). Clean up imports.
### Phase H — Cleanup + WHATS_NEW + version bump
**Commit H1:** Final cleanup pass — remove backward-compat shims that are no longer needed (legacy `self.active_downloads` properties etc., once nothing reaches in for them).
**Commit H2:** WHATS_NEW entry, PR description.
## Total estimated scope
- ~25-30 commits
- ~2000 LOC removed (duplicated thread workers, search retries, etc.)
- ~1200 LOC added (engine + per-source slim adapters)
- Net reduction: ~800 LOC
- ~50 new tests (Phase A pinning) + ~20 engine-level tests
- 1-2 days of focused work
## Risk profile
**Low risk:**
- Phase A (only adds tests, never changes behavior)
- Phase B1 (new file, doesn't touch existing code)
- Phase H (cleanup of dead code)
**Medium risk:**
- Phase B2-B3 (state lift — race conditions, lock contention)
- Phase C (thread worker extraction — semaphore semantics, exception propagation)
- Phase G (contract narrows — anything reaching in for removed methods breaks)
**High risk:**
- Phase D (search retry — easy to subtly change retry ladder shape)
- Phase E (rate-limit — wrong order can cause deadlocks or under-limit violations)
- Phase F (fallback — easy to accidentally change hybrid mode behavior)
**Mitigation:** Phase A pinning tests catch behavior drift in every later phase. Each commit must pass full suite. Manual smoke test per source after Phase C and again at end.
## Coordination with Cin
- Cin's metadata engine PR will likely set the precedent for HOW abstractions look (Protocol vs ABC, sync vs async, state location). This plan defaults to Protocol + async (matches what we already have) but easy to mirror Cin's exact pattern when his PR lands.
- If his pattern differs significantly, we may need to redo some commits. Best mitigation: don't dig too deep on contract shape (Phase G) until his PR is visible. Phases A-C don't depend on contract shape; they're safe to do regardless.
- Send Cin a heads-up before starting — he may have feedback on the plan that saves a redesign later.
## Compatibility commitments
- Backward-compat `orchestrator.soulseek` / `orchestrator.youtube` / etc. attributes preserved through every phase.
- Backward-compat `clear_all_searches`, `_make_request`, `signal_download_completion`, etc. (Soulseek-specific reaches) preserved through every phase.
- Frontend status dashboard keys (`deezer_dl` alias) preserved.
- Config format unchanged.
- DB schema unchanged.
- API endpoint surface unchanged.
## What's NOT in this PR
- Cin's metadata engine work (separate, his domain)
- Media server client refactor (different subsystem, separate PR)
- Match engine refactor (different subsystem, separate PR)
- Adding new download sources (out of scope; the new contract makes them easier later)

View file

@ -7,7 +7,7 @@ from core.spotify_client import SpotifyClient, Playlist as SpotifyPlaylist, Trac
from core.plex_client import PlexClient, PlexTrackInfo
from core.jellyfin_client import JellyfinClient
from core.navidrome_client import NavidromeClient
from core.soulseek_client import SoulseekClient
from core.download_orchestrator import DownloadOrchestrator
from core.matching_engine import MusicMatchingEngine, MatchResult
logger = get_logger("sync_service")
@ -44,12 +44,12 @@ class SyncProgress:
failed_tracks: int = 0
class PlaylistSyncService:
def __init__(self, spotify_client: SpotifyClient, plex_client: PlexClient, soulseek_client: SoulseekClient, jellyfin_client: JellyfinClient = None, navidrome_client = None):
def __init__(self, spotify_client: SpotifyClient, plex_client: PlexClient, download_orchestrator: DownloadOrchestrator, jellyfin_client: JellyfinClient = None, navidrome_client = None):
self.spotify_client = spotify_client
self.plex_client = plex_client
self.jellyfin_client = jellyfin_client
self.navidrome_client = navidrome_client
self.soulseek_client = soulseek_client
self.download_orchestrator = download_orchestrator
self.progress_callbacks = {} # Playlist-specific progress callbacks
self.syncing_playlists = set() # Track multiple syncing playlists
self._cancelled = False
@ -635,7 +635,7 @@ class PlaylistSyncService:
query = self.matching_engine.generate_download_query(match_result.spotify_track)
logger.info(f"Attempting to download: {query}")
download_id = await self.soulseek_client.search_and_download_best(query, expected_track=match_result.spotify_track)
download_id = await self.download_orchestrator.search_and_download_best(query, expected_track=match_result.spotify_track)
if download_id:
downloaded_count += 1

View file

@ -0,0 +1,542 @@
"""Tests for `BackgroundDownloadWorker` (Phase C1).
These tests pin the worker's state-machine semantics, semaphore
serialization, rate-limit-delay behavior, and exception handling.
Future phases (C2C7) migrate each per-source client onto this
worker these tests stay green as the regression net.
"""
from __future__ import annotations
import threading
import time
from core.download_engine import DownloadEngine
# ---------------------------------------------------------------------------
# Dispatch — initial state + thread spawn
# ---------------------------------------------------------------------------
def test_dispatch_returns_uuid_download_id():
engine = DownloadEngine()
def impl(download_id, target_id, display_name):
return '/tmp/file.flac'
download_id = engine.worker.dispatch(
source_name='youtube',
target_id='abc123',
display_name='Some Song',
original_filename='abc123||Some Song',
impl_callable=impl,
)
assert len(download_id) == 36 # UUID4
assert download_id.count('-') == 4
def test_dispatch_inserts_initial_record_with_canonical_state():
"""Pinning: initial record matches the legacy per-client shape so
consumers reading the state dict via API or context-key lookup
keep working unchanged after migration."""
engine = DownloadEngine()
captured = threading.Event()
def impl(download_id, target_id, display_name):
captured.wait(timeout=1.0) # block so we can read 'Initializing' / 'InProgress' state
return '/tmp/file.flac'
download_id = engine.worker.dispatch(
source_name='youtube',
target_id='abc',
display_name='X',
original_filename='abc||X',
impl_callable=impl,
)
record = engine.get_record('youtube', download_id)
assert record is not None
assert record['id'] == download_id
assert record['filename'] == 'abc||X'
assert record['username'] == 'youtube'
assert record['state'] in ('Initializing', 'InProgress, Downloading')
assert record['progress'] == 0.0
assert record['file_path'] is None
captured.set() # release impl
def test_dispatch_merges_extra_record_fields():
"""Pinning: source-specific slots (video_id, track_id, etc.)
merge into the initial record so frontend + status APIs that
read those keys keep working."""
engine = DownloadEngine()
started = threading.Event()
release = threading.Event()
def impl(download_id, target_id, display_name):
started.set()
release.wait(timeout=1.0)
return '/tmp/x.flac'
download_id = engine.worker.dispatch(
source_name='youtube',
target_id='vid123',
display_name='Title',
original_filename='vid123||Title',
impl_callable=impl,
extra_record_fields={
'video_id': 'vid123',
'url': 'https://youtube.com/watch?v=vid123',
'title': 'Title',
},
)
started.wait(timeout=1.0)
record = engine.get_record('youtube', download_id)
assert record['video_id'] == 'vid123'
assert record['url'] == 'https://youtube.com/watch?v=vid123'
assert record['title'] == 'Title'
release.set()
def test_dispatch_username_override_preserves_legacy_slot():
"""Pinning: Deezer's record stores `'deezer_dl'` (legacy) in the
username slot, not the canonical `'deezer'`. Worker accepts
override so frontend status indicators keep their key."""
engine = DownloadEngine()
release = threading.Event()
def impl(download_id, target_id, display_name):
release.wait(timeout=1.0)
return '/tmp/x.flac'
download_id = engine.worker.dispatch(
source_name='deezer',
target_id='999',
display_name='X',
original_filename='999||X',
impl_callable=impl,
username_override='deezer_dl',
)
record = engine.get_record('deezer', download_id)
assert record['username'] == 'deezer_dl'
release.set()
# ---------------------------------------------------------------------------
# Worker lifecycle — state transitions
# ---------------------------------------------------------------------------
def test_worker_marks_completed_on_successful_impl():
engine = DownloadEngine()
def impl(download_id, target_id, display_name):
return '/tmp/done.flac'
download_id = engine.worker.dispatch(
source_name='youtube',
target_id='vid',
display_name='X',
original_filename='vid||X',
impl_callable=impl,
)
# Wait for thread to finish.
deadline = time.time() + 2.0
while time.time() < deadline:
record = engine.get_record('youtube', download_id)
if record and record['state'] == 'Completed, Succeeded':
break
time.sleep(0.01)
record = engine.get_record('youtube', download_id)
assert record['state'] == 'Completed, Succeeded'
assert record['progress'] == 100.0
assert record['file_path'] == '/tmp/done.flac'
def test_worker_preserves_cancelled_when_impl_returns_none():
"""Pinning: if the user cancels mid-download (state flips to
Cancelled via engine.update_record from cancel_download), the
worker must NOT clobber it back to Errored when impl returns
None. The legacy per-client thread workers had this guard
(``if state != 'Cancelled': state = 'Errored'``); the shared
worker preserves that contract."""
engine = DownloadEngine()
def impl(download_id, target_id, display_name):
# Simulate user cancelling mid-impl by writing Cancelled.
engine.update_record('youtube', download_id, {'state': 'Cancelled'})
return None # impl returns None because download was interrupted
download_id = engine.worker.dispatch(
source_name='youtube',
target_id='vid',
display_name='X',
original_filename='vid||X',
impl_callable=impl,
)
deadline = time.time() + 2.0
while time.time() < deadline:
record = engine.get_record('youtube', download_id)
if record and record['state'] in ('Cancelled', 'Errored'):
break
time.sleep(0.01)
record = engine.get_record('youtube', download_id)
assert record['state'] == 'Cancelled', (
f"Worker clobbered user's Cancelled with {record['state']}"
)
def test_worker_preserves_cancelled_when_impl_returns_success():
"""Cin's bug 3 follow-up: the success path also has a read-then-write
race. If the user cancels between the impl returning a valid file
path and the worker writing 'Completed, Succeeded', the cancel is
overwritten. The success-path write must use the same atomic
Cancelled-preserve guard as _mark_terminal."""
engine = DownloadEngine()
def impl(download_id, target_id, display_name):
# User cancels mid-impl, then impl finishes successfully.
engine.update_record('youtube', download_id, {'state': 'Cancelled'})
return '/tmp/file.flac'
download_id = engine.worker.dispatch(
source_name='youtube',
target_id='vid',
display_name='X',
original_filename='vid||X',
impl_callable=impl,
)
deadline = time.time() + 2.0
while time.time() < deadline:
record = engine.get_record('youtube', download_id)
if record and record['state'] in ('Cancelled', 'Completed, Succeeded'):
break
time.sleep(0.01)
record = engine.get_record('youtube', download_id)
assert record['state'] == 'Cancelled', (
f"Worker clobbered user's Cancelled with {record['state']}"
)
def test_worker_preserves_cancelled_when_impl_raises():
"""Same Cancelled-preserve guard, but for the impl-raises path."""
engine = DownloadEngine()
def impl(download_id, target_id, display_name):
engine.update_record('youtube', download_id, {'state': 'Cancelled'})
raise RuntimeError("simulated mid-cancel exception")
download_id = engine.worker.dispatch(
source_name='youtube',
target_id='vid',
display_name='X',
original_filename='vid||X',
impl_callable=impl,
)
deadline = time.time() + 2.0
while time.time() < deadline:
record = engine.get_record('youtube', download_id)
if record and record['state'] in ('Cancelled', 'Errored'):
break
time.sleep(0.01)
assert engine.get_record('youtube', download_id)['state'] == 'Cancelled'
def test_worker_marks_errored_when_impl_returns_none():
engine = DownloadEngine()
def impl(download_id, target_id, display_name):
return None # signaling failure
download_id = engine.worker.dispatch(
source_name='youtube',
target_id='vid',
display_name='X',
original_filename='vid||X',
impl_callable=impl,
)
deadline = time.time() + 2.0
while time.time() < deadline:
record = engine.get_record('youtube', download_id)
if record and record['state'] == 'Errored':
break
time.sleep(0.01)
record = engine.get_record('youtube', download_id)
assert record['state'] == 'Errored'
# file_path stays None (default).
assert record['file_path'] is None
def test_worker_marks_errored_and_captures_message_when_impl_raises():
engine = DownloadEngine()
def impl(download_id, target_id, display_name):
raise RuntimeError("api blew up")
download_id = engine.worker.dispatch(
source_name='youtube',
target_id='vid',
display_name='X',
original_filename='vid||X',
impl_callable=impl,
)
deadline = time.time() + 2.0
while time.time() < deadline:
record = engine.get_record('youtube', download_id)
if record and record['state'] == 'Errored':
break
time.sleep(0.01)
record = engine.get_record('youtube', download_id)
assert record['state'] == 'Errored'
assert 'api blew up' in record.get('error', '')
# ---------------------------------------------------------------------------
# Per-source semaphore serialization
# ---------------------------------------------------------------------------
def test_semaphore_serializes_downloads_for_same_source():
"""Pinning: with concurrency=1 (default), two dispatches against
the same source run sequentially. The legacy per-client
semaphore did the same consumers depend on this for
rate-limit safety against APIs like YouTube."""
engine = DownloadEngine()
in_progress = threading.Event()
can_finish = threading.Event()
overlap_count = 0
overlap_lock = threading.Lock()
active_count = [0]
def impl(download_id, target_id, display_name):
nonlocal overlap_count
with overlap_lock:
active_count[0] += 1
if active_count[0] > 1:
overlap_count += 1
in_progress.set()
can_finish.wait(timeout=2.0)
with overlap_lock:
active_count[0] -= 1
return '/tmp/x.flac'
# Default concurrency=1 — two dispatches must serialize.
dl1 = engine.worker.dispatch(
source_name='youtube', target_id='a', display_name='A',
original_filename='a||A', impl_callable=impl,
)
in_progress.wait(timeout=1.0)
in_progress.clear()
dl2 = engine.worker.dispatch(
source_name='youtube', target_id='b', display_name='B',
original_filename='b||B', impl_callable=impl,
)
# Give second dispatch a chance to attempt running in parallel
# (it should be blocked on the semaphore).
time.sleep(0.1)
assert overlap_count == 0, "second dispatch should be blocked behind semaphore"
# Release first; second proceeds.
can_finish.set()
# Wait for both to finish.
deadline = time.time() + 3.0
while time.time() < deadline:
r1 = engine.get_record('youtube', dl1)
r2 = engine.get_record('youtube', dl2)
if r1 and r2 and r1['state'] == 'Completed, Succeeded' and r2['state'] == 'Completed, Succeeded':
break
time.sleep(0.01)
assert overlap_count == 0
def test_semaphore_concurrency_can_be_increased():
"""When `set_concurrency(source, N)` is called, N downloads can
run in parallel for that source. Used by sources that support
parallel transfers (none today, but contract supports it)."""
engine = DownloadEngine()
engine.worker.set_concurrency('parallel-source', 3)
in_flight = []
in_flight_lock = threading.Lock()
can_finish = threading.Event()
max_observed = [0]
def impl(download_id, target_id, display_name):
with in_flight_lock:
in_flight.append(download_id)
max_observed[0] = max(max_observed[0], len(in_flight))
can_finish.wait(timeout=2.0)
with in_flight_lock:
in_flight.remove(download_id)
return '/tmp/x.flac'
for i in range(3):
engine.worker.dispatch(
source_name='parallel-source',
target_id=str(i),
display_name=f'd{i}',
original_filename=f'{i}||d{i}',
impl_callable=impl,
)
# Give threads time to ramp up.
time.sleep(0.2)
can_finish.set()
# Wait for them to finish.
time.sleep(0.5)
assert max_observed[0] == 3
# ---------------------------------------------------------------------------
# Per-source rate-limit delay
# ---------------------------------------------------------------------------
def test_impl_can_observe_cancel_mid_flight_via_state_check():
"""Per JohnBaumb: existing tests cover Cancelled-preserve AFTER impl
returns but plugins also poll engine state mid-download (via
``_is_cancelled`` helpers) to abort partial transfers. Pin that
contract: a cancel landing while impl is mid-flight must be
visible to a subsequent ``engine.get_record()`` from the impl
thread.
"""
engine = DownloadEngine()
impl_started = threading.Event()
impl_can_finish = threading.Event()
observed_state_during_impl = []
def impl(download_id, target_id, display_name):
impl_started.set()
# Wait for the test thread to write Cancelled, then check
# what we can observe from inside the impl callback.
impl_can_finish.wait(timeout=2.0)
record = engine.get_record('youtube', download_id)
observed_state_during_impl.append(record.get('state') if record else None)
return None # impl noticed cancel + bailed out
download_id = engine.worker.dispatch(
source_name='youtube',
target_id='vid',
display_name='X',
original_filename='vid||X',
impl_callable=impl,
)
# Wait for impl to start, then inject a cancel.
impl_started.wait(timeout=1.0)
engine.update_record('youtube', download_id, {'state': 'Cancelled'})
impl_can_finish.set()
# Wait for impl to finish (its append populates observed_state).
# Engine state is already Cancelled at this point, so polling on
# state would race: it'd break before impl ran the get_record line.
deadline = time.time() + 2.0
while not observed_state_during_impl and time.time() < deadline:
time.sleep(0.01)
# impl observed the Cancelled state mid-flight via get_record,
# AND the worker preserved Cancelled after impl returned None.
assert observed_state_during_impl == ['Cancelled']
assert engine.get_record('youtube', download_id)['state'] == 'Cancelled'
def test_per_source_delays_dont_block_other_sources():
"""Per JohnBaumb: per-source semaphores + delays must not let one
slow source stall another. YouTube's 3s rate-limit delay should
not delay a Tidal download starting in parallel.
Configure YouTube with a 0.5s delay, dispatch one YouTube download
(which holds the source's serial slot + arms the next-call delay),
then immediately dispatch a Tidal download. Tidal must complete
well before YouTube's delay window would have elapsed.
"""
engine = DownloadEngine()
engine.worker.set_delay('youtube', 0.5)
yt_completed = threading.Event()
td_completed = threading.Event()
def yt_impl(download_id, target_id, display_name):
time.sleep(0.05)
yt_completed.set()
return '/tmp/yt.mp3'
def td_impl(download_id, target_id, display_name):
td_completed.set()
return '/tmp/td.flac'
# First YouTube call. After it finishes, the worker arms the
# 0.5s delay BEFORE the next youtube dispatch can run.
engine.worker.dispatch(
source_name='youtube', target_id='a', display_name='A',
original_filename='a||A', impl_callable=yt_impl,
)
yt_completed.wait(timeout=1.0)
# Second YouTube would now block 0.5s on the rate-limit delay.
# Dispatch one to occupy that wait, then dispatch Tidal — Tidal
# must NOT wait on YouTube's delay.
engine.worker.dispatch(
source_name='youtube', target_id='b', display_name='B',
original_filename='b||B', impl_callable=lambda *a: '/tmp/y.mp3',
)
td_start = time.time()
engine.worker.dispatch(
source_name='tidal', target_id='t1', display_name='T',
original_filename='t1||T', impl_callable=td_impl,
)
td_completed.wait(timeout=0.4)
td_elapsed = time.time() - td_start
# Tidal must have finished in well under 0.5s (the YouTube delay).
assert td_completed.is_set(), "Tidal blocked on YouTube's per-source delay"
assert td_elapsed < 0.4, f"Tidal took {td_elapsed:.2f}s — should be near-instant"
def test_delay_enforces_minimum_gap_between_downloads():
"""Pinning: YouTube uses 3s delay today (legacy
`_download_delay`). Worker-driven delay must enforce the same
gap so YouTube doesn't 429."""
engine = DownloadEngine()
engine.worker.set_delay('youtube', 0.2) # 200ms — short for test speed
completion_times = []
def impl(download_id, target_id, display_name):
completion_times.append(time.time())
return '/tmp/x.flac'
# Two back-to-back dispatches.
engine.worker.dispatch(
source_name='youtube', target_id='a', display_name='A',
original_filename='a||A', impl_callable=impl,
)
engine.worker.dispatch(
source_name='youtube', target_id='b', display_name='B',
original_filename='b||B', impl_callable=impl,
)
# Wait for both to finish (semaphore serializes + delay).
deadline = time.time() + 3.0
while time.time() < deadline and len(completion_times) < 2:
time.sleep(0.01)
assert len(completion_times) == 2
gap = completion_times[1] - completion_times[0]
# Gap is at LEAST the configured delay.
assert gap >= 0.18, f"expected gap >= 0.2s, got {gap:.3f}"

View file

@ -0,0 +1,154 @@
"""Phase A pinning tests for DeezerDownloadClient — UPDATED for Phase C6.
Deezer has the same engine-driven dispatch as the other streaming
sources, with three Deezer-specific quirks preserved:
- track_id stays as STRING (Deezer GW API uses string IDs).
- Engine record's `username` slot is the legacy `'deezer_dl'`
via worker username_override.
- Worker thread is named `deezer-dl-<track_id>` for diagnostics.
"""
from __future__ import annotations
import asyncio
import threading
from pathlib import Path
from unittest.mock import patch
import pytest
from core.download_engine import DownloadEngine
from core.deezer_download_client import DeezerDownloadClient
def _run_async(coro):
loop = asyncio.new_event_loop()
try:
return loop.run_until_complete(coro)
finally:
loop.close()
@pytest.fixture
def deezer_client_with_engine():
client = DeezerDownloadClient.__new__(DeezerDownloadClient)
client.download_path = Path('./test_deezer_downloads')
client.shutdown_check = None
client._authenticated = True
client._engine = None
engine = DownloadEngine()
client.set_engine(engine)
return client, engine
def test_download_returns_none_when_not_authenticated(deezer_client_with_engine):
client, _ = deezer_client_with_engine
client._authenticated = False
result = _run_async(client.download('deezer_dl', '12345||x', 0))
assert result is None
def test_download_raises_when_engine_not_wired():
"""Defensive: client without engine reference must raise so the
orchestrator's download_with_fallback surfaces the error and
moves on to the next source. Returning None silently would drop
the download with no user feedback (per JohnBaumb)."""
import pytest
client = DeezerDownloadClient.__new__(DeezerDownloadClient)
client._engine = None
# Bypass auth gate so we exercise the engine check.
client._authenticated = True
with pytest.raises(RuntimeError, match="engine reference"):
_run_async(client.download('deezer', 'v||t', 0))
def test_download_track_id_stays_as_string(deezer_client_with_engine):
"""Pinning: Deezer GW API uses string IDs — engine record must
keep track_id as str."""
client, engine = deezer_client_with_engine
started = threading.Event()
release = threading.Event()
def slow_impl(*args, **kwargs):
started.set()
release.wait(timeout=1.0)
return '/tmp/done.flac'
with patch.object(client, '_download_sync', side_effect=slow_impl):
download_id = _run_async(client.download('deezer_dl', '999||X', 0))
started.wait(timeout=1.0)
record = engine.get_record('deezer', download_id)
assert record['track_id'] == '999'
assert isinstance(record['track_id'], str)
release.set()
def test_download_username_slot_is_legacy_deezer_dl(deezer_client_with_engine):
"""Pinning: frontend status indicators key off `'deezer_dl'`,
not the canonical `'deezer'`."""
client, engine = deezer_client_with_engine
started = threading.Event()
release = threading.Event()
def slow_impl(*args, **kwargs):
started.set()
release.wait(timeout=1.0)
return '/tmp/done.flac'
with patch.object(client, '_download_sync', side_effect=slow_impl):
download_id = _run_async(client.download('deezer_dl', '999||x', 0))
started.wait(timeout=1.0)
assert engine.get_record('deezer', download_id)['username'] == 'deezer_dl'
release.set()
def test_download_handles_missing_display_name_with_fallback(deezer_client_with_engine):
"""Pinning: filename without `||` synthesizes display name `Track <id>`."""
client, engine = deezer_client_with_engine
started = threading.Event()
release = threading.Event()
def slow_impl(*args, **kwargs):
started.set()
release.wait(timeout=1.0)
return '/tmp/x.flac'
with patch.object(client, '_download_sync', side_effect=slow_impl):
download_id = _run_async(client.download('deezer_dl', '12345', 0))
started.wait(timeout=1.0)
assert engine.get_record('deezer', download_id)['display_name'] == 'Track 12345'
release.set()
def test_download_engine_record_carries_error_slot(deezer_client_with_engine):
"""Pinning: Deezer-specific `error` slot for ARL re-auth failure
messages must be present on init."""
client, engine = deezer_client_with_engine
started = threading.Event()
release = threading.Event()
def slow_impl(*args, **kwargs):
started.set()
release.wait(timeout=1.0)
return '/tmp/x.flac'
with patch.object(client, '_download_sync', side_effect=slow_impl):
download_id = _run_async(client.download('deezer_dl', '999||X', 1024))
started.wait(timeout=1.0)
record = engine.get_record('deezer', download_id)
assert 'error' in record
assert record['error'] is None
assert record['size'] == 1024
release.set()
def test_get_all_downloads_reads_engine_records(deezer_client_with_engine):
client, engine = deezer_client_with_engine
engine.add_record('deezer', 'dl-1', {
'id': 'dl-1', 'filename': '111||A', 'username': 'deezer_dl',
'state': 'InProgress, Downloading', 'progress': 50.0,
})
result = _run_async(client.get_all_downloads())
assert len(result) == 1
assert result[0].id == 'dl-1'
assert result[0].username == 'deezer_dl'

View file

@ -0,0 +1,724 @@
"""Tests for the DownloadEngine skeleton (Phase B).
Pinning the engine's state-storage contract: add/update/remove,
per-source iteration, find-by-id, plugin registration, lock-held
mutations vs lock-released reads. Future phases (C/D/E/F) bolt
behavior on top of this surface these tests stay green and act
as the regression net while behavior moves in.
"""
from __future__ import annotations
import threading
import pytest
from core.download_engine import DownloadEngine
# ---------------------------------------------------------------------------
# Plugin registration
# ---------------------------------------------------------------------------
def test_register_plugin_stores_under_source_name():
engine = DownloadEngine()
plugin = object()
engine.register_plugin('soulseek', plugin)
assert engine.get_plugin('soulseek') is plugin
assert 'soulseek' in engine.registered_sources()
def test_get_plugin_returns_none_for_unknown_source():
engine = DownloadEngine()
assert engine.get_plugin('made_up') is None
def test_register_plugin_swallows_set_engine_failure(caplog):
"""Per JohnBaumb: if a plugin's ``set_engine`` callback raises,
registration shouldn't take down the engine — the failure gets
logged + the plugin stays registered. The plugin's own download()
method is responsible for surfacing the missing-engine state to
the user (see ``test_download_raises_when_engine_not_wired`` per
source). Pinning this so a future refactor can't accidentally
propagate the set_engine exception and crash boot.
"""
class _BrokenSetEngine:
def set_engine(self, engine):
raise RuntimeError("plugin's set_engine blew up")
engine = DownloadEngine()
plugin = _BrokenSetEngine()
# Should not raise.
engine.register_plugin('flaky', plugin)
# Plugin still registered + lookupable.
assert engine.get_plugin('flaky') is plugin
assert 'flaky' in engine.registered_sources()
def test_register_plugin_overwrites_on_duplicate(caplog):
"""Re-registering under the same name overwrites and warns. Not a
common path but useful so test fixtures that build a fresh engine
can swap a mock plugin in without setup gymnastics."""
engine = DownloadEngine()
first = object()
second = object()
engine.register_plugin('soulseek', first)
engine.register_plugin('soulseek', second)
assert engine.get_plugin('soulseek') is second
# ---------------------------------------------------------------------------
# Active-download state — add / get / update / remove
# ---------------------------------------------------------------------------
def test_add_record_inserts_under_composite_key():
engine = DownloadEngine()
engine.add_record('youtube', 'dl-1', {'state': 'Initializing', 'progress': 0.0})
rec = engine.get_record('youtube', 'dl-1')
assert rec is not None
assert rec['state'] == 'Initializing'
assert rec['progress'] == 0.0
def test_get_record_returns_shallow_copy():
"""Mutating the returned dict must NOT affect engine state.
Engine reads should be safe to hold / iterate without locks."""
engine = DownloadEngine()
engine.add_record('youtube', 'dl-1', {'state': 'Initializing'})
rec = engine.get_record('youtube', 'dl-1')
rec['state'] = 'TamperedByCaller'
# Engine state still has the original.
fresh = engine.get_record('youtube', 'dl-1')
assert fresh['state'] == 'Initializing'
def test_update_record_applies_partial_patch():
engine = DownloadEngine()
engine.add_record('tidal', 'dl-2', {'state': 'Initializing', 'progress': 0.0,
'file_path': None})
engine.update_record('tidal', 'dl-2', {'state': 'Completed, Succeeded',
'progress': 100.0,
'file_path': '/tmp/song.flac'})
rec = engine.get_record('tidal', 'dl-2')
assert rec['state'] == 'Completed, Succeeded'
assert rec['progress'] == 100.0
assert rec['file_path'] == '/tmp/song.flac'
def test_update_record_is_noop_when_record_removed():
"""If a record was removed (e.g. user cancelled mid-download),
the worker thread's late update is silently dropped — never
raises. Mirrors the per-client `if download_id in active_downloads`
guard pattern that's all over the existing clients."""
engine = DownloadEngine()
engine.add_record('tidal', 'dl-2', {'state': 'Initializing'})
engine.remove_record('tidal', 'dl-2')
# Should not raise.
engine.update_record('tidal', 'dl-2', {'state': 'Completed, Succeeded'})
assert engine.get_record('tidal', 'dl-2') is None
def test_remove_record_returns_removed_record():
engine = DownloadEngine()
engine.add_record('qobuz', 'dl-3', {'state': 'InProgress'})
removed = engine.remove_record('qobuz', 'dl-3')
assert removed is not None
assert removed['state'] == 'InProgress'
assert engine.get_record('qobuz', 'dl-3') is None
def test_remove_record_returns_none_when_missing():
engine = DownloadEngine()
assert engine.remove_record('qobuz', 'never-existed') is None
def test_per_source_locks_dont_block_each_other():
"""Per JohnBaumb: each source must have its own lock so a long-held
write on one source doesn't block writes on another. Pre-refactor
each client owned its own download lock; the engine has to match
that semantic.
Hold source-A's lock from one thread, then mutate source-B from
another thread. Source-B's mutation must complete promptly even
while source-A is locked.
"""
engine = DownloadEngine()
engine.add_record('youtube', 'yt-1', {'state': 'InProgress'})
engine.add_record('tidal', 'td-1', {'state': 'InProgress'})
held = threading.Event()
release = threading.Event()
other_done = threading.Event()
def hold_youtube_lock():
with engine._source_lock('youtube'):
held.set()
release.wait(timeout=2.0)
def update_tidal_while_youtube_held():
held.wait(timeout=1.0)
engine.update_record('tidal', 'td-1', {'state': 'Completed, Succeeded'})
other_done.set()
holder = threading.Thread(target=hold_youtube_lock)
other = threading.Thread(target=update_tidal_while_youtube_held)
holder.start()
other.start()
# Tidal write must complete even though YouTube's lock is held.
assert other_done.wait(timeout=1.0), (
"Tidal mutation blocked by YouTube's lock — sources are not "
"independently shardable"
)
release.set()
holder.join()
other.join()
assert engine.get_record('tidal', 'td-1')['state'] == 'Completed, Succeeded'
def test_remove_record_drops_empty_source_bucket():
"""Per JohnBaumb: nested layout makes per-source iteration
O(source_records). Removing the last record for a source must
also drop the empty source bucket so future iter_records_for_source
calls don't see a stale source key."""
engine = DownloadEngine()
engine.add_record('youtube', 'yt-1', {'title': 'A'})
engine.remove_record('youtube', 'yt-1')
# Source bucket gone — internal state matches the contract.
assert 'youtube' not in engine._records
# Public surface still answers correctly.
assert list(engine.iter_records_for_source('youtube')) == []
assert engine.get_record('youtube', 'yt-1') is None
# ---------------------------------------------------------------------------
# Iteration
# ---------------------------------------------------------------------------
def test_iter_records_for_source_filters_correctly():
engine = DownloadEngine()
engine.add_record('youtube', 'yt-1', {'title': 'A'})
engine.add_record('youtube', 'yt-2', {'title': 'B'})
engine.add_record('tidal', 'td-1', {'title': 'C'})
yt_records = list(engine.iter_records_for_source('youtube'))
assert len(yt_records) == 2
assert {r['title'] for r in yt_records} == {'A', 'B'}
td_records = list(engine.iter_records_for_source('tidal'))
assert len(td_records) == 1
assert td_records[0]['title'] == 'C'
def test_iter_yields_shallow_copies():
"""Iteration returns COPIES — caller can hold the list and mutate
each record without affecting engine state. Important: future
Phase B3's `get_all_downloads` will iterate then build
DownloadStatus objects from the snapshots."""
engine = DownloadEngine()
engine.add_record('youtube', 'yt-1', {'title': 'A'})
snapshot = list(engine.iter_records_for_source('youtube'))
snapshot[0]['title'] = 'TAMPERED'
fresh = engine.get_record('youtube', 'yt-1')
assert fresh['title'] == 'A'
# ---------------------------------------------------------------------------
# Thread safety — basic concurrent-mutation smoke
# ---------------------------------------------------------------------------
def test_concurrent_adds_dont_lose_records():
"""Hammer the engine with concurrent add_record from multiple
threads. With proper locking, every record lands in state.
Future Phase C BackgroundDownloadWorker spawns N threads doing
exactly this kind of mutation."""
engine = DownloadEngine()
def add_records(source, base):
for i in range(50):
engine.add_record(source, f'{base}-{i}', {'i': i})
threads = [
threading.Thread(target=add_records, args=(f'src-{n}', f'dl-{n}'))
for n in range(4)
]
for t in threads:
t.start()
for t in threads:
t.join()
total = sum(
1
for n in range(4)
for _ in engine.iter_records_for_source(f'src-{n}')
)
assert total == 4 * 50 # 200 records, none lost
# ---------------------------------------------------------------------------
# Cross-source query dispatch (Phase B2)
# ---------------------------------------------------------------------------
def _run_async(coro):
import asyncio
loop = asyncio.new_event_loop()
try:
return loop.run_until_complete(coro)
finally:
loop.close()
class _FakePlugin:
"""Minimal plugin double for engine query tests. Exposes the
methods engine.get_all_downloads / get_download_status /
cancel_download / clear_all_completed_downloads call."""
def __init__(self, name, configured=True, downloads=None,
cancel_result=True, clear_result=True):
self.name = name
self._configured = configured
self._downloads = downloads or []
self._cancel_result = cancel_result
self._clear_result = clear_result
self.cancel_calls = []
self.clear_calls = 0
def is_configured(self):
return self._configured
async def get_all_downloads(self):
return list(self._downloads)
async def get_download_status(self, download_id):
for d in self._downloads:
if getattr(d, 'id', None) == download_id:
return d
return None
async def cancel_download(self, download_id, source_hint, remove):
self.cancel_calls.append((download_id, source_hint, remove))
return self._cancel_result
async def clear_all_completed_downloads(self):
self.clear_calls += 1
return self._clear_result
class _FakeStatus:
def __init__(self, id, source):
self.id = id
self.source = source
def test_engine_get_all_downloads_aggregates_across_plugins():
"""Engine concatenates every plugin's get_all_downloads output —
same behavior as the legacy orchestrator."""
engine = DownloadEngine()
yt_plugin = _FakePlugin('youtube', downloads=[_FakeStatus('yt-1', 'youtube')])
td_plugin = _FakePlugin('tidal', downloads=[_FakeStatus('td-1', 'tidal'),
_FakeStatus('td-2', 'tidal')])
engine.register_plugin('youtube', yt_plugin)
engine.register_plugin('tidal', td_plugin)
result = _run_async(engine.get_all_downloads())
assert len(result) == 3
assert {r.id for r in result} == {'yt-1', 'td-1', 'td-2'}
def test_engine_get_all_downloads_excludes_dont_invoke_plugin():
"""Per JohnBaumb: the monitor calls engine.get_all_downloads(
exclude=('soulseek',)) AFTER already pulling slskd transfers via
the transfers/downloads endpoint. The whole point of the exclude
is to NOT touch soulseek's plugin a second time — so the plugin's
get_all_downloads must literally not be called when its name is
in the exclude list. Pin that semantic explicitly (a test that
just checks IDs would pass even if soulseek was called and
returned [])."""
engine = DownloadEngine()
sl_plugin = _FakePlugin('soulseek', downloads=[_FakeStatus('sl-1', 'soulseek')])
yt_plugin = _FakePlugin('youtube', downloads=[_FakeStatus('yt-1', 'youtube')])
engine.register_plugin('soulseek', sl_plugin)
engine.register_plugin('youtube', yt_plugin)
# Sentinel — flips True if get_all_downloads is called on soulseek.
soulseek_called = []
original_get_all = sl_plugin.get_all_downloads
async def _tracking_get_all():
soulseek_called.append(True)
return await original_get_all()
sl_plugin.get_all_downloads = _tracking_get_all
_run_async(engine.get_all_downloads(exclude=('soulseek',)))
assert soulseek_called == [], (
"soulseek's get_all_downloads was invoked despite being in exclude — "
"monitor would still double-fetch slskd"
)
def test_engine_get_all_downloads_skips_excluded_sources():
"""Per JohnBaumb: monitor pulls slskd transfers via the
transfers/downloads endpoint earlier in its loop, so engine
aggregation must skip soulseek to avoid double-fetching."""
engine = DownloadEngine()
sl_plugin = _FakePlugin('soulseek', downloads=[_FakeStatus('sl-1', 'soulseek')])
yt_plugin = _FakePlugin('youtube', downloads=[_FakeStatus('yt-1', 'youtube')])
td_plugin = _FakePlugin('tidal', downloads=[_FakeStatus('td-1', 'tidal')])
engine.register_plugin('soulseek', sl_plugin)
engine.register_plugin('youtube', yt_plugin)
engine.register_plugin('tidal', td_plugin)
result = _run_async(engine.get_all_downloads(exclude=('soulseek',)))
ids = {r.id for r in result}
assert ids == {'yt-1', 'td-1'}
assert 'sl-1' not in ids
def test_engine_get_all_downloads_swallows_per_plugin_exceptions():
"""One plugin throwing must NOT take down the whole list — same
defensive behavior as the legacy orchestrator (matched by
`try ... except: pass` on every iteration)."""
engine = DownloadEngine()
class _BrokenPlugin:
async def get_all_downloads(self):
raise RuntimeError("boom")
yt_plugin = _FakePlugin('youtube', downloads=[_FakeStatus('yt-1', 'youtube')])
engine.register_plugin('broken', _BrokenPlugin())
engine.register_plugin('youtube', yt_plugin)
result = _run_async(engine.get_all_downloads())
assert [r.id for r in result] == ['yt-1']
def test_engine_get_download_status_returns_first_match():
engine = DownloadEngine()
yt_plugin = _FakePlugin('youtube', downloads=[_FakeStatus('shared', 'youtube')])
td_plugin = _FakePlugin('tidal', downloads=[])
engine.register_plugin('youtube', yt_plugin)
engine.register_plugin('tidal', td_plugin)
result = _run_async(engine.get_download_status('shared'))
assert result is not None
assert result.id == 'shared'
def test_engine_cancel_routes_streaming_source_directly():
"""When source_hint is a known streaming-source name (not
'soulseek'), engine routes the cancel to that specific plugin
only doesn't ask every other plugin first."""
engine = DownloadEngine()
yt_plugin = _FakePlugin('youtube')
td_plugin = _FakePlugin('tidal')
engine.register_plugin('youtube', yt_plugin)
engine.register_plugin('tidal', td_plugin)
_run_async(engine.cancel_download('dl-1', 'tidal', remove=False))
assert yt_plugin.cancel_calls == []
assert td_plugin.cancel_calls == [('dl-1', 'tidal', False)]
def test_engine_cancel_routes_unknown_source_hint_to_soulseek():
"""A username that's NOT in the plugin registry is a real
Soulseek peer name route to the soulseek plugin."""
engine = DownloadEngine()
sl_plugin = _FakePlugin('soulseek')
yt_plugin = _FakePlugin('youtube')
engine.register_plugin('soulseek', sl_plugin)
engine.register_plugin('youtube', yt_plugin)
_run_async(engine.cancel_download('dl-1', 'random_peer_username', remove=False))
assert sl_plugin.cancel_calls == [('dl-1', 'random_peer_username', False)]
assert yt_plugin.cancel_calls == []
def test_engine_cancel_falls_back_to_iterating_all_plugins_without_hint():
"""No source hint → ask every plugin until one accepts the
cancel (returns True). Mirrors legacy orchestrator behavior."""
engine = DownloadEngine()
yt_plugin = _FakePlugin('youtube', cancel_result=False)
td_plugin = _FakePlugin('tidal', cancel_result=True)
engine.register_plugin('youtube', yt_plugin)
engine.register_plugin('tidal', td_plugin)
result = _run_async(engine.cancel_download('dl-1', None, remove=False))
assert result is True
# Both plugins were asked; tidal accepted.
assert len(yt_plugin.cancel_calls) == 1
assert len(td_plugin.cancel_calls) == 1
def test_engine_clear_all_skips_unconfigured_plugins():
"""Unconfigured plugins are silently skipped (no API call, no
error) matches legacy orchestrator's defensive handling."""
engine = DownloadEngine()
configured = _FakePlugin('youtube', configured=True, clear_result=True)
unconfigured = _FakePlugin('tidal', configured=False)
engine.register_plugin('youtube', configured)
engine.register_plugin('tidal', unconfigured)
result = _run_async(engine.clear_all_completed_downloads())
assert result is True
assert configured.clear_calls == 1
assert unconfigured.clear_calls == 0
def test_engine_clear_all_returns_false_when_any_configured_plugin_fails():
engine = DownloadEngine()
failing = _FakePlugin('youtube', configured=True, clear_result=False)
engine.register_plugin('youtube', failing)
result = _run_async(engine.clear_all_completed_downloads())
assert result is False
# ---------------------------------------------------------------------------
# Hybrid fallback (Phase F)
# ---------------------------------------------------------------------------
class _FakeSearchPlugin:
def __init__(self, name, configured=True, search_result=None, raises=None):
self.name = name
self._configured = configured
self._search_result = search_result if search_result is not None else ([], [])
self._raises = raises
self.search_calls = 0
def is_configured(self):
return self._configured
async def search(self, query, timeout=None, progress_callback=None):
self.search_calls += 1
if self._raises:
raise self._raises
return self._search_result
class _FakeDownloadPlugin:
def __init__(self, name, configured=True, download_result=None, raises=None):
self.name = name
self._configured = configured
self._download_result = download_result
self._raises = raises
self.download_calls = []
def is_configured(self):
return self._configured
async def download(self, username, filename, file_size):
self.download_calls.append((username, filename, file_size))
if self._raises:
raise self._raises
return self._download_result
def test_search_with_fallback_returns_first_non_empty_result():
engine = DownloadEngine()
yt = _FakeSearchPlugin('youtube', search_result=([], []))
td = _FakeSearchPlugin('tidal', search_result=(['track1'], []))
qz = _FakeSearchPlugin('qobuz', search_result=(['track2'], []))
engine.register_plugin('youtube', yt)
engine.register_plugin('tidal', td)
engine.register_plugin('qobuz', qz)
tracks, _ = _run_async(engine.search_with_fallback('q', ['youtube', 'tidal', 'qobuz']))
assert tracks == ['track1']
# Tidal short-circuits — qobuz never queried.
assert yt.search_calls == 1
assert td.search_calls == 1
assert qz.search_calls == 0
def test_search_with_fallback_skips_unconfigured_plugins():
engine = DownloadEngine()
yt = _FakeSearchPlugin('youtube', configured=False)
td = _FakeSearchPlugin('tidal', configured=True, search_result=(['hit'], []))
engine.register_plugin('youtube', yt)
engine.register_plugin('tidal', td)
tracks, _ = _run_async(engine.search_with_fallback('q', ['youtube', 'tidal']))
assert tracks == ['hit']
assert yt.search_calls == 0 # skipped
def test_search_with_fallback_continues_after_per_source_exception():
engine = DownloadEngine()
yt = _FakeSearchPlugin('youtube', raises=RuntimeError("yt down"))
td = _FakeSearchPlugin('tidal', search_result=(['fallback-hit'], []))
engine.register_plugin('youtube', yt)
engine.register_plugin('tidal', td)
tracks, _ = _run_async(engine.search_with_fallback('q', ['youtube', 'tidal']))
assert tracks == ['fallback-hit']
def test_search_with_fallback_returns_empty_when_chain_exhausted():
engine = DownloadEngine()
yt = _FakeSearchPlugin('youtube', search_result=([], []))
td = _FakeSearchPlugin('tidal', search_result=([], []))
engine.register_plugin('youtube', yt)
engine.register_plugin('tidal', td)
tracks, _ = _run_async(engine.search_with_fallback('q', ['youtube', 'tidal']))
assert tracks == []
assert yt.search_calls == 1
assert td.search_calls == 1
def test_download_with_fallback_returns_first_accepted_download_id():
"""Phase F bug fix: legacy hybrid download routed to one source
via username hint with no retry. Engine now falls through chain."""
engine = DownloadEngine()
yt = _FakeDownloadPlugin('youtube', download_result=None) # refuses
td = _FakeDownloadPlugin('tidal', download_result='td-id')
engine.register_plugin('youtube', yt)
engine.register_plugin('tidal', td)
result = _run_async(engine.download_with_fallback(
'youtube', 'v||t', 0, ['youtube', 'tidal'],
))
assert result == 'td-id'
assert len(yt.download_calls) == 1 # tried first
assert len(td.download_calls) == 1 # took over
def test_download_with_fallback_promotes_username_hint_to_head():
"""A username hint that matches a source-chain entry tries that
source FIRST regardless of declared chain order."""
engine = DownloadEngine()
yt = _FakeDownloadPlugin('youtube', download_result='yt-id')
td = _FakeDownloadPlugin('tidal', download_result='td-id')
engine.register_plugin('youtube', yt)
engine.register_plugin('tidal', td)
# Chain says tidal-first, but username hint promotes youtube.
result = _run_async(engine.download_with_fallback(
'youtube', 'v||t', 0, ['tidal', 'youtube'],
))
assert result == 'yt-id'
assert len(yt.download_calls) == 1
assert len(td.download_calls) == 0 # never reached
def test_download_with_fallback_returns_none_when_all_refuse():
engine = DownloadEngine()
yt = _FakeDownloadPlugin('youtube', download_result=None)
td = _FakeDownloadPlugin('tidal', download_result=None)
engine.register_plugin('youtube', yt)
engine.register_plugin('tidal', td)
result = _run_async(engine.download_with_fallback(
'youtube', 'v||t', 0, ['youtube', 'tidal'],
))
assert result is None
assert len(yt.download_calls) == 1
assert len(td.download_calls) == 1
def test_download_with_fallback_continues_past_exception():
engine = DownloadEngine()
yt = _FakeDownloadPlugin('youtube', raises=RuntimeError("yt died"))
td = _FakeDownloadPlugin('tidal', download_result='td-id')
engine.register_plugin('youtube', yt)
engine.register_plugin('tidal', td)
result = _run_async(engine.download_with_fallback(
'youtube', 'v||t', 0, ['youtube', 'tidal'],
))
assert result == 'td-id'
# ---------------------------------------------------------------------------
# Cin bug 1: alias resolution on cancel_download
# ---------------------------------------------------------------------------
def test_register_plugin_records_aliases():
"""Aliases passed to register_plugin resolve to the canonical plugin
via get_plugin. Cin caught engine.cancel_download routing 'deezer_dl'
to soulseek because the alias never made it to the engine."""
engine = DownloadEngine()
deezer = _FakePlugin('deezer')
engine.register_plugin('deezer', deezer, aliases=('deezer_dl',))
assert engine.get_plugin('deezer') is deezer
assert engine.get_plugin('deezer_dl') is deezer
def test_cancel_download_resolves_alias_to_canonical_plugin():
"""The legacy 'deezer_dl' source_hint must route to the deezer
plugin, not fall through to soulseek. This was Cin's bug 1 —
cancel of a Deezer download silently no-op'd."""
engine = DownloadEngine()
soulseek = _FakePlugin('soulseek')
deezer = _FakePlugin('deezer')
engine.register_plugin('soulseek', soulseek)
engine.register_plugin('deezer', deezer, aliases=('deezer_dl',))
_run_async(engine.cancel_download('dl-1', 'deezer_dl', remove=False))
assert deezer.cancel_calls == [('dl-1', 'deezer_dl', False)]
assert soulseek.cancel_calls == []
# ---------------------------------------------------------------------------
# Cin bug 3: atomic update_record_unless_state
# ---------------------------------------------------------------------------
def test_update_record_unless_state_applies_when_state_not_blocked():
engine = DownloadEngine()
engine.add_record('youtube', 'dl-1', {'state': 'InProgress, Downloading'})
applied = engine.update_record_unless_state(
'youtube', 'dl-1',
{'state': 'Completed, Succeeded', 'progress': 100.0},
skip_if_state_in=('Cancelled',),
)
assert applied is True
assert engine.get_record('youtube', 'dl-1')['state'] == 'Completed, Succeeded'
assert engine.get_record('youtube', 'dl-1')['progress'] == 100.0
def test_update_record_unless_state_skips_when_state_blocked():
"""A worker-thread terminal write must NOT clobber a Cancelled
state set by the user. Returns False so caller knows the patch
was skipped."""
engine = DownloadEngine()
engine.add_record('youtube', 'dl-1', {'state': 'Cancelled'})
applied = engine.update_record_unless_state(
'youtube', 'dl-1',
{'state': 'Completed, Succeeded'},
skip_if_state_in=('Cancelled',),
)
assert applied is False
assert engine.get_record('youtube', 'dl-1')['state'] == 'Cancelled'
def test_update_record_unless_state_returns_false_for_missing_record():
engine = DownloadEngine()
applied = engine.update_record_unless_state(
'youtube', 'never-existed',
{'state': 'Completed, Succeeded'},
skip_if_state_in=('Cancelled',),
)
assert applied is False

View file

@ -1,4 +1,6 @@
from core.download_engine import DownloadEngine
from core.download_orchestrator import DownloadOrchestrator
from core.download_plugins.registry import DownloadPluginRegistry, PluginSpec
class _FakeClient:
@ -16,15 +18,45 @@ class _FakeClient:
def _build_orchestrator(**clients):
"""Build an orchestrator with mock clients via the registry.
The orchestrator iterates `self.registry.all_plugins()` to drive
every per-source operation, so the test must set up a real
registry with mock plugins (not just stuff attributes on the
orchestrator). Source slots not provided in `clients` are
skipped registry only holds the ones the test cares about.
"""
registry = DownloadPluginRegistry()
name_to_display = {
'soulseek': 'Soulseek', 'youtube': 'YouTube', 'tidal': 'Tidal',
'qobuz': 'Qobuz', 'hifi': 'HiFi', 'deezer_dl': 'Deezer',
'lidarr': 'Lidarr', 'soundcloud': 'SoundCloud',
}
# 'deezer_dl' is the legacy attr name; canonical registry name is 'deezer'.
aliases_for = {'deezer_dl': ('deezer_dl',)}
canonical_for = {'deezer_dl': 'deezer'}
for slot, client in clients.items():
if client is None:
continue
canonical_name = canonical_for.get(slot, slot)
registry.register(PluginSpec(
name=canonical_name,
factory=lambda c=client: c,
display_name=name_to_display.get(slot, slot),
aliases=aliases_for.get(slot, ()),
))
registry.initialize()
orch = DownloadOrchestrator.__new__(DownloadOrchestrator)
orch.soulseek = clients.get("soulseek")
orch.youtube = clients.get("youtube")
orch.tidal = clients.get("tidal")
orch.qobuz = clients.get("qobuz")
orch.hifi = clients.get("hifi")
orch.deezer_dl = clients.get("deezer_dl")
orch.lidarr = clients.get("lidarr")
orch.soundcloud = clients.get("soundcloud")
orch.registry = registry
orch._init_failures = registry.init_failures
# Engine — orchestrator delegates per-source query/cancel
# methods to it, so the test fixture must build one and
# register every mock plugin under its canonical name.
orch.engine = DownloadEngine()
for source_name, plugin in registry.all_plugins():
orch.engine.register_plugin(source_name, plugin)
return orch
@ -47,8 +79,8 @@ def test_clear_all_completed_downloads_ignores_unconfigured_clients():
result = _run_async(orch.clear_all_completed_downloads())
assert result is True
assert orch.soulseek.clear_calls == 1
assert orch.youtube.clear_calls == 0
assert orch.client('soulseek').clear_calls == 1
assert orch.client('youtube').clear_calls == 0
def test_clear_all_completed_downloads_propagates_configured_failures():
@ -59,4 +91,177 @@ def test_clear_all_completed_downloads_propagates_configured_failures():
result = _run_async(orch.clear_all_completed_downloads())
assert result is False
assert orch.soulseek.clear_calls == 1
assert orch.client('soulseek').clear_calls == 1
# ---------------------------------------------------------------------------
# Cin-2 generic accessors
# ---------------------------------------------------------------------------
def test_client_returns_registered_client_by_name():
"""Cin's review feedback: orch.client('hifi') is the canonical
way to reach a per-source client, replacing orch.hifi attribute
access."""
soulseek = _FakeClient()
youtube = _FakeClient()
orch = _build_orchestrator(soulseek=soulseek, youtube=youtube)
assert orch.client('soulseek') is soulseek
assert orch.client('youtube') is youtube
assert orch.client('made_up') is None
def test_configured_clients_excludes_unconfigured_sources():
"""Replaces the legacy iteration pattern: 6+ if/hasattr/is_configured
checks per source. Single call returns dict of configured clients."""
configured = _FakeClient(configured=True)
unconfigured = _FakeClient(configured=False)
orch = _build_orchestrator(
soulseek=configured,
youtube=unconfigured,
)
result = orch.configured_clients()
assert 'soulseek' in result
assert 'youtube' not in result
assert result['soulseek'] is configured
def test_configured_clients_skips_clients_whose_is_configured_raises():
"""Per JohnBaumb: configured_clients() has a try/except so a single
broken is_configured() call doesn't crash the whole iteration —
pin it so a future refactor can't quietly drop the guard. The
broken plugin is skipped; the rest still come back."""
class _BrokenIsConfigured(_FakeClient):
def is_configured(self):
raise RuntimeError("is_configured blew up")
broken = _BrokenIsConfigured()
healthy = _FakeClient(configured=True)
orch = _build_orchestrator(soulseek=healthy, youtube=broken)
result = orch.configured_clients()
# Healthy plugin still surfaces; broken one is silently skipped.
assert 'soulseek' in result
assert result['soulseek'] is healthy
assert 'youtube' not in result
def test_reload_instances_dispatches_to_named_source():
"""Generic dispatch — caller passes source name instead of
reaching for orch.hifi.reload_instances() directly."""
class _ReloadableClient(_FakeClient):
def __init__(self):
super().__init__(configured=True)
self.reload_called = False
def reload_instances(self):
self.reload_called = True
hifi = _ReloadableClient()
soulseek = _FakeClient() # No reload_instances method
orch = _build_orchestrator(soulseek=soulseek, hifi=hifi)
assert orch.reload_instances('hifi') is True
assert hifi.reload_called is True
def test_reload_instances_skips_clients_without_method():
"""Sources that don't expose reload_instances are skipped, not
treated as failures."""
soulseek = _FakeClient() # No reload_instances method
orch = _build_orchestrator(soulseek=soulseek)
# Calling on a source without the method = silent no-op
assert orch.reload_instances('soulseek') is True
def test_reload_instances_with_no_args_reloads_every_source():
"""When called with no source argument, hits every registered
source that exposes reload_instances."""
class _ReloadableClient(_FakeClient):
def __init__(self):
super().__init__()
self.reload_called = False
def reload_instances(self):
self.reload_called = True
a = _ReloadableClient()
b = _ReloadableClient()
orch = _build_orchestrator(soulseek=a, hifi=b)
orch.reload_instances()
assert a.reload_called is True
assert b.reload_called is True
# ---------------------------------------------------------------------------
# Singleton factory (matches Cin's get_metadata_engine pattern)
# ---------------------------------------------------------------------------
# ---------------------------------------------------------------------------
# Cin bug 2: hybrid_order alias normalization
# ---------------------------------------------------------------------------
def test_resolve_source_chain_normalizes_legacy_aliases():
"""Cin's bug 2: hybrid_order config containing the legacy alias
'deezer_dl' was silently dropped because the canonical-name
membership check rejected it. Orchestrator must normalize via
the registry alias map first."""
orch = _build_orchestrator(
soulseek=_FakeClient(),
deezer_dl=_FakeClient(),
youtube=_FakeClient(),
)
orch.hybrid_order = ['deezer_dl', 'soulseek', 'youtube']
orch.hybrid_primary = None
orch.hybrid_secondary = None
chain = orch._resolve_source_chain()
assert chain == ['deezer', 'soulseek', 'youtube']
def test_resolve_source_chain_dedupes_alias_and_canonical():
"""If both 'deezer' and 'deezer_dl' appear, dedupe to single entry."""
orch = _build_orchestrator(
soulseek=_FakeClient(),
deezer_dl=_FakeClient(),
)
orch.hybrid_order = ['deezer_dl', 'deezer', 'soulseek']
orch.hybrid_primary = None
orch.hybrid_secondary = None
chain = orch._resolve_source_chain()
assert chain == ['deezer', 'soulseek']
def test_resolve_source_chain_drops_unknown_names():
orch = _build_orchestrator(soulseek=_FakeClient(), youtube=_FakeClient())
orch.hybrid_order = ['nonsense', 'soulseek', 'also_fake', 'youtube']
orch.hybrid_primary = None
orch.hybrid_secondary = None
chain = orch._resolve_source_chain()
assert chain == ['soulseek', 'youtube']
def test_get_download_orchestrator_returns_set_singleton():
"""When set_download_orchestrator has been called (web_server.py
does this at boot), get_download_orchestrator returns the
installed instance instead of building a fresh one."""
from core.download_orchestrator import (
get_download_orchestrator,
set_download_orchestrator,
)
orch = _build_orchestrator(soulseek=_FakeClient())
set_download_orchestrator(orch)
try:
assert get_download_orchestrator() is orch
finally:
set_download_orchestrator(None)

View file

@ -101,7 +101,7 @@ def _build_deps(
on_complete=None,
):
deps = dc.CandidatesDeps(
soulseek_client=soulseek or _FakeSoulseek(),
download_orchestrator=soulseek or _FakeSoulseek(),
spotify_client=spotify or _FakeSpotify(),
run_async=_run_async,
get_database=lambda: db or _FakeDB(),
@ -136,7 +136,7 @@ def test_first_candidate_starts_download_and_returns_true():
result = dc.attempt_download_with_candidates("t1", candidates, track, batch_id="b1", deps=deps)
assert result is True
assert deps.soulseek_client.download_calls == [("user1", "best.flac", 1000)]
assert deps.download_orchestrator.download_calls == [("user1", "best.flac", 1000)]
assert download_tasks["t1"]["download_id"] == "dl-1"
assert "user1::best.flac" in matched_downloads_context
@ -155,7 +155,7 @@ def test_candidates_tried_in_confidence_order():
dc.attempt_download_with_candidates("t2", candidates, track, batch_id=None, deps=deps)
# First call should be the highest-confidence one
assert deps.soulseek_client.download_calls[0][1] == "high.flac"
assert deps.download_orchestrator.download_calls[0][1] == "high.flac"
# ---------------------------------------------------------------------------
@ -175,8 +175,8 @@ def test_already_tried_source_skipped():
dc.attempt_download_with_candidates("t3", candidates, track, batch_id=None, deps=deps)
# First candidate skipped (already used), second one tried
assert len(deps.soulseek_client.download_calls) == 1
assert deps.soulseek_client.download_calls[0][1] == "fresh.flac"
assert len(deps.download_orchestrator.download_calls) == 1
assert deps.download_orchestrator.download_calls[0][1] == "fresh.flac"
# ---------------------------------------------------------------------------
@ -196,7 +196,7 @@ def test_blacklisted_source_skipped():
dc.attempt_download_with_candidates("t4", candidates, track, batch_id=None, deps=deps)
assert deps.soulseek_client.download_calls[0][1] == "ok.flac"
assert deps.download_orchestrator.download_calls[0][1] == "ok.flac"
# ---------------------------------------------------------------------------
@ -213,7 +213,7 @@ def test_cancellation_before_attempt_returns_false():
result = dc.attempt_download_with_candidates("t5", candidates, track, batch_id=None, deps=deps)
assert result is False
assert deps.soulseek_client.download_calls == []
assert deps.download_orchestrator.download_calls == []
def test_task_deleted_returns_false():
@ -238,7 +238,7 @@ def test_active_download_id_skips_new_download():
dc.attempt_download_with_candidates("t6", candidates, track, batch_id=None, deps=deps)
# Both candidates skipped (download_id already present)
assert deps.soulseek_client.download_calls == []
assert deps.download_orchestrator.download_calls == []
def test_cancellation_after_download_starts_calls_cancel_and_lifecycle():
@ -266,7 +266,7 @@ def test_cancellation_after_download_starts_calls_cancel_and_lifecycle():
assert result is False
# cancel_download was called for the in-flight transfer
assert deps.soulseek_client.cancel_calls
assert deps.download_orchestrator.cancel_calls
# on_download_completed fired with success=False to free the worker slot
assert completion_calls == [("b7", "t7", False)]
@ -276,7 +276,7 @@ def test_cancellation_after_download_starts_calls_cancel_and_lifecycle():
# ---------------------------------------------------------------------------
def test_all_candidates_failed_returns_false():
"""If soulseek_client.download returns None (failure) for all candidates, returns False."""
"""If download_orchestrator.download returns None (failure) for all candidates, returns False."""
soulseek = _FakeSoulseek(download_id=None)
deps = _build_deps(soulseek=soulseek)
_seed_task("t8")
@ -411,5 +411,5 @@ def test_candidates_with_equal_confidence_both_tried():
dc.attempt_download_with_candidates("t13", candidates, track, batch_id=None, deps=deps)
# First one wins — second never tried because download succeeded
assert len(deps.soulseek_client.download_calls) == 1
assert deps.soulseek_client.download_calls[0][1] == "a.flac"
assert len(deps.download_orchestrator.download_calls) == 1
assert deps.download_orchestrator.download_calls[0][1] == "a.flac"

View file

@ -178,7 +178,7 @@ def _build_deps(
):
return mw.MasterDeps(
config_manager=config or _FakeConfig(),
soulseek_client=soulseek or _FakeSoulseekWrapper(_FakeSoulseek()),
download_orchestrator=soulseek or _FakeSoulseekWrapper(_FakeSoulseek()),
run_async=run_async or _make_run_async(),
mb_worker=mb_worker,
mb_release_cache=mb_release_cache if mb_release_cache is not None else {},

View file

@ -49,7 +49,7 @@ class _Recorder:
def _build_deps(
*,
config=None,
soulseek_client=None,
download_orchestrator=None,
run_async=None,
docker_resolve_path=None,
extract_filename=None,
@ -64,7 +64,7 @@ def _build_deps(
rec = _Recorder()
return pp.PostProcessDeps(
config_manager=config or _FakeConfig(),
soulseek_client=soulseek_client,
download_orchestrator=download_orchestrator,
run_async=run_async or (lambda c: None),
docker_resolve_path=docker_resolve_path or (lambda p: p),
extract_filename=extract_filename or (lambda f: os.path.basename(f) if f else ''),
@ -315,7 +315,7 @@ def test_youtube_task_uses_get_download_status_to_resolve_path(monkeypatch):
monkeypatch.setattr(pp.os.path, 'exists', lambda p: p == '/downloads/Money.mp3')
deps, rec = _build_deps(
soulseek_client=_FakeYTClient(),
download_orchestrator=_FakeYTClient(),
run_async=lambda coro: coro, # not async — direct call
)
pp.run_post_processing_worker('t1', 'b1', deps)

View file

@ -31,13 +31,28 @@ class _Recorder:
class _FakeClient:
"""Stub soulseek client. `mode` defaults to non-hybrid."""
"""Stub download orchestrator. `mode` defaults to non-hybrid.
Per-source stubs passed via ``subclients`` are surfaced through
``client(name)`` (the registry-backed accessor on the real
orchestrator); plain attrs (hybrid_order, hybrid_primary, etc.)
are set as attributes so getattr() lookups still resolve them."""
_CLIENT_NAMES = {'soulseek', 'youtube', 'tidal', 'qobuz', 'hifi',
'deezer_dl', 'lidarr', 'soundcloud'}
def __init__(self, results=None, mode='soulseek', subclients=None):
self._results = results if results is not None else []
self.mode = mode
self.search_calls = []
self._client_map = {}
for k, v in (subclients or {}).items():
setattr(self, k, v)
if k in self._CLIENT_NAMES:
self._client_map[k] = v
else:
# config-style attrs (hybrid_order, hybrid_primary, etc.)
setattr(self, k, v)
def client(self, name):
return self._client_map.get(name)
async def search(self, query, timeout=30):
self.search_calls.append((query, timeout))
@ -76,7 +91,7 @@ def _build_deps(
):
rec = _Recorder()
return tw.TaskWorkerDeps(
soulseek_client=soulseek or _FakeClient(),
download_orchestrator=soulseek or _FakeClient(),
matching_engine=matching or _FakeMatchEngine(),
run_async=_sync_run_async,
try_source_reuse=try_source_reuse,

View file

@ -0,0 +1,92 @@
"""Tests for core/downloads/validation.py — SoundCloud preview filter.
The SoundCloud anonymous tier serves a ~30s preview clip for tracks
gated behind Go+ / login. ``filter_soundcloud_previews`` drops these
candidates before they reach the matcher, the modal cache, or the
manual-pick download path.
"""
from __future__ import annotations
from dataclasses import dataclass
from typing import Optional
from core.downloads.validation import filter_soundcloud_previews
@dataclass
class _Track:
duration_ms: int
@dataclass
class _Candidate:
username: str
duration: Optional[int] # milliseconds
title: str = ''
def test_drops_soundcloud_30s_preview_when_expected_long():
"""A 30s SC candidate against a 5-minute expected track is the
canonical preview-snippet case must be dropped."""
expected = _Track(duration_ms=338_000) # ~5:38
cands = [
_Candidate(username='soundcloud', duration=30_000, title='Preview'),
_Candidate(username='soundcloud', duration=338_000, title='Real'),
]
result = filter_soundcloud_previews(cands, expected)
assert len(result) == 1
assert result[0].title == 'Real'
def test_drops_under_half_expected_duration():
"""SC candidate at 100s against 300s expected = clearly truncated /
wrong content. Must be dropped even if not at the 30s boundary."""
expected = _Track(duration_ms=300_000)
cand = _Candidate(username='soundcloud', duration=100_000)
assert filter_soundcloud_previews([cand], expected) == []
def test_keeps_soundcloud_when_expected_track_is_short():
"""Genuinely short SC tracks (intros, sound effects, sub-minute
songs) must pass through when the expected track is also short.
Filter only kicks in when expected > 60s."""
expected = _Track(duration_ms=45_000) # 45s expected
cand = _Candidate(username='soundcloud', duration=30_000)
result = filter_soundcloud_previews([cand], expected)
assert result == [cand]
def test_does_not_filter_non_soundcloud_sources():
"""A 30s candidate from another streaming source isn't a SoundCloud
preview leave it for the generic matching engine to score."""
expected = _Track(duration_ms=338_000)
yt = _Candidate(username='youtube', duration=30_000)
tidal = _Candidate(username='tidal', duration=30_000)
assert filter_soundcloud_previews([yt, tidal], expected) == [yt, tidal]
def test_returns_input_unchanged_without_expected_duration():
"""Without a Spotify-track / expected duration we can't reason
about previews pass everything through."""
cands = [
_Candidate(username='soundcloud', duration=30_000),
_Candidate(username='soundcloud', duration=300_000),
]
assert filter_soundcloud_previews(cands, None) == cands
assert filter_soundcloud_previews(cands, _Track(duration_ms=0)) == cands
def test_empty_input_returns_empty_list():
assert filter_soundcloud_previews([], _Track(duration_ms=200_000)) == []
def test_keeps_soundcloud_candidate_at_threshold():
"""Boundary check: 35s candidate against 200s expected — exactly
at the 35s preview boundary, but 35s is also above
expected*0.5 (100s) check (35 < 100, so still drops). Use a
higher value to confirm the just-above threshold passes."""
expected = _Track(duration_ms=200_000) # 200s
# 110s passes both checks: > 35s AND > 100s (half of 200s)
cand = _Candidate(username='soundcloud', duration=110_000)
assert filter_soundcloud_previews([cand], expected) == [cand]

View file

@ -0,0 +1,104 @@
"""Phase A pinning tests for HiFiClient — UPDATED for Phase C5."""
from __future__ import annotations
import asyncio
import threading
from pathlib import Path
from unittest.mock import patch
import pytest
from core.download_engine import DownloadEngine
from core.hifi_client import HiFiClient
def _run_async(coro):
loop = asyncio.new_event_loop()
try:
return loop.run_until_complete(coro)
finally:
loop.close()
@pytest.fixture
def hifi_client_with_engine():
client = HiFiClient.__new__(HiFiClient)
client.download_path = Path('./test_hifi_downloads')
client.shutdown_check = None
client._engine = None
engine = DownloadEngine()
client.set_engine(engine)
return client, engine
def test_download_returns_none_for_invalid_filename_format(hifi_client_with_engine):
client, _ = hifi_client_with_engine
result = _run_async(client.download('hifi', 'no-separator', 0))
assert result is None
def test_download_returns_none_for_non_integer_track_id(hifi_client_with_engine):
client, _ = hifi_client_with_engine
result = _run_async(client.download('hifi', 'not-int||title', 0))
assert result is None
def test_download_raises_when_engine_not_wired():
"""Defensive: client without engine reference must raise so the
orchestrator's download_with_fallback surfaces the error and
moves on to the next source. Returning None silently would drop
the download with no user feedback (per JohnBaumb)."""
import pytest
client = HiFiClient.__new__(HiFiClient)
client._engine = None
with pytest.raises(RuntimeError, match="engine reference"):
_run_async(client.download('hifi', 'v||t', 0))
def test_download_returns_uuid_for_valid_filename(hifi_client_with_engine):
client, _ = hifi_client_with_engine
with patch.object(client, '_download_sync', return_value='/tmp/x.flac'):
result = _run_async(client.download('hifi', '12345||Some Song', 0))
assert result is not None
assert len(result) == 36
def test_download_populates_engine_record_with_initial_state(hifi_client_with_engine):
client, engine = hifi_client_with_engine
started = threading.Event()
release = threading.Event()
def slow_impl(*args, **kwargs):
started.set()
release.wait(timeout=1.0)
return '/tmp/done.flac'
with patch.object(client, '_download_sync', side_effect=slow_impl):
download_id = _run_async(client.download('hifi', '999||My HiFi Song', 0))
started.wait(timeout=1.0)
record = engine.get_record('hifi', download_id)
assert record['filename'] == '999||My HiFi Song'
assert record['username'] == 'hifi'
assert record['track_id'] == 999
assert record['display_name'] == 'My HiFi Song'
release.set()
def test_get_all_downloads_reads_engine_records(hifi_client_with_engine):
client, engine = hifi_client_with_engine
engine.add_record('hifi', 'dl-1', {
'id': 'dl-1', 'filename': '111||A', 'username': 'hifi',
'state': 'InProgress, Downloading', 'progress': 50.0,
})
result = _run_async(client.get_all_downloads())
assert len(result) == 1
assert result[0].id == 'dl-1'
def test_cancel_download_marks_cancelled(hifi_client_with_engine):
client, engine = hifi_client_with_engine
engine.add_record('hifi', 'dl-1', {'id': 'dl-1', 'state': 'InProgress, Downloading'})
ok = _run_async(client.cancel_download('dl-1', None, remove=False))
assert ok is True
assert engine.get_record('hifi', 'dl-1')['state'] == 'Cancelled'

View file

@ -0,0 +1,138 @@
"""Phase A pinning tests for LidarrDownloadClient's download lifecycle.
Lidarr is the special case in the dispatcher it's an
ALBUM-grabber, not a track-grabber. When the user asks for a
track, Lidarr grabs the whole album, then we pick the wanted
track out (logic at the end of `_download_thread_worker`).
Engine refactor's plugin contract must accommodate album-only
sources OR Lidarr stays special. Pinning the current contract
forces the design decision to be conscious during Phase G.
"""
from __future__ import annotations
import asyncio
import threading
from pathlib import Path
from unittest.mock import patch
import pytest
from core.lidarr_download_client import LidarrDownloadClient
def _run_async(coro):
loop = asyncio.new_event_loop()
try:
return loop.run_until_complete(coro)
finally:
loop.close()
@pytest.fixture
def lidarr_client():
client = LidarrDownloadClient.__new__(LidarrDownloadClient)
client.download_path = Path('./test_lidarr_downloads')
client.shutdown_check = None
client.active_downloads = {}
client._download_lock = threading.Lock()
client._url = 'http://lidarr.test'
client._api_key = 'test-key'
return client
def test_download_returns_none_when_not_configured():
"""Pinning: no Lidarr URL/key → None. Orchestrator hybrid skip
behavior depends on this."""
client = LidarrDownloadClient.__new__(LidarrDownloadClient)
client._url = ''
client._api_key = ''
result = _run_async(client.download('lidarr', '12345||Album Name', 0))
assert result is None
def test_download_returns_uuid_for_valid_filename(lidarr_client):
"""Pinning: valid filename → UUID download_id."""
with patch('core.lidarr_download_client.threading.Thread') as fake:
fake.return_value.start = lambda: None
result = _run_async(lidarr_client.download(
'lidarr', '12345||Some Album', 0,
))
assert result is not None
assert len(result) == 36
def test_download_parses_album_foreign_id_from_filename(lidarr_client):
"""Pinning: filename format is ``album_foreign_id||display`` where
`album_foreign_id` is the MusicBrainz album MBID Lidarr lookups
use. Engine refactor's plugin contract must respect that this
is an ALBUM identifier, not a track."""
with patch('core.lidarr_download_client.threading.Thread') as fake:
fake.return_value.start = lambda: None
download_id = _run_async(lidarr_client.download(
'lidarr', 'mbid-album-123||Some Album by Artist', 0,
))
record = lidarr_client.active_downloads[download_id]
assert record['album_foreign_id'] == 'mbid-album-123'
assert record['display_name'] == 'Some Album by Artist'
def test_download_handles_filename_without_separator(lidarr_client):
"""Pinning: defensive — filename without `||` still produces a
download record (album_foreign_id stays empty, display_name is
the whole filename). Lidarr's worker tries lookup-by-display."""
with patch('core.lidarr_download_client.threading.Thread') as fake:
fake.return_value.start = lambda: None
download_id = _run_async(lidarr_client.download(
'lidarr', 'just-some-display-name', 0,
))
assert download_id is not None
record = lidarr_client.active_downloads[download_id]
assert record['album_foreign_id'] == ''
assert record['display_name'] == 'just-some-display-name'
def test_download_populates_active_downloads_with_album_oriented_state(lidarr_client):
"""Pinning: Lidarr's state-dict is SMALLER than streaming sources
(no track_id, no transferred/speed/time_remaining Lidarr
polls Lidarr's queue API for those, doesn't track byte-level
progress locally). Engine extraction must accommodate the
smaller schema."""
with patch('core.lidarr_download_client.threading.Thread') as fake:
fake.return_value.start = lambda: None
download_id = _run_async(lidarr_client.download(
'lidarr', 'mbid-1||Album', 0,
))
record = lidarr_client.active_downloads[download_id]
assert record['id'] == download_id
assert record['username'] == 'lidarr'
assert record['state'] == 'Initializing'
assert record['progress'] == 0.0
assert record['album_foreign_id'] == 'mbid-1'
assert record['file_path'] is None
def test_download_spawns_daemon_thread_targeting_worker(lidarr_client):
"""Pinning: thread target is `_download_thread_worker(download_id,
album_foreign_id, display_name)` 3 args, not 4 like streaming
sources. Lidarr doesn't need original_filename because the album
foreign id IS the unique key."""
captured_kwargs = {}
def capture_thread(*args, **kwargs):
captured_kwargs.update(kwargs)
return type('FakeThread', (), {'start': lambda self: None})()
with patch('core.lidarr_download_client.threading.Thread', side_effect=capture_thread):
_run_async(lidarr_client.download('lidarr', 'mbid-x||Album', 0))
assert captured_kwargs.get('daemon') is True
assert captured_kwargs.get('target') == lidarr_client._download_thread_worker
args = captured_kwargs.get('args', ())
assert len(args) == 3 # 3-arg signature
assert args[1] == 'mbid-x'
assert args[2] == 'Album'

View file

@ -0,0 +1,121 @@
"""Phase A pinning tests for QobuzClient — UPDATED for Phase C4.
Post-C4 the client uses engine.worker for thread + state management.
"""
from __future__ import annotations
import asyncio
import threading
from pathlib import Path
from unittest.mock import patch
import pytest
from core.download_engine import DownloadEngine
from core.qobuz_client import QobuzClient
def _run_async(coro):
loop = asyncio.new_event_loop()
try:
return loop.run_until_complete(coro)
finally:
loop.close()
@pytest.fixture
def qobuz_client_with_engine():
client = QobuzClient.__new__(QobuzClient)
client.download_path = Path('./test_qobuz_downloads')
client.shutdown_check = None
client._engine = None
engine = DownloadEngine()
client.set_engine(engine)
return client, engine
def test_download_returns_none_for_invalid_filename_format(qobuz_client_with_engine):
client, _ = qobuz_client_with_engine
result = _run_async(client.download('qobuz', 'no-separator', 0))
assert result is None
def test_download_returns_none_for_non_integer_track_id(qobuz_client_with_engine):
client, _ = qobuz_client_with_engine
result = _run_async(client.download('qobuz', 'not-int||title', 0))
assert result is None
def test_download_raises_when_engine_not_wired():
"""Defensive: client without engine reference must raise so the
orchestrator's download_with_fallback surfaces the error and
moves on to the next source. Returning None silently would drop
the download with no user feedback (per JohnBaumb)."""
import pytest
client = QobuzClient.__new__(QobuzClient)
client._engine = None
with pytest.raises(RuntimeError, match="engine reference"):
_run_async(client.download('qobuz', 'v||t', 0))
def test_download_returns_uuid_for_valid_filename(qobuz_client_with_engine):
client, _ = qobuz_client_with_engine
with patch.object(client, '_download_sync', return_value='/tmp/x.flac'):
result = _run_async(client.download('qobuz', '12345||Some Song', 0))
assert result is not None
assert len(result) == 36
def test_download_populates_engine_record_with_initial_state(qobuz_client_with_engine):
client, engine = qobuz_client_with_engine
started = threading.Event()
release = threading.Event()
def slow_impl(*args, **kwargs):
started.set()
release.wait(timeout=1.0)
return '/tmp/done.flac'
with patch.object(client, '_download_sync', side_effect=slow_impl):
download_id = _run_async(client.download('qobuz', '999||My Qobuz Song', 0))
started.wait(timeout=1.0)
record = engine.get_record('qobuz', download_id)
assert record is not None
assert record['filename'] == '999||My Qobuz Song'
assert record['username'] == 'qobuz'
assert record['track_id'] == 999
assert record['display_name'] == 'My Qobuz Song'
release.set()
def test_get_all_downloads_reads_engine_records(qobuz_client_with_engine):
client, engine = qobuz_client_with_engine
engine.add_record('qobuz', 'dl-1', {
'id': 'dl-1', 'filename': '111||A', 'username': 'qobuz',
'state': 'InProgress, Downloading', 'progress': 50.0,
})
result = _run_async(client.get_all_downloads())
assert len(result) == 1
assert result[0].id == 'dl-1'
def test_cancel_download_marks_cancelled(qobuz_client_with_engine):
client, engine = qobuz_client_with_engine
engine.add_record('qobuz', 'dl-1', {'id': 'dl-1', 'state': 'InProgress, Downloading'})
ok = _run_async(client.cancel_download('dl-1', None, remove=False))
assert ok is True
assert engine.get_record('qobuz', 'dl-1')['state'] == 'Cancelled'
def test_clear_all_completed_drops_only_terminal_records(qobuz_client_with_engine):
client, engine = qobuz_client_with_engine
engine.add_record('qobuz', 'done', {'id': 'done', 'state': 'Completed, Succeeded'})
engine.add_record('qobuz', 'live', {'id': 'live', 'state': 'InProgress, Downloading'})
_run_async(client.clear_all_completed_downloads())
assert engine.get_record('qobuz', 'done') is None
assert engine.get_record('qobuz', 'live') is not None

View file

@ -0,0 +1,105 @@
"""Tests for the per-source RateLimitPolicy declaration mechanism (Phase E1)."""
from __future__ import annotations
from core.download_engine import DownloadEngine, RateLimitPolicy
from core.download_engine.rate_limit import DEFAULT_POLICY, resolve_policy
# ---------------------------------------------------------------------------
# resolve_policy
# ---------------------------------------------------------------------------
def test_resolve_policy_returns_default_when_plugin_declares_nothing():
plugin = object()
assert resolve_policy(plugin) is DEFAULT_POLICY
def test_resolve_policy_reads_class_attribute():
class _Plugin:
RATE_LIMIT_POLICY = RateLimitPolicy(download_delay_seconds=5.0)
policy = resolve_policy(_Plugin())
assert policy.download_delay_seconds == 5.0
def test_resolve_policy_prefers_method_over_class_attribute():
class _Plugin:
RATE_LIMIT_POLICY = RateLimitPolicy(download_delay_seconds=1.0)
def rate_limit_policy(self):
return RateLimitPolicy(download_delay_seconds=10.0)
assert resolve_policy(_Plugin()).download_delay_seconds == 10.0
def test_resolve_policy_falls_back_to_default_when_method_returns_garbage():
class _Plugin:
def rate_limit_policy(self):
return "not a policy object"
assert resolve_policy(_Plugin()) is DEFAULT_POLICY
def test_resolve_policy_falls_back_to_default_when_method_raises():
class _Plugin:
def rate_limit_policy(self):
raise RuntimeError("boom")
assert resolve_policy(_Plugin()) is DEFAULT_POLICY
# ---------------------------------------------------------------------------
# Engine applies policy on register
# ---------------------------------------------------------------------------
def test_engine_applies_declared_policy_on_register():
"""Pinning: when a plugin is registered, its declared
RateLimitPolicy is pushed into the worker's per-source semaphore +
delay registry. Future dispatches use those values."""
class _ThrottledPlugin:
RATE_LIMIT_POLICY = RateLimitPolicy(download_concurrency=1, download_delay_seconds=2.5)
engine = DownloadEngine()
engine.register_plugin('throttled', _ThrottledPlugin())
assert engine.worker._get_delay('throttled') == 2.5
def test_engine_applies_default_policy_when_plugin_declares_nothing():
"""Plugins without a declaration get the conservative default
(concurrency=1, delay=0)."""
class _DefaultPlugin:
pass
engine = DownloadEngine()
engine.register_plugin('default', _DefaultPlugin())
assert engine.worker._get_delay('default') == 0.0
def test_set_engine_callback_runs_after_policy_applied():
"""Pinning: set_engine fires AFTER policy registration, so
config-driven sources can override their declared policy.
YouTube uses this set_engine reads the user-tunable
youtube.download_delay config and overrides the declared default."""
fired_at: list = []
class _Plugin:
RATE_LIMIT_POLICY = RateLimitPolicy(download_delay_seconds=1.0)
def set_engine(self, engine):
# Capture worker state at the moment set_engine fires.
fired_at.append(engine.worker._get_delay('flexible'))
# Then override.
engine.worker.set_delay('flexible', 99.0)
engine = DownloadEngine()
engine.register_plugin('flexible', _Plugin())
# The class-attribute value was applied first.
assert fired_at == [1.0]
# Then set_engine overrode it.
assert engine.worker._get_delay('flexible') == 99.0

View file

@ -0,0 +1,258 @@
"""Phase A pinning tests for SoulseekClient's download lifecycle.
These tests pin the OBSERVABLE BEHAVIOR of `SoulseekClient.download` /
`get_all_downloads` / `cancel_download` so the upcoming download
engine refactor (which lifts shared state + thread workers + search
retry into a central engine) can't drift the per-source contract.
The contract these tests pin is what the engine will call into via
`plugin.download_raw(target_id)` / `plugin.cancel_raw(target_id)`
after the refactor lands. If a future commit breaks any of these
expectations, the diff fails fast long before a real download
attempt against a live slskd would have surfaced the bug.
NOTE: Soulseek is structurally different from the streaming sources.
It has NO local thread worker slskd manages downloads server-side
and the client just polls for state. So Soulseek skips most of the
engine refactor's thread-extraction work; what stays critical is
the slskd HTTP API contract (endpoints, payload shape, id
extraction). That's what these tests pin.
"""
from __future__ import annotations
import asyncio
from pathlib import Path
from unittest.mock import AsyncMock, patch
import pytest
from core.soulseek_client import SoulseekClient
def _run_async(coro):
loop = asyncio.new_event_loop()
try:
return loop.run_until_complete(coro)
finally:
loop.close()
# ---------------------------------------------------------------------------
# Configuration / lifecycle
# ---------------------------------------------------------------------------
def test_is_configured_returns_false_when_no_base_url():
"""Pinning: an unconfigured client (no slskd URL set) reports
is_configured() == False. The orchestrator's hybrid fallback +
every consumer that gates on is_configured() depends on this."""
client = SoulseekClient.__new__(SoulseekClient)
client.base_url = None
client.api_key = None
assert client.is_configured() is False
def test_is_configured_returns_true_when_base_url_set():
"""Pinning: configured client (slskd URL present) reports True."""
client = SoulseekClient.__new__(SoulseekClient)
client.base_url = 'http://localhost:5030'
client.api_key = 'test-key'
assert client.is_configured() is True
# ---------------------------------------------------------------------------
# download()
# ---------------------------------------------------------------------------
@pytest.fixture
def configured_client():
"""A SoulseekClient with the slskd URL set but no real network. Tests
individually patch `_make_request` to return whatever shape they
want to exercise."""
client = SoulseekClient.__new__(SoulseekClient)
client.base_url = 'http://localhost:5030'
client.api_key = 'test-key'
client.download_path = Path('./test_downloads')
return client
def test_download_returns_none_when_not_configured():
"""Pinning: an unconfigured client refuses downloads — returns
None silently rather than raising. Used as the soft-fail signal
by the orchestrator's per-source fallback chain."""
client = SoulseekClient.__new__(SoulseekClient)
client.base_url = None
result = _run_async(client.download('user', 'song.flac', 1024))
assert result is None
def test_download_hits_transfers_downloads_username_endpoint(configured_client):
"""Pinning: the primary download endpoint is
`transfers/downloads/<username>` POST. This shape was chosen to
match slskd's web-interface API exactly. Changing it breaks
every download against current slskd builds."""
captured = []
async def fake_request(method, endpoint, json=None, **kwargs):
captured.append((method, endpoint, json))
return {'id': 'dl-id-from-slskd'}
with patch.object(configured_client, '_make_request', side_effect=fake_request):
result = _run_async(configured_client.download('user', 'song.flac', 1024))
assert result == 'dl-id-from-slskd'
method, endpoint, payload = captured[0]
assert method == 'POST'
assert endpoint == 'transfers/downloads/user'
# Payload is the slskd web-interface array format.
assert isinstance(payload, list)
assert payload[0]['filename'] == 'song.flac'
assert payload[0]['size'] == 1024
def test_download_extracts_id_from_dict_response(configured_client):
"""Pinning: when slskd returns `{id: ...}`, that's the
download_id the orchestrator uses to track the download."""
with patch.object(configured_client, '_make_request',
AsyncMock(return_value={'id': 'abc123'})):
result = _run_async(configured_client.download('user', 'song.flac', 1024))
assert result == 'abc123'
def test_download_extracts_id_from_list_response(configured_client):
"""Pinning: slskd sometimes returns a list of file objects.
The first item's id is the download_id."""
with patch.object(configured_client, '_make_request',
AsyncMock(return_value=[{'id': 'list-id'}, {'id': 'second'}])):
result = _run_async(configured_client.download('user', 'song.flac', 1024))
assert result == 'list-id'
def test_download_falls_back_to_filename_when_no_id_in_response(configured_client):
"""Pinning: defensive — older slskd builds returned 201 Created
with no id field. The client uses the filename as the download
identifier in that case so downstream tracking still works."""
with patch.object(configured_client, '_make_request',
AsyncMock(return_value={'status': 'queued'})):
result = _run_async(configured_client.download('user', 'song.flac', 1024))
assert result == 'song.flac'
# ---------------------------------------------------------------------------
# get_all_downloads()
# ---------------------------------------------------------------------------
def test_get_all_downloads_returns_empty_when_not_configured():
client = SoulseekClient.__new__(SoulseekClient)
client.base_url = None
result = _run_async(client.get_all_downloads())
assert result == []
def test_get_all_downloads_parses_nested_user_directory_files_response(configured_client):
"""Pinning: slskd's `transfers/downloads` returns
`[{username, directories: [{files: [...]}]}]`. The client
flattens that into a list of DownloadStatus objects, one per
file. Engine refactor's state aggregation depends on this shape."""
fake_response = [
{
'username': 'peer1',
'directories': [{
'files': [
{'id': 'f1', 'filename': 'a.flac', 'state': 'InProgress',
'size': 100, 'bytesTransferred': 50, 'averageSpeed': 1024},
{'id': 'f2', 'filename': 'b.flac', 'state': 'Completed, Succeeded',
'size': 200, 'bytesTransferred': 200, 'averageSpeed': 2048},
],
}],
},
]
with patch.object(configured_client, '_make_request',
AsyncMock(return_value=fake_response)):
result = _run_async(configured_client.get_all_downloads())
assert len(result) == 2
assert result[0].id == 'f1'
assert result[0].username == 'peer1'
assert result[0].state == 'InProgress'
assert result[1].id == 'f2'
# Pinning: 'Completed' state forces progress=100 regardless of source data.
assert result[1].progress == 100.0
def test_get_all_downloads_endpoint_is_transfers_downloads(configured_client):
"""Pinning: the listing endpoint is `transfers/downloads` (no
username). The 404'd `users/.../downloads` variant was tried
once and removed keep it gone."""
captured = []
async def fake_request(method, endpoint, **kwargs):
captured.append((method, endpoint))
return []
with patch.object(configured_client, '_make_request', side_effect=fake_request):
_run_async(configured_client.get_all_downloads())
assert captured == [('GET', 'transfers/downloads')]
# ---------------------------------------------------------------------------
# cancel_download()
# ---------------------------------------------------------------------------
def test_cancel_download_returns_false_when_not_configured():
client = SoulseekClient.__new__(SoulseekClient)
client.base_url = None
result = _run_async(client.cancel_download('dl-id', 'user', remove=False))
assert result is False
def test_cancel_download_looks_up_username_when_not_provided(configured_client):
"""Pinning: orchestrator may call cancel_download without a
username hint. The client falls back to scanning all downloads
to find which peer owns it. Engine refactor must preserve this
so existing API endpoints that don't pass username keep working."""
fake_listing = [
{
'username': 'peer-owner',
'directories': [{
'files': [{'id': 'target-dl', 'filename': 'x.flac',
'state': 'InProgress', 'size': 0,
'bytesTransferred': 0, 'averageSpeed': 0}],
}],
},
]
captured_endpoints = []
async def fake_request(method, endpoint, **kwargs):
captured_endpoints.append((method, endpoint))
if method == 'GET' and endpoint == 'transfers/downloads':
return fake_listing
# The DELETE for cancel — return success
return True
with patch.object(configured_client, '_make_request', side_effect=fake_request):
_run_async(configured_client.cancel_download('target-dl', None, remove=False))
# The lookup hit get_all_downloads first, then the DELETE used the discovered username.
assert ('GET', 'transfers/downloads') in captured_endpoints
delete_calls = [(m, e) for m, e in captured_endpoints if m == 'DELETE']
assert delete_calls, "Expected at least one DELETE after username lookup"
# The cancel endpoint URL contains the discovered username.
assert any('peer-owner' in e for _, e in delete_calls)
def test_cancel_download_returns_false_when_username_lookup_fails(configured_client):
"""Pinning: if the download_id isn't in the active list, return
False rather than raising. Orchestrator treats False as "couldn't
cancel" and continues; an exception would propagate to the user."""
with patch.object(configured_client, '_make_request',
AsyncMock(return_value=[])):
result = _run_async(configured_client.cancel_download('missing-id', None))
assert result is False

View file

@ -0,0 +1,131 @@
"""Phase A pinning tests for SoundcloudClient — UPDATED for Phase C7.
SoundCloud's quirk: 3-part filename `track_id||permalink_url||display_name`
because yt-dlp consumes the URL, not the track_id, to actually download.
The engine record holds both fields so the worker can call
`_download_sync(download_id, permalink_url, display_name)` correctly.
"""
from __future__ import annotations
import asyncio
import threading
from pathlib import Path
from unittest.mock import patch
import pytest
from core.download_engine import DownloadEngine
from core.soundcloud_client import SoundcloudClient
def _run_async(coro):
loop = asyncio.new_event_loop()
try:
return loop.run_until_complete(coro)
finally:
loop.close()
@pytest.fixture
def sc_client_with_engine():
client = SoundcloudClient.__new__(SoundcloudClient)
client.download_path = Path('./test_sc_downloads')
client.shutdown_check = None
client._engine = None
engine = DownloadEngine()
client.set_engine(engine)
return client, engine
def test_download_returns_none_for_filename_with_too_few_parts(sc_client_with_engine):
client, _ = sc_client_with_engine
result = _run_async(client.download('soundcloud', 'just-id-no-url', 0))
assert result is None
def test_download_returns_none_for_empty_track_id_or_url(sc_client_with_engine):
client, _ = sc_client_with_engine
assert _run_async(client.download('soundcloud', '||https://x.com/y', 0)) is None
assert _run_async(client.download('soundcloud', 'track123||', 0)) is None
def test_download_raises_when_engine_not_wired():
"""Defensive: client without engine reference must raise so the
orchestrator's download_with_fallback surfaces the error and
moves on to the next source. Returning None silently would drop
the download with no user feedback (per JohnBaumb)."""
import pytest
client = SoundcloudClient.__new__(SoundcloudClient)
client._engine = None
with pytest.raises(RuntimeError, match="engine reference"):
_run_async(client.download('soundcloud', 'v||t', 0))
def test_download_accepts_three_part_filename_with_display(sc_client_with_engine):
"""Pinning: 3-part filename `track_id||permalink_url||display`
is the canonical form. All three fields go into the engine record."""
client, engine = sc_client_with_engine
started = threading.Event()
release = threading.Event()
def slow_impl(*args, **kwargs):
started.set()
release.wait(timeout=1.0)
return '/tmp/done.mp3'
with patch.object(client, '_download_sync', side_effect=slow_impl):
download_id = _run_async(client.download(
'soundcloud',
'sc-12345||https://soundcloud.com/artist/song||Some Display Title',
0,
))
started.wait(timeout=1.0)
record = engine.get_record('soundcloud', download_id)
assert record['track_id'] == 'sc-12345'
assert record['permalink_url'] == 'https://soundcloud.com/artist/song'
assert record['display_name'] == 'Some Display Title'
release.set()
def test_download_falls_back_display_name_to_track_id_when_two_part(sc_client_with_engine):
"""Pinning: 2-part filename → display name = track_id."""
client, engine = sc_client_with_engine
started = threading.Event()
release = threading.Event()
def slow_impl(*args, **kwargs):
started.set()
release.wait(timeout=1.0)
return '/tmp/x.mp3'
with patch.object(client, '_download_sync', side_effect=slow_impl):
download_id = _run_async(client.download(
'soundcloud', 'sc-99||https://soundcloud.com/x/y', 0,
))
started.wait(timeout=1.0)
assert engine.get_record('soundcloud', download_id)['display_name'] == 'sc-99'
release.set()
def test_download_populates_engine_record_with_initial_state(sc_client_with_engine):
client, engine = sc_client_with_engine
started = threading.Event()
release = threading.Event()
def slow_impl(*args, **kwargs):
started.set()
release.wait(timeout=1.0)
return '/tmp/x.mp3'
with patch.object(client, '_download_sync', side_effect=slow_impl):
download_id = _run_async(client.download(
'soundcloud', 'sc-1||https://soundcloud.com/x||Title', 0,
))
started.wait(timeout=1.0)
record = engine.get_record('soundcloud', download_id)
assert record['username'] == 'soundcloud'
assert record['state'] in ('Initializing', 'InProgress, Downloading')
assert 'permalink_url' in record
release.set()

View file

@ -0,0 +1,172 @@
"""Phase A pinning tests for TidalDownloadClient — UPDATED for Phase C3.
Post-C3 the client no longer owns its own ``active_downloads`` dict
or thread spawn both moved into the engine's BackgroundDownloadWorker.
Pinning tests now read state from ``engine.get_record('tidal', ...)``
instead of ``client.active_downloads[...]``.
"""
from __future__ import annotations
import asyncio
import threading
from pathlib import Path
from unittest.mock import patch
import pytest
from core.download_engine import DownloadEngine
from core.tidal_download_client import TidalDownloadClient
def _run_async(coro):
loop = asyncio.new_event_loop()
try:
return loop.run_until_complete(coro)
finally:
loop.close()
@pytest.fixture
def tidal_client_with_engine():
client = TidalDownloadClient.__new__(TidalDownloadClient)
client.download_path = Path('./test_tidal_downloads')
client.shutdown_check = None
client.session = None
client._device_auth_future = None
client._device_auth_link = None
client._engine = None
engine = DownloadEngine()
client.set_engine(engine)
return client, engine
# ---------------------------------------------------------------------------
# is_configured / is_authenticated
# ---------------------------------------------------------------------------
def test_is_authenticated_false_when_no_session(tidal_client_with_engine):
client, _ = tidal_client_with_engine
assert client.is_authenticated() is False
def test_is_authenticated_false_when_session_check_login_raises(tidal_client_with_engine):
client, _ = tidal_client_with_engine
fake_session = type('FakeSession', (), {
'check_login': lambda self: (_ for _ in ()).throw(RuntimeError("expired")),
})()
client.session = fake_session
assert client.is_authenticated() is False
# ---------------------------------------------------------------------------
# download() — filename parsing + id contract
# ---------------------------------------------------------------------------
def test_download_returns_none_for_invalid_filename_format(tidal_client_with_engine):
client, _ = tidal_client_with_engine
result = _run_async(client.download('tidal', 'no-separator', 0))
assert result is None
def test_download_returns_none_for_non_integer_track_id(tidal_client_with_engine):
client, _ = tidal_client_with_engine
result = _run_async(client.download('tidal', 'not-int||title', 0))
assert result is None
def test_download_raises_when_engine_not_wired():
"""Defensive: client without engine reference must raise so the
orchestrator's download_with_fallback surfaces the error and
moves on to the next source. Returning None silently would drop
the download with no user feedback (per JohnBaumb)."""
import pytest
client = TidalDownloadClient.__new__(TidalDownloadClient)
client._engine = None
with pytest.raises(RuntimeError, match="engine reference"):
_run_async(client.download('tidal', 'v||t', 0))
def test_download_returns_uuid_for_valid_filename(tidal_client_with_engine):
client, _ = tidal_client_with_engine
with patch.object(client, '_download_sync', return_value='/tmp/x.flac'):
result = _run_async(client.download('tidal', '12345||Some Song', 0))
assert result is not None
assert len(result) == 36
def test_download_populates_engine_record_with_initial_state(tidal_client_with_engine):
client, engine = tidal_client_with_engine
started = threading.Event()
release = threading.Event()
def slow_impl(*args, **kwargs):
started.set()
release.wait(timeout=1.0)
return '/tmp/done.flac'
with patch.object(client, '_download_sync', side_effect=slow_impl):
download_id = _run_async(client.download('tidal', '999||My Tidal Song', 0))
started.wait(timeout=1.0)
record = engine.get_record('tidal', download_id)
assert record is not None
assert record['id'] == download_id
assert record['filename'] == '999||My Tidal Song'
assert record['username'] == 'tidal'
assert record['state'] in ('Initializing', 'InProgress, Downloading')
assert record['progress'] == 0.0
assert record['track_id'] == 999 # parsed as int
assert record['display_name'] == 'My Tidal Song'
assert record['file_path'] is None
release.set()
# ---------------------------------------------------------------------------
# Query / cancel — engine-backed reads
# ---------------------------------------------------------------------------
def test_get_all_downloads_reads_engine_records(tidal_client_with_engine):
client, engine = tidal_client_with_engine
engine.add_record('tidal', 'dl-1', {
'id': 'dl-1', 'filename': '111||Song A', 'username': 'tidal',
'state': 'InProgress, Downloading', 'progress': 50.0,
'size': 1000, 'transferred': 500, 'speed': 100,
})
engine.add_record('tidal', 'dl-2', {
'id': 'dl-2', 'filename': '222||Song B', 'username': 'tidal',
'state': 'Completed, Succeeded', 'progress': 100.0,
'size': 2000, 'transferred': 2000, 'speed': 0,
})
result = _run_async(client.get_all_downloads())
assert len(result) == 2
assert {r.id for r in result} == {'dl-1', 'dl-2'}
assert {r.username for r in result} == {'tidal'}
def test_cancel_download_marks_cancelled(tidal_client_with_engine):
client, engine = tidal_client_with_engine
engine.add_record('tidal', 'dl-1', {'id': 'dl-1', 'state': 'InProgress, Downloading'})
ok = _run_async(client.cancel_download('dl-1', None, remove=False))
assert ok is True
assert engine.get_record('tidal', 'dl-1')['state'] == 'Cancelled'
ok = _run_async(client.cancel_download('dl-1', None, remove=True))
assert ok is True
assert engine.get_record('tidal', 'dl-1') is None
def test_clear_all_completed_drops_only_terminal_records(tidal_client_with_engine):
client, engine = tidal_client_with_engine
engine.add_record('tidal', 'done', {'id': 'done', 'state': 'Completed, Succeeded'})
engine.add_record('tidal', 'live', {'id': 'live', 'state': 'InProgress, Downloading'})
_run_async(client.clear_all_completed_downloads())
assert engine.get_record('tidal', 'done') is None
assert engine.get_record('tidal', 'live') is not None

View file

@ -0,0 +1,212 @@
"""Phase A pinning tests for YouTubeClient — UPDATED for Phase C2.
Post-C2 the client no longer owns its own ``active_downloads`` dict
or thread spawn both moved into the engine's BackgroundDownloadWorker.
These tests still pin the same OBSERVABLE CONTRACT (filename
encoding, UUID download_id, initial-record schema, source-specific
extras like video_id/url/title) but read state from
``engine.get_record(...)`` instead of ``client.active_downloads[...]``.
What pre-C2 pinning tests caught and what these still catch:
- Filename format: `video_id||title`
- Invalid filename None
- UUID download_id format
- Per-download record schema (id, filename, username, state,
progress, video_id, url, title, file_path)
- Source name in record's username slot is `'youtube'` ✓
What dropped (covered by other tests):
- Direct thread-spawn assertion (engine.worker has its own tests).
- `_download_thread_worker` target (gone from client; engine owns
it).
"""
from __future__ import annotations
import asyncio
import threading
from pathlib import Path
from unittest.mock import patch
import pytest
from core.download_engine import DownloadEngine
from core.youtube_client import YouTubeClient
def _run_async(coro):
loop = asyncio.new_event_loop()
try:
return loop.run_until_complete(coro)
finally:
loop.close()
@pytest.fixture
def yt_client_with_engine():
"""A bare YouTubeClient wired into a real engine. The engine
callback is invoked manually since we bypass orchestrator init."""
client = YouTubeClient.__new__(YouTubeClient)
client.download_path = Path('./test_yt_downloads')
client.shutdown_check = None
client.matching_engine = None
client._download_delay = 3
client.current_download_id = None
client.current_download_progress = {
'status': 'idle', 'percent': 0.0, 'downloaded_bytes': 0,
'total_bytes': 0, 'speed': 0, 'eta': 0, 'filename': '',
}
client.progress_callback = None
client.download_opts = {}
client._engine = None
engine = DownloadEngine()
client.set_engine(engine)
return client, engine
# ---------------------------------------------------------------------------
# download() — filename parsing + id contract
# ---------------------------------------------------------------------------
def test_download_returns_none_for_invalid_filename_format(yt_client_with_engine):
"""Pinning: missing `||` → None (not exception)."""
client, _ = yt_client_with_engine
result = _run_async(client.download('youtube', 'no-separator', 0))
assert result is None
def test_download_raises_when_engine_not_wired():
"""Defensive: client without engine reference must raise so the
orchestrator's download_with_fallback surfaces the error and
moves on to the next source. Returning None silently would drop
the download with no user feedback (per JohnBaumb)."""
import pytest
client = YouTubeClient.__new__(YouTubeClient)
client._engine = None
with pytest.raises(RuntimeError, match="engine reference"):
_run_async(client.download('youtube', 'v||t', 0))
def test_download_returns_uuid_download_id_for_valid_filename(yt_client_with_engine):
"""Pinning: valid `video_id||title` → UUID download_id."""
client, engine = yt_client_with_engine
# Patch _download_sync so the worker thread's impl returns
# without doing real yt-dlp work.
with patch.object(client, '_download_sync', return_value='/tmp/x.mp3'):
result = _run_async(client.download('youtube', 'abc123||Some Song', 0))
assert result is not None
assert len(result) == 36
assert result.count('-') == 4
def test_download_populates_engine_record_with_initial_state(yt_client_with_engine):
"""Pinning: per-download record schema. STATE LOCATION CHANGED
in C2 (now in engine), but the SHAPE of the record is the same
frontend / status APIs / context-key matching depend on these
keys."""
client, engine = yt_client_with_engine
# Hold the impl so we can read 'Initializing' / 'InProgress' state
# before the worker completes.
started = threading.Event()
release = threading.Event()
def slow_impl(*args, **kwargs):
started.set()
release.wait(timeout=1.0)
return '/tmp/done.mp3'
with patch.object(client, '_download_sync', side_effect=slow_impl):
download_id = _run_async(
client.download('youtube', 'video123||My Title', 5000)
)
started.wait(timeout=1.0)
record = engine.get_record('youtube', download_id)
assert record is not None
assert record['id'] == download_id
assert record['filename'] == 'video123||My Title' # ORIGINAL form
assert record['username'] == 'youtube'
assert record['state'] in ('Initializing', 'InProgress, Downloading')
assert record['progress'] == 0.0
assert record['file_path'] is None
# Source-specific extras must merge into the record.
assert record['video_id'] == 'video123'
assert record['url'] == 'https://www.youtube.com/watch?v=video123'
assert record['title'] == 'My Title'
release.set()
def test_set_engine_configures_worker_delay(yt_client_with_engine):
"""Pinning: when engine is wired, the YouTube download_delay
config (3s default) propagates to the worker so successive
downloads serialize with the same gap they did pre-C2."""
client, engine = yt_client_with_engine
# Default delay is 3s.
assert engine.worker._get_delay('youtube') == 3.0
def test_rate_limit_policy_reflects_configured_delay(yt_client_with_engine):
"""Pinning (Phase E): YouTube's rate_limit_policy() returns a
RateLimitPolicy with the configured download_delay (3s default
from `youtube.download_delay` config). Engine reads this at
register_plugin time."""
client, _ = yt_client_with_engine
policy = client.rate_limit_policy()
assert policy.download_delay_seconds == 3.0
assert policy.download_concurrency == 1
# ---------------------------------------------------------------------------
# Query / cancel — engine-backed reads
# ---------------------------------------------------------------------------
def test_get_all_downloads_reads_engine_records(yt_client_with_engine):
client, engine = yt_client_with_engine
# Seed engine with a fake record to mirror what dispatch would do.
engine.add_record('youtube', 'dl-1', {
'id': 'dl-1', 'filename': 'v||t', 'username': 'youtube',
'state': 'InProgress, Downloading', 'progress': 50.0,
'size': 1000, 'transferred': 500, 'speed': 100,
})
result = _run_async(client.get_all_downloads())
assert len(result) == 1
assert result[0].id == 'dl-1'
assert result[0].state == 'InProgress, Downloading'
def test_cancel_download_marks_cancelled_and_optionally_removes(yt_client_with_engine):
client, engine = yt_client_with_engine
engine.add_record('youtube', 'dl-1', {
'id': 'dl-1', 'filename': 'v||t', 'username': 'youtube',
'state': 'InProgress, Downloading', 'progress': 50.0,
})
ok = _run_async(client.cancel_download('dl-1', None, remove=False))
assert ok is True
assert engine.get_record('youtube', 'dl-1')['state'] == 'Cancelled'
ok = _run_async(client.cancel_download('dl-1', None, remove=True))
assert ok is True
assert engine.get_record('youtube', 'dl-1') is None
def test_clear_all_completed_drops_only_terminal_records(yt_client_with_engine):
client, engine = yt_client_with_engine
engine.add_record('youtube', 'done', {'id': 'done', 'state': 'Completed, Succeeded'})
engine.add_record('youtube', 'erred', {'id': 'erred', 'state': 'Errored'})
engine.add_record('youtube', 'live', {'id': 'live', 'state': 'InProgress, Downloading'})
_run_async(client.clear_all_completed_downloads())
assert engine.get_record('youtube', 'done') is None
assert engine.get_record('youtube', 'erred') is None
assert engine.get_record('youtube', 'live') is not None

View file

@ -129,7 +129,7 @@ def _build_deps(**overrides):
spotify_client=None,
hydrabase_client=None,
hydrabase_worker=None,
soulseek_client=None,
download_orchestrator=None,
fix_artist_image_url=lambda u: f'FIXED::{u}' if u else None,
is_hydrabase_active=lambda: False,
get_metadata_fallback_source=lambda: 'spotify',
@ -462,24 +462,27 @@ class _FakeYouTube:
class _FakeSoulseekWithYT:
def __init__(self, youtube):
self.youtube = youtube
self._youtube = youtube
def client(self, name):
return self._youtube if name == 'youtube' else None
def test_resolve_youtube_videos_returns_subclient():
yt = _FakeYouTube()
deps = _build_deps(soulseek_client=_FakeSoulseekWithYT(yt))
deps = _build_deps(download_orchestrator=_FakeSoulseekWithYT(yt))
assert orchestrator.resolve_youtube_videos_client(deps) is yt
def test_resolve_youtube_videos_no_soulseek_returns_none():
deps = _build_deps(soulseek_client=None)
deps = _build_deps(download_orchestrator=None)
assert orchestrator.resolve_youtube_videos_client(deps) is None
def test_resolve_youtube_videos_no_youtube_attr_returns_none():
class _NoYT:
pass
deps = _build_deps(soulseek_client=_NoYT())
deps = _build_deps(download_orchestrator=_NoYT())
assert orchestrator.resolve_youtube_videos_client(deps) is None

View file

@ -53,15 +53,20 @@ class _FakeStreamClient:
class _FakeSoulseek:
def __init__(self, youtube=None, tidal=None, qobuz=None, hifi=None, deezer_dl=None, lidarr=None, results_per_query=None):
self.youtube = youtube
self.tidal = tidal
self.qobuz = qobuz
self.hifi = hifi
self.deezer_dl = deezer_dl
self.lidarr = lidarr
self._clients = {
'youtube': youtube,
'tidal': tidal,
'qobuz': qobuz,
'hifi': hifi,
'deezer_dl': deezer_dl,
'lidarr': lidarr,
}
self._results = results_per_query or {}
self.search_calls = []
def client(self, name):
return self._clients.get(name)
async def search(self, query, timeout=15):
self.search_calls.append(query)
return self._results.get(query, ([], []))
@ -164,7 +169,7 @@ def test_stream_finds_match_on_first_query():
result = stream.stream_search_track(
track_name='Money', artist_name='Pink Floyd', album_name=None,
duration_ms=180000,
config_manager=cfg, soulseek_client=soul, matching_engine=engine,
config_manager=cfg, download_orchestrator=soul, matching_engine=engine,
run_async=_run_async,
)
assert result is not None
@ -185,7 +190,7 @@ def test_stream_walks_to_second_query_on_no_match():
result = stream.stream_search_track(
track_name='Money (Remastered)', artist_name='Pink Floyd', album_name=None,
duration_ms=180000,
config_manager=cfg, soulseek_client=soul, matching_engine=engine,
config_manager=cfg, download_orchestrator=soul, matching_engine=engine,
run_async=_run_async,
)
assert result is not None
@ -204,7 +209,7 @@ def test_stream_returns_none_when_no_matches():
result = stream.stream_search_track(
track_name='Money', artist_name='Pink Floyd', album_name=None,
duration_ms=180000,
config_manager=cfg, soulseek_client=soul, matching_engine=engine,
config_manager=cfg, download_orchestrator=soul, matching_engine=engine,
run_async=_run_async,
)
assert result is None
@ -212,7 +217,7 @@ def test_stream_returns_none_when_no_matches():
def test_stream_falls_back_to_default_soulseek_when_no_direct_client():
# Effective mode = 'youtube' (coerced from soulseek), but soul.youtube is None
# → code falls through to soulseek_client.search directly. Streaming-mode
# → code falls through to download_orchestrator.search directly. Streaming-mode
# query gen → "Pink Floyd Money".
soul = _FakeSoulseek(results_per_query={'Pink Floyd Money': ([object()], [])})
cfg = _FakeConfig({
@ -224,7 +229,7 @@ def test_stream_falls_back_to_default_soulseek_when_no_direct_client():
result = stream.stream_search_track(
track_name='Money', artist_name='Pink Floyd', album_name=None,
duration_ms=180000,
config_manager=cfg, soulseek_client=soul, matching_engine=engine,
config_manager=cfg, download_orchestrator=soul, matching_engine=engine,
run_async=_run_async,
)
assert result is not None
@ -252,7 +257,7 @@ def test_stream_continues_past_per_query_exception():
result = stream.stream_search_track(
track_name='Money (Live)', artist_name='Pink Floyd', album_name=None,
duration_ms=180000,
config_manager=cfg, soulseek_client=soul, matching_engine=engine,
config_manager=cfg, download_orchestrator=soul, matching_engine=engine,
run_async=_run_async,
)
# First query raised, second succeeded

View file

@ -10,7 +10,7 @@ from core.streaming import prepare as sp
class _FakeSoulseek:
"""Minimal soulseek_client stub for the stream-prep worker."""
"""Minimal download_orchestrator stub for the stream-prep worker."""
def __init__(self, *, download_id='dl-1', all_downloads=None):
self._download_id = download_id
@ -43,7 +43,7 @@ def _build_deps(
state = state if state is not None else {}
deps = sp.PrepareStreamDeps(
config_manager=type('C', (), {'get': lambda self, k, d=None: d})(),
soulseek_client=soulseek or _FakeSoulseek(),
download_orchestrator=soulseek or _FakeSoulseek(),
stream_lock=threading.Lock(),
project_root=project_root,
docker_resolve_path=lambda p: p,
@ -107,7 +107,7 @@ def test_stream_folder_cleared_before_download(tmp_path):
# ---------------------------------------------------------------------------
def test_download_returns_none_marks_error(tmp_path):
"""soulseek_client.download() returning None → state.error."""
"""download_orchestrator.download() returning None → state.error."""
sk = _FakeSoulseek(download_id=None)
deps = _build_deps(soulseek=sk, project_root=str(tmp_path))

View file

@ -39,18 +39,18 @@ def orchestrator() -> DownloadOrchestrator:
def test_orchestrator_constructs_soundcloud_client(orchestrator: DownloadOrchestrator) -> None:
assert orchestrator.soundcloud is not None
assert isinstance(orchestrator.soundcloud, SoundcloudClient)
assert orchestrator.client('soundcloud') is not None
assert isinstance(orchestrator.client('soundcloud'), SoundcloudClient)
def test_client_lookup_resolves_soundcloud(orchestrator: DownloadOrchestrator) -> None:
"""Verify the dict-based name → client lookup includes SoundCloud."""
assert orchestrator._client('soundcloud') is orchestrator.soundcloud
"""Verify the registry-backed name → client lookup includes SoundCloud."""
assert orchestrator.client('soundcloud') is orchestrator.registry.get('soundcloud')
def test_client_lookup_returns_none_for_unknown(orchestrator: DownloadOrchestrator) -> None:
"""Sanity: unknown sources don't somehow resolve to SoundCloud."""
assert orchestrator._client('made_up') is None
assert orchestrator.client('made_up') is None
# ---------------------------------------------------------------------------
@ -80,7 +80,7 @@ def test_download_routes_soundcloud_username_to_client(orchestrator: DownloadOrc
async def _fake_download(username, filename, file_size=0):
return sentinel
with patch.object(orchestrator.soundcloud, 'download', side_effect=_fake_download) as mock_dl:
with patch.object(orchestrator.client('soundcloud'), 'download', side_effect=_fake_download) as mock_dl:
result = _run(orchestrator.download(
'soundcloud',
'999||https://soundcloud.com/x/y||Display',
@ -93,13 +93,13 @@ def test_download_routes_soundcloud_username_to_client(orchestrator: DownloadOrc
def test_download_unknown_username_still_falls_to_soulseek(orchestrator: DownloadOrchestrator) -> None:
"""Adding SoundCloud must not change the legacy Soulseek-fallback
behavior for unrecognized usernames."""
if orchestrator.soulseek is None:
if orchestrator.client('soulseek') is None:
pytest.skip("Soulseek client unavailable in this environment")
async def _fake_soulseek_download(username, filename, file_size=0):
return 'soulseek-id'
with patch.object(orchestrator.soulseek, 'download', side_effect=_fake_soulseek_download) as mock_dl:
with patch.object(orchestrator.client('soulseek'), 'download', side_effect=_fake_soulseek_download) as mock_dl:
result = _run(orchestrator.download('some_random_user', 'file.mp3', 0))
assert result == 'soulseek-id'
mock_dl.assert_called_once()
@ -122,8 +122,8 @@ def test_hybrid_search_iterates_soundcloud_when_in_order(orchestrator: DownloadO
async def _fake_search(query, timeout=None, progress_callback=None):
return ([fake_track], [])
with patch.object(orchestrator.soundcloud, 'search', side_effect=_fake_search), \
patch.object(orchestrator.soundcloud, 'is_configured', return_value=True):
with patch.object(orchestrator.client('soundcloud'), 'search', side_effect=_fake_search), \
patch.object(orchestrator.client('soundcloud'), 'is_configured', return_value=True):
tracks, albums = _run(orchestrator.search("any query"))
assert tracks == [fake_track]
@ -136,7 +136,7 @@ def test_hybrid_search_skips_unconfigured_soundcloud(orchestrator: DownloadOrche
orchestrator.mode = 'hybrid'
orchestrator.hybrid_order = ['soundcloud', 'soulseek']
if orchestrator.soulseek is None:
if orchestrator.client('soulseek') is None:
pytest.skip("Soulseek client unavailable in this environment")
soulseek_track = MagicMock()
@ -145,9 +145,9 @@ def test_hybrid_search_skips_unconfigured_soundcloud(orchestrator: DownloadOrche
async def _fake_soulseek_search(query, timeout=None, progress_callback=None):
return ([soulseek_track], [])
with patch.object(orchestrator.soundcloud, 'is_configured', return_value=False), \
patch.object(orchestrator.soulseek, 'is_configured', return_value=True), \
patch.object(orchestrator.soulseek, 'search', side_effect=_fake_soulseek_search):
with patch.object(orchestrator.client('soundcloud'), 'is_configured', return_value=False), \
patch.object(orchestrator.client('soulseek'), 'is_configured', return_value=True), \
patch.object(orchestrator.client('soulseek'), 'search', side_effect=_fake_soulseek_search):
tracks, _ = _run(orchestrator.search("any"))
assert tracks == [soulseek_track]
@ -166,7 +166,7 @@ def test_get_all_downloads_walks_soundcloud(orchestrator: DownloadOrchestrator)
async def _fake_get_all():
return [fake_status]
with patch.object(orchestrator.soundcloud, 'get_all_downloads', side_effect=_fake_get_all):
with patch.object(orchestrator.client('soundcloud'), 'get_all_downloads', side_effect=_fake_get_all):
all_dl = _run(orchestrator.get_all_downloads())
assert any(d is fake_status for d in all_dl)
@ -180,7 +180,7 @@ def test_get_download_status_finds_soundcloud_id(orchestrator: DownloadOrchestra
async def _fake_get_status(download_id):
return fake_status if download_id == 'sc-2' else None
with patch.object(orchestrator.soundcloud, 'get_download_status', side_effect=_fake_get_status):
with patch.object(orchestrator.client('soundcloud'), 'get_download_status', side_effect=_fake_get_status):
result = _run(orchestrator.get_download_status('sc-2'))
assert result is fake_status
@ -192,7 +192,7 @@ def test_cancel_routes_soundcloud_username(orchestrator: DownloadOrchestrator) -
async def _fake_cancel(download_id, username=None, remove=False):
return True
with patch.object(orchestrator.soundcloud, 'cancel_download', side_effect=_fake_cancel) as mock_cancel:
with patch.object(orchestrator.client('soundcloud'), 'cancel_download', side_effect=_fake_cancel) as mock_cancel:
ok = _run(orchestrator.cancel_download('sc-3', username='soundcloud'))
assert ok is True
mock_cancel.assert_called_once()
@ -210,7 +210,7 @@ def test_clear_completed_walks_soundcloud(orchestrator: DownloadOrchestrator) ->
async def _fake_clear():
return True
with patch.object(orchestrator.soundcloud, 'clear_all_completed_downloads', side_effect=_fake_clear) as mock_clear:
with patch.object(orchestrator.client('soundcloud'), 'clear_all_completed_downloads', side_effect=_fake_clear) as mock_clear:
_run(orchestrator.clear_all_completed_downloads())
mock_clear.assert_called_once()
@ -228,8 +228,8 @@ def test_soundcloud_only_mode_uses_soundcloud(orchestrator: DownloadOrchestrator
async def _fake_search(query, timeout=None, progress_callback=None):
return ([MagicMock(username='soundcloud')], [])
with patch.object(orchestrator.soundcloud, 'search', side_effect=_fake_search) as mock_sc, \
patch.object(orchestrator.soulseek, 'search', side_effect=AssertionError("soulseek must not be searched")):
with patch.object(orchestrator.client('soundcloud'), 'search', side_effect=_fake_search) as mock_sc, \
patch.object(orchestrator.client('soulseek'), 'search', side_effect=AssertionError("soulseek must not be searched")):
tracks, _ = _run(orchestrator.search("any"))
assert len(tracks) == 1

View file

@ -0,0 +1,163 @@
"""Pin the structural conformance of every download source plugin
class to ``DownloadSourcePlugin``.
Each registered source class MUST:
- Implement every protocol method by name.
- Mark async methods as `async def` so the orchestrator can `await`
them uniformly.
When someone adds a new source (e.g. Usenet) and forgets one of
these methods, this test fails at the contract long before the
first real download attempt would have raised AttributeError in
production. When someone CHANGES the contract (adds a method to
the protocol), this test forces every existing source to be
updated.
Catches the smell that motivated the refactor in the first place:
8 sources independently grew the same shape because every
consumer site needed the same calls, but nothing enforced parity.
NOTE on test design: these tests check CLASSES, not instances.
Instantiating real client classes (TidalDownloadClient, etc.) at
fixture setup pollutes module-level state in tidalapi / spotipy
imports and breaks downstream tests that rely on a clean import
graph. Class-level checks are equally strict for structural
conformance the protocol only constrains the method surface, not
runtime instance behavior.
"""
from __future__ import annotations
import inspect
import pytest
REQUIRED_SYNC_METHODS = {'is_configured'}
REQUIRED_ASYNC_METHODS = {
'check_connection',
'search',
'download',
'get_all_downloads',
'get_download_status',
'cancel_download',
'clear_all_completed_downloads',
}
def _import_plugin_classes():
"""Import every download source class lazily inside the test
rather than at module load avoids dragging tidalapi /
spotipy / yt-dlp imports into every other test module's
collection phase."""
from core.soulseek_client import SoulseekClient
from core.youtube_client import YouTubeClient
from core.tidal_download_client import TidalDownloadClient
from core.qobuz_client import QobuzClient
from core.hifi_client import HiFiClient
from core.deezer_download_client import DeezerDownloadClient
from core.lidarr_download_client import LidarrDownloadClient
from core.soundcloud_client import SoundcloudClient
return {
'soulseek': SoulseekClient,
'youtube': YouTubeClient,
'tidal': TidalDownloadClient,
'qobuz': QobuzClient,
'hifi': HiFiClient,
'deezer': DeezerDownloadClient,
'lidarr': LidarrDownloadClient,
'soundcloud': SoundcloudClient,
}
def test_default_registry_registers_all_eight_sources():
"""Smoke check that the foundation registry knows about every
source the orchestrator historically dispatched to. If someone
drops a registration here, every other test in this module would
silently miss the missing source."""
from core.download_plugins.registry import build_default_registry
registry = build_default_registry()
expected = {
'soulseek', 'youtube', 'tidal', 'qobuz',
'hifi', 'deezer', 'lidarr', 'soundcloud',
}
assert set(registry.names()) == expected
def test_deezer_dl_alias_is_registered_against_deezer_spec():
"""Legacy ``deezer_dl`` source-name string used in config + per-
source dispatch must keep resolving frontend, settings,
download_orchestrator's username dispatch all depend on it."""
from core.download_plugins.registry import build_default_registry
registry = build_default_registry()
spec = registry.get_spec('deezer_dl')
assert spec is not None
assert spec.name == 'deezer'
assert 'deezer_dl' in spec.aliases
@pytest.mark.parametrize('plugin_name', [
'soulseek', 'youtube', 'tidal', 'qobuz',
'hifi', 'deezer', 'lidarr', 'soundcloud',
])
def test_plugin_class_has_all_required_methods(plugin_name):
"""Every registered plugin class exposes every protocol method
by name. Diagnostic-friendly: tells you WHICH method is missing
when a new source is added without all the required methods."""
classes = _import_plugin_classes()
cls = classes[plugin_name]
missing = []
for method_name in REQUIRED_SYNC_METHODS | REQUIRED_ASYNC_METHODS:
if not hasattr(cls, method_name):
missing.append(method_name)
assert not missing, (
f"{plugin_name} ({cls.__name__}) missing methods: {missing}"
)
@pytest.mark.parametrize('plugin_name', [
'soulseek', 'youtube', 'tidal', 'qobuz',
'hifi', 'deezer', 'lidarr', 'soundcloud',
])
def test_plugin_class_async_methods_are_coroutines(plugin_name):
"""Methods declared async in the protocol must be async on every
plugin class. A sync `download()` would silently skip the
orchestrator's `await` and return a coroutine object instead of
a download_id the kind of bug that only surfaces at runtime
against a live user."""
classes = _import_plugin_classes()
cls = classes[plugin_name]
not_async = []
for method_name in REQUIRED_ASYNC_METHODS:
method = getattr(cls, method_name, None)
if method is None:
continue
if not inspect.iscoroutinefunction(method):
not_async.append(method_name)
assert not not_async, (
f"{plugin_name} ({cls.__name__}) declared these methods as "
f"sync but the protocol requires async: {not_async}"
)
def test_orchestrator_uses_registry_for_dispatch():
"""The orchestrator must hold a registry reference and the generic
``client(name)`` accessor must return the same instances the
registry holds. Per-source attribute aliases (``orchestrator.soulseek``
etc.) were removed in favor of ``orchestrator.client('soulseek')``;
the legacy alias name (``deezer_dl``) still resolves to the canonical
deezer plugin via the registry's alias map."""
from core.download_orchestrator import DownloadOrchestrator
orchestrator = DownloadOrchestrator()
assert hasattr(orchestrator, 'registry')
assert orchestrator.client('soulseek') is orchestrator.registry.get('soulseek')
assert orchestrator.client('youtube') is orchestrator.registry.get('youtube')
assert orchestrator.client('deezer_dl') is orchestrator.registry.get('deezer')
assert orchestrator.client('lidarr') is orchestrator.registry.get('lidarr')

View file

@ -6,7 +6,7 @@ Settings showed "Connected: <username> (Active)" but underneath an error
yellow even after a successful login.
Root cause: SoulSync runs two QobuzClient instances side by side one
through ``soulseek_client.qobuz`` for the auth-flow endpoints, and a
through ``download_orchestrator.client('qobuz')`` for the auth-flow endpoints, and a
second owned by the enrichment worker thread for thread safety. Login
only updated the first instance's in-memory state. The dashboard's
"configured" check (and the connection-test step) read the worker

View file

@ -30,7 +30,7 @@ import pytest
from core import soundcloud_client
from core.soundcloud_client import SoundcloudClient, _sanitize_filename
from core.soulseek_client import AlbumResult, DownloadStatus, TrackResult
from core.download_plugins.types import AlbumResult, DownloadStatus, TrackResult
# ---------------------------------------------------------------------------
@ -91,6 +91,19 @@ def tmp_dl(tmp_path: Path) -> Path:
return p
def _wire_engine(client: SoundcloudClient) -> 'DownloadEngine':
"""Phase C7: SoundCloud no longer owns its own active_downloads
dict state lives on engine.DownloadEngine. Tests that
construct a bare client must wire an engine so download() can
dispatch and the client's query/cancel methods read from
somewhere. Returns the engine for tests that want to inspect
state directly."""
from core.download_engine import DownloadEngine
engine = DownloadEngine()
client.set_engine(engine)
return engine
def test_is_available_when_yt_dlp_installed(tmp_dl: Path) -> None:
client = SoundcloudClient(download_path=str(tmp_dl))
# In our test env yt-dlp is installed (it's a hard dep)
@ -261,9 +274,10 @@ def test_download_rejects_invalid_filename_format(tmp_dl: Path) -> None:
def test_download_starts_thread_and_returns_id(tmp_dl: Path) -> None:
"""Verify the contract: returns a download_id, populates active_downloads,
spawns a thread that ultimately drives state to terminal."""
"""Verify the contract: returns a download_id, engine record is
populated, the worker drives state to terminal."""
client = SoundcloudClient(download_path=str(tmp_dl))
engine = _wire_engine(client)
completed_path = tmp_dl / "track.mp3"
completed_path.write_bytes(b"x" * (200 * 1024)) # > MIN_AUDIO_SIZE
@ -275,16 +289,14 @@ def test_download_starts_thread_and_returns_id(tmp_dl: Path) -> None:
))
assert download_id is not None
# Thread runs async; wait briefly for terminal state
deadline = time.time() + 2
while time.time() < deadline:
with client._download_lock:
state = client.active_downloads[download_id]['state']
if state == 'Completed, Succeeded':
record = engine.get_record('soundcloud', download_id)
if record and record['state'] == 'Completed, Succeeded':
break
time.sleep(0.05)
info = client.active_downloads[download_id]
info = engine.get_record('soundcloud', download_id)
assert info['state'] == 'Completed, Succeeded'
assert info['progress'] == 100.0
assert info['file_path'] == str(completed_path)
@ -293,6 +305,7 @@ def test_download_starts_thread_and_returns_id(tmp_dl: Path) -> None:
def test_download_thread_marks_failed_when_sync_returns_none(tmp_dl: Path) -> None:
client = SoundcloudClient(download_path=str(tmp_dl))
engine = _wire_engine(client)
with patch.object(client, '_download_sync', return_value=None):
download_id = _run(client.download(
'soundcloud',
@ -300,25 +313,25 @@ def test_download_thread_marks_failed_when_sync_returns_none(tmp_dl: Path) -> No
))
deadline = time.time() + 2
while time.time() < deadline:
with client._download_lock:
state = client.active_downloads[download_id]['state']
if state == 'Errored':
record = engine.get_record('soundcloud', download_id)
if record and record['state'] == 'Errored':
break
time.sleep(0.05)
assert client.active_downloads[download_id]['state'] == 'Errored'
assert engine.get_record('soundcloud', download_id)['state'] == 'Errored'
def test_download_thread_does_not_clobber_cancelled_state(tmp_dl: Path) -> None:
"""If a user cancels mid-download and the sync function then returns
None, the thread should NOT overwrite the explicit Cancelled state
with a generic Errored state."""
None, the worker must NOT overwrite the explicit Cancelled state
with a generic Errored state. The legacy per-client thread had
this guard; engine.worker._mark_terminal preserves it for every
source via a single check."""
client = SoundcloudClient(download_path=str(tmp_dl))
engine = _wire_engine(client)
def _slow_sync(download_id, *_):
# Simulate cancellation racing a None return
time.sleep(0.05)
with client._download_lock:
client.active_downloads[download_id]['state'] = 'Cancelled'
engine.update_record('soundcloud', download_id, {'state': 'Cancelled'})
return None
with patch.object(client, '_download_sync', side_effect=_slow_sync):
@ -326,12 +339,11 @@ def test_download_thread_does_not_clobber_cancelled_state(tmp_dl: Path) -> None:
deadline = time.time() + 2
while time.time() < deadline:
with client._download_lock:
state = client.active_downloads[download_id]['state']
if state == 'Cancelled':
record = engine.get_record('soundcloud', download_id)
if record and record['state'] in ('Cancelled', 'Errored'):
break
time.sleep(0.05)
assert client.active_downloads[download_id]['state'] == 'Cancelled'
assert engine.get_record('soundcloud', download_id)['state'] == 'Cancelled'
# ---------------------------------------------------------------------------
@ -375,15 +387,15 @@ def test_download_sync_writes_file_and_returns_path(tmp_dl: Path, monkeypatch) -
)
monkeypatch.setattr(soundcloud_client, "yt_dlp", fake_yt_dlp)
client = SoundcloudClient(download_path=str(tmp_dl))
engine = _wire_engine(client)
with client._download_lock:
client.active_downloads['dl1'] = {
'id': 'dl1', 'filename': '', 'username': 'soundcloud',
'state': 'Initializing', 'progress': 0.0, 'size': 0,
'transferred': 0, 'speed': 0, 'time_remaining': None,
'track_id': 'abc', 'permalink_url': 'u', 'display_name': 'My Track',
'file_path': None,
}
engine.add_record('soundcloud', 'dl1', {
'id': 'dl1', 'filename': '', 'username': 'soundcloud',
'state': 'Initializing', 'progress': 0.0, 'size': 0,
'transferred': 0, 'speed': 0, 'time_remaining': None,
'track_id': 'abc', 'permalink_url': 'u', 'display_name': 'My Track',
'file_path': None,
})
result = client._download_sync('dl1', 'https://soundcloud.com/x/y', 'My Track')
assert result is not None
@ -414,15 +426,15 @@ def test_download_sync_rejects_too_small_file(tmp_dl: Path, monkeypatch) -> None
)
monkeypatch.setattr(soundcloud_client, "yt_dlp", fake_yt_dlp)
client = SoundcloudClient(download_path=str(tmp_dl))
engine = _wire_engine(client)
with client._download_lock:
client.active_downloads['dl2'] = {
'id': 'dl2', 'filename': '', 'username': 'soundcloud',
'state': 'Initializing', 'progress': 0.0, 'size': 0,
'transferred': 0, 'speed': 0, 'time_remaining': None,
'track_id': 'tiny', 'permalink_url': 'u', 'display_name': 'Tiny',
'file_path': None,
}
engine.add_record('soundcloud', 'dl2', {
'id': 'dl2', 'filename': '', 'username': 'soundcloud',
'state': 'Initializing', 'progress': 0.0, 'size': 0,
'transferred': 0, 'speed': 0, 'time_remaining': None,
'track_id': 'tiny', 'permalink_url': 'u', 'display_name': 'Tiny',
'file_path': None,
})
result = client._download_sync('dl2', 'https://soundcloud.com/x/y', 'Tiny')
assert result is None
# File got cleaned up after rejection
@ -451,13 +463,13 @@ def test_download_sync_handles_yt_dlp_raising(tmp_dl: Path, monkeypatch) -> None
)
monkeypatch.setattr(soundcloud_client, "yt_dlp", fake_yt_dlp)
client = SoundcloudClient(download_path=str(tmp_dl))
engine = _wire_engine(client)
with client._download_lock:
client.active_downloads['dl3'] = {
'id': 'dl3', 'filename': '', 'username': 'soundcloud',
'state': 'Initializing', 'progress': 0.0, 'size': 0,
'transferred': 0, 'speed': 0, 'time_remaining': None,
}
engine.add_record('soundcloud', 'dl3', {
'id': 'dl3', 'filename': '', 'username': 'soundcloud',
'state': 'Initializing', 'progress': 0.0, 'size': 0,
'transferred': 0, 'speed': 0, 'time_remaining': None,
})
assert client._download_sync('dl3', 'https://soundcloud.com/x/y', 'Boom') is None
@ -474,104 +486,96 @@ def test_download_sync_returns_none_when_yt_dlp_unavailable(tmp_dl: Path, monkey
def test_update_download_progress_populates_ledger(tmp_dl: Path) -> None:
client = SoundcloudClient(download_path=str(tmp_dl))
with client._download_lock:
client.active_downloads['p1'] = {
'id': 'p1', 'filename': '', 'username': 'soundcloud',
'state': 'InProgress, Downloading', 'progress': 0.0, 'size': 0,
'transferred': 0, 'speed': 0, 'time_remaining': None,
}
engine = _wire_engine(client)
engine.add_record('soundcloud', 'p1', {
'id': 'p1', 'filename': '', 'username': 'soundcloud',
'state': 'InProgress, Downloading', 'progress': 0.0, 'size': 0,
'transferred': 0, 'speed': 0, 'time_remaining': None,
})
speed_start = time.time() - 1.0 # 1 second ago
client._update_download_progress('p1', downloaded=512_000, total=1_024_000,
speed_start=speed_start)
info = client.active_downloads['p1']
info = engine.get_record('soundcloud', 'p1')
assert info['transferred'] == 512_000
assert info['size'] == 1_024_000
# 50% complete, capped below 100
assert 49.0 <= info['progress'] <= 51.0
# Speed roughly 512KB/s
assert info['speed'] > 0
# Time remaining should be roughly 1 second
assert info['time_remaining'] is not None
assert 0 < info['time_remaining'] < 5
def test_update_download_progress_caps_at_99_9(tmp_dl: Path) -> None:
client = SoundcloudClient(download_path=str(tmp_dl))
with client._download_lock:
client.active_downloads['p2'] = {
'id': 'p2', 'filename': '', 'username': 'soundcloud',
'state': 'InProgress, Downloading', 'progress': 0.0, 'size': 0,
'transferred': 0, 'speed': 0, 'time_remaining': None,
}
engine = _wire_engine(client)
engine.add_record('soundcloud', 'p2', {
'id': 'p2', 'filename': '', 'username': 'soundcloud',
'state': 'InProgress, Downloading', 'progress': 0.0, 'size': 0,
'transferred': 0, 'speed': 0, 'time_remaining': None,
})
client._update_download_progress('p2', downloaded=1_000_000,
total=1_000_000, speed_start=time.time() - 1)
assert client.active_downloads['p2']['progress'] == 99.9
assert engine.get_record('soundcloud', 'p2')['progress'] == 99.9
def test_update_download_progress_silently_skips_unknown_id(tmp_dl: Path) -> None:
"""No-op if the download id isn't tracked — defensive against late hooks."""
client = SoundcloudClient(download_path=str(tmp_dl))
_wire_engine(client)
# Should not raise
client._update_download_progress('does_not_exist', 100, 1000, time.time())
def test_update_download_progress_fragmented_uses_fragment_count(tmp_dl: Path) -> None:
"""HLS-aware progress: fragment 5 of 10 → ~50%, regardless of byte
estimate. Pins the SoundCloud HLS UX fix where byte-based progress
stayed stuck near 0."""
estimate."""
client = SoundcloudClient(download_path=str(tmp_dl))
with client._download_lock:
client.active_downloads['hls1'] = {
'id': 'hls1', 'filename': '', 'username': 'soundcloud',
'state': 'InProgress, Downloading', 'progress': 0.0, 'size': 0,
'transferred': 0, 'speed': 0, 'time_remaining': None,
}
engine = _wire_engine(client)
engine.add_record('soundcloud', 'hls1', {
'id': 'hls1', 'filename': '', 'username': 'soundcloud',
'state': 'InProgress, Downloading', 'progress': 0.0, 'size': 0,
'transferred': 0, 'speed': 0, 'time_remaining': None,
})
client._update_download_progress_fragmented(
'hls1', downloaded=512_000, fragment_index=5, fragment_count=10,
speed_start=time.time() - 1.0,
)
info = client.active_downloads['hls1']
info = engine.get_record('soundcloud', 'hls1')
assert 49.0 <= info['progress'] <= 51.0
# Total size estimate extrapolates from per-fragment average
assert info['size'] == 1_024_000 # 512000 * (10/5)
# Time remaining computed
assert info['time_remaining'] is not None and info['time_remaining'] > 0
def test_update_download_progress_fragmented_caps_at_99_9(tmp_dl: Path) -> None:
"""Final fragment shouldn't push percentage to 100 — outer thread
owns the final flip. Mirrors byte-based behavior."""
"""Final fragment shouldn't push percentage to 100."""
client = SoundcloudClient(download_path=str(tmp_dl))
with client._download_lock:
client.active_downloads['hls2'] = {
'id': 'hls2', 'filename': '', 'username': 'soundcloud',
'state': 'InProgress, Downloading', 'progress': 0.0, 'size': 0,
'transferred': 0, 'speed': 0, 'time_remaining': None,
}
engine = _wire_engine(client)
engine.add_record('soundcloud', 'hls2', {
'id': 'hls2', 'filename': '', 'username': 'soundcloud',
'state': 'InProgress, Downloading', 'progress': 0.0, 'size': 0,
'transferred': 0, 'speed': 0, 'time_remaining': None,
})
client._update_download_progress_fragmented(
'hls2', downloaded=10_000_000, fragment_index=10, fragment_count=10,
speed_start=time.time() - 5.0,
)
assert client.active_downloads['hls2']['progress'] == 99.9
assert engine.get_record('soundcloud', 'hls2')['progress'] == 99.9
def test_update_download_progress_fragmented_handles_zero_index(tmp_dl: Path) -> None:
"""Defensive: yt-dlp can call the progress hook with fragment_index=0
on the very first callback (HLS init segment). Progress is 0% and
nothing crashes."""
"""Defensive: fragment_index=0 on first callback → progress 0,
no crash."""
client = SoundcloudClient(download_path=str(tmp_dl))
with client._download_lock:
client.active_downloads['hls3'] = {
'id': 'hls3', 'filename': '', 'username': 'soundcloud',
'state': 'InProgress, Downloading', 'progress': 0.0, 'size': 0,
'transferred': 0, 'speed': 0, 'time_remaining': None,
}
engine = _wire_engine(client)
engine.add_record('soundcloud', 'hls3', {
'id': 'hls3', 'filename': '', 'username': 'soundcloud',
'state': 'InProgress, Downloading', 'progress': 0.0, 'size': 0,
'transferred': 0, 'speed': 0, 'time_remaining': None,
})
client._update_download_progress_fragmented(
'hls3', downloaded=0, fragment_index=0, fragment_count=10,
speed_start=time.time(),
)
# No exception, progress stays 0
assert client.active_downloads['hls3']['progress'] == 0.0
assert engine.get_record('soundcloud', 'hls3')['progress'] == 0.0
def test_update_download_progress_fragmented_silently_skips_unknown_id(tmp_dl: Path) -> None:
@ -590,13 +594,13 @@ def test_update_download_progress_fragmented_silently_skips_unknown_id(tmp_dl: P
def test_get_all_downloads_returns_status_objects(tmp_dl: Path) -> None:
client = SoundcloudClient(download_path=str(tmp_dl))
with client._download_lock:
client.active_downloads['s1'] = {
'id': 's1', 'filename': 'f', 'username': 'soundcloud',
'state': 'InProgress, Downloading', 'progress': 33.3, 'size': 1000,
'transferred': 333, 'speed': 100, 'time_remaining': 7,
'file_path': None,
}
engine = _wire_engine(client)
engine.add_record('soundcloud', 's1', {
'id': 's1', 'filename': 'f', 'username': 'soundcloud',
'state': 'InProgress, Downloading', 'progress': 33.3, 'size': 1000,
'transferred': 333, 'speed': 100, 'time_remaining': 7,
'file_path': None,
})
out = _run(client.get_all_downloads())
assert len(out) == 1
assert isinstance(out[0], DownloadStatus)
@ -606,54 +610,56 @@ def test_get_all_downloads_returns_status_objects(tmp_dl: Path) -> None:
def test_get_download_status_returns_none_for_unknown(tmp_dl: Path) -> None:
client = SoundcloudClient(download_path=str(tmp_dl))
_wire_engine(client)
assert _run(client.get_download_status('nope')) is None
def test_cancel_download_marks_state(tmp_dl: Path) -> None:
client = SoundcloudClient(download_path=str(tmp_dl))
with client._download_lock:
client.active_downloads['c1'] = {
'id': 'c1', 'filename': '', 'username': 'soundcloud',
'state': 'InProgress, Downloading', 'progress': 50.0, 'size': 0,
'transferred': 0, 'speed': 0, 'time_remaining': None,
}
engine = _wire_engine(client)
engine.add_record('soundcloud', 'c1', {
'id': 'c1', 'filename': '', 'username': 'soundcloud',
'state': 'InProgress, Downloading', 'progress': 50.0, 'size': 0,
'transferred': 0, 'speed': 0, 'time_remaining': None,
})
assert _run(client.cancel_download('c1')) is True
assert client.active_downloads['c1']['state'] == 'Cancelled'
assert engine.get_record('soundcloud', 'c1')['state'] == 'Cancelled'
def test_cancel_download_with_remove_drops_entry(tmp_dl: Path) -> None:
client = SoundcloudClient(download_path=str(tmp_dl))
with client._download_lock:
client.active_downloads['c2'] = {
'id': 'c2', 'filename': '', 'username': 'soundcloud',
'state': 'InProgress, Downloading', 'progress': 0.0, 'size': 0,
'transferred': 0, 'speed': 0, 'time_remaining': None,
}
engine = _wire_engine(client)
engine.add_record('soundcloud', 'c2', {
'id': 'c2', 'filename': '', 'username': 'soundcloud',
'state': 'InProgress, Downloading', 'progress': 0.0, 'size': 0,
'transferred': 0, 'speed': 0, 'time_remaining': None,
})
assert _run(client.cancel_download('c2', remove=True)) is True
assert 'c2' not in client.active_downloads
assert engine.get_record('soundcloud', 'c2') is None
def test_cancel_download_returns_false_for_unknown(tmp_dl: Path) -> None:
client = SoundcloudClient(download_path=str(tmp_dl))
_wire_engine(client)
assert _run(client.cancel_download('not_real')) is False
def test_clear_completed_drops_terminal_entries_only(tmp_dl: Path) -> None:
"""Terminal states get cleared; in-flight downloads survive."""
client = SoundcloudClient(download_path=str(tmp_dl))
engine = _wire_engine(client)
base = {'filename': '', 'username': 'soundcloud', 'progress': 0.0,
'size': 0, 'transferred': 0, 'speed': 0, 'time_remaining': None}
with client._download_lock:
client.active_downloads['done'] = {**base, 'id': 'done', 'state': 'Completed, Succeeded'}
client.active_downloads['err'] = {**base, 'id': 'err', 'state': 'Errored'}
client.active_downloads['cnc'] = {**base, 'id': 'cnc', 'state': 'Cancelled'}
client.active_downloads['live'] = {**base, 'id': 'live', 'state': 'InProgress, Downloading'}
engine.add_record('soundcloud', 'done', {**base, 'id': 'done', 'state': 'Completed, Succeeded'})
engine.add_record('soundcloud', 'err', {**base, 'id': 'err', 'state': 'Errored'})
engine.add_record('soundcloud', 'cnc', {**base, 'id': 'cnc', 'state': 'Cancelled'})
engine.add_record('soundcloud', 'live', {**base, 'id': 'live', 'state': 'InProgress, Downloading'})
assert _run(client.clear_all_completed_downloads()) is True
assert 'done' not in client.active_downloads
assert 'err' not in client.active_downloads
assert 'cnc' not in client.active_downloads
assert 'live' in client.active_downloads
assert engine.get_record('soundcloud', 'done') is None
assert engine.get_record('soundcloud', 'err') is None
assert engine.get_record('soundcloud', 'cnc') is None
assert engine.get_record('soundcloud', 'live') is not None
# ---------------------------------------------------------------------------
@ -723,6 +729,7 @@ def test_live_download_a_known_public_track(tmp_dl: Path) -> None:
another reliably-public free track.
"""
client = SoundcloudClient(download_path=str(tmp_dl))
engine = _wire_engine(client)
# Search-then-download flow: pick the first hit for a popular query
tracks, _ = _run(client.search("creative commons electronic music"))
assert tracks, "Live search returned no results"
@ -735,7 +742,7 @@ def test_live_download_a_known_public_track(tmp_dl: Path) -> None:
final_state = None
final_path = None
while time.time() < deadline:
info = client.active_downloads.get(download_id, {})
info = engine.get_record('soundcloud', download_id) or {}
final_state = info.get('state')
final_path = info.get('file_path')
if final_state in {'Completed, Succeeded', 'Errored', 'Cancelled'}:

View file

@ -88,7 +88,7 @@ from plexapi.myplex import MyPlexAccount, MyPlexPinLogin
from core.jellyfin_client import JellyfinClient
from core.navidrome_client import NavidromeClient
from core.soulseek_client import SoulseekClient
from core.download_orchestrator import DownloadOrchestrator
from core.download_orchestrator import DownloadOrchestrator, set_download_orchestrator
from core.tidal_client import TidalClient # Added import for Tidal
from core.matching_engine import MusicMatchingEngine
from core.database_update_worker import DatabaseUpdateWorker
@ -570,7 +570,7 @@ IS_SHUTTING_DOWN = False
# Each client is initialized independently so one failure doesn't take down everything.
# Previously, a single exception set ALL clients to None, breaking the entire app.
logger.info("Initializing SoulSync services for Web UI...")
spotify_client = plex_client = jellyfin_client = navidrome_client = soulsync_library_client = soulseek_client = tidal_client = matching_engine = sync_service = web_scan_manager = None
spotify_client = plex_client = jellyfin_client = navidrome_client = soulsync_library_client = download_orchestrator = tidal_client = matching_engine = sync_service = web_scan_manager = None
try:
spotify_client = get_spotify_client()
@ -604,7 +604,11 @@ except Exception as e:
logger.error(f" SoulSync library client failed to initialize: {e}")
try:
soulseek_client = DownloadOrchestrator()
download_orchestrator = DownloadOrchestrator()
# Install as the process-wide singleton so callers reaching for
# get_download_orchestrator() see the same instance web_server.py
# constructs at boot. Matches Cin's metadata engine pattern.
set_download_orchestrator(download_orchestrator)
logger.info(" Download orchestrator initialized")
except Exception as e:
logger.error(f" Download orchestrator failed to initialize: {e}")
@ -622,28 +626,22 @@ except Exception as e:
logger.error(f" Matching engine failed to initialize: {e}")
try:
sync_service = PlaylistSyncService(spotify_client, plex_client, soulseek_client, jellyfin_client, navidrome_client)
sync_service = PlaylistSyncService(spotify_client, plex_client, download_orchestrator, jellyfin_client, navidrome_client)
logger.info(" Playlist sync service initialized")
except Exception as e:
logger.error(f" Playlist sync service failed to initialize: {e}")
# Inject shutdown check callback into YouTube and Tidal clients (avoids circular imports)
if soulseek_client:
if hasattr(soulseek_client, 'youtube'):
soulseek_client.youtube.set_shutdown_check(lambda: IS_SHUTTING_DOWN)
logger.info(" Configured YouTube client shutdown callback")
if hasattr(soulseek_client, 'tidal'):
soulseek_client.tidal.set_shutdown_check(lambda: IS_SHUTTING_DOWN)
logger.info(" Configured Tidal download client shutdown callback")
if hasattr(soulseek_client, 'qobuz'):
soulseek_client.qobuz.set_shutdown_check(lambda: IS_SHUTTING_DOWN)
logger.info(" Configured Qobuz client shutdown callback")
if hasattr(soulseek_client, 'hifi'):
soulseek_client.hifi.set_shutdown_check(lambda: IS_SHUTTING_DOWN)
logger.info(" Configured HiFi client shutdown callback")
if hasattr(soulseek_client, 'soundcloud') and soulseek_client.soundcloud:
soulseek_client.soundcloud.set_shutdown_check(lambda: IS_SHUTTING_DOWN)
logger.info(" Configured SoundCloud client shutdown callback")
# Inject shutdown check callback into every download source that
# accepts one. Generic dispatch via the registry — no per-source
# attribute reaches needed.
if download_orchestrator and hasattr(download_orchestrator, 'registry'):
for _src_name, _src_client in download_orchestrator.registry.all_plugins():
if _src_client is not None and hasattr(_src_client, 'set_shutdown_check'):
try:
_src_client.set_shutdown_check(lambda: IS_SHUTTING_DOWN)
logger.info(" Configured %s client shutdown callback", _src_name)
except Exception as _exc:
logger.warning(" %s set_shutdown_check failed: %s", _src_name, _exc)
# Initialize web scan manager for automatic post-download scanning
try:
@ -1999,9 +1997,9 @@ def _register_automation_handlers():
hybrid_order = config_manager.get('download_source.hybrid_order', ['hifi', 'youtube', 'soulseek'])
soulseek_active = (dl_mode == 'soulseek' or
(dl_mode == 'hybrid' and 'soulseek' in hybrid_order))
# soulseek_client is a DownloadOrchestrator; the real client lives on
# .soulseek. Match the getattr pattern used at the other call sites.
slskd = getattr(soulseek_client, 'soulseek', None) if soulseek_client else None
# Reach the underlying SoulseekClient via the orchestrator's
# generic accessor.
slskd = download_orchestrator.client('soulseek') if download_orchestrator else None
if not soulseek_active or not slskd or not slskd.base_url:
_update_automation_progress(automation_id,
log_line='Soulseek not active — skipped', log_type='skip')
@ -2011,7 +2009,7 @@ def _register_automation_handlers():
log_line='Auto-clear disabled in settings', log_type='skip')
return {'status': 'skipped'}
try:
success = run_async(soulseek_client.maintain_search_history_with_buffer(
success = run_async(download_orchestrator.maintain_search_history_with_buffer(
keep_searches=50, trigger_threshold=200
))
if success:
@ -2047,7 +2045,7 @@ def _register_automation_handlers():
log_line='Skipped — downloads active', log_type='skip')
return {'status': 'completed'}
run_async(soulseek_client.clear_all_completed_downloads())
run_async(download_orchestrator.clear_all_completed_downloads())
if not has_post_processing:
_sweep_empty_download_directories()
_update_automation_progress(automation_id,
@ -2102,7 +2100,7 @@ def _register_automation_handlers():
log_line='Download queue: skipped (active batches)', log_type='skip')
else:
try:
run_async(soulseek_client.clear_all_completed_downloads())
run_async(download_orchestrator.clear_all_completed_downloads())
steps.append('Download queue: cleared')
_update_automation_progress(automation_id,
log_line='Download queue: cleared', log_type='success')
@ -2160,7 +2158,7 @@ def _register_automation_handlers():
_update_automation_progress(automation_id,
log_line='Search cleanup: disabled in settings', log_type='skip')
else:
run_async(soulseek_client.maintain_search_history_with_buffer(
run_async(download_orchestrator.maintain_search_history_with_buffer(
keep_searches=50, trigger_threshold=200
))
steps.append('Search history: cleaned')
@ -2292,7 +2290,7 @@ def _register_automation_handlers():
if automation_id:
_update_automation_progress(automation_id,
phase='Searching', log_line=f'Searching: {query}', log_type='info')
result = run_async(soulseek_client.search_and_download_best(query))
result = run_async(download_orchestrator.search_and_download_best(query))
if result:
if automation_id:
_update_automation_progress(automation_id,
@ -2354,7 +2352,7 @@ try:
app.register_blueprint(api_bp, url_prefix='/api/v1')
app.soulsync = {
'spotify_client': spotify_client,
'soulseek_client': soulseek_client,
'download_orchestrator': download_orchestrator,
'tidal_client': tidal_client,
'matching_engine': matching_engine,
'config_manager': config_manager,
@ -2459,8 +2457,9 @@ def get_cached_transfer_data():
# First, get Soulseek downloads from API
transfers_data = None
if not soulseek_known_down and soulseek_client and getattr(soulseek_client, 'soulseek', None) and soulseek_client.soulseek.base_url:
transfers_data = run_async(soulseek_client._make_request('GET', 'transfers/downloads'))
_slsk = download_orchestrator.client("soulseek") if download_orchestrator else None
if not soulseek_known_down and _slsk and _slsk.base_url:
transfers_data = run_async(download_orchestrator._make_request('GET', 'transfers/downloads'))
if transfers_data:
all_transfers = []
for user_data in transfers_data:
@ -2475,33 +2474,35 @@ def get_cached_transfer_data():
key = _make_context_key(transfer.get('username'), transfer.get('filename', ''))
live_transfers_lookup[key] = transfer
# Also add non-Soulseek downloads (avoid redundant slskd call through orchestrator).
# Every streaming source must appear here — task progress for in-flight
# downloads comes from this lookup. Missing a source = task.progress
# stays at 0 even when the underlying client knows the real percent.
# Also add non-Soulseek downloads. Soulseek is excluded
# because slskd's transfers endpoint was already pulled
# above — without the exclude both fetch paths run.
# Every streaming source must appear here — task progress
# for in-flight downloads comes from this lookup. Missing a
# source = task.progress stays at 0 even when the
# underlying client knows the real percent.
try:
all_downloads = []
for _dl_client in [soulseek_client.youtube, soulseek_client.tidal, soulseek_client.qobuz,
soulseek_client.hifi, soulseek_client.deezer_dl, soulseek_client.lidarr,
getattr(soulseek_client, 'soundcloud', None)]:
if _dl_client:
try:
all_downloads.extend(run_async(_dl_client.get_all_downloads()))
except Exception:
pass
if download_orchestrator and hasattr(download_orchestrator, 'engine'):
try:
all_downloads = run_async(
download_orchestrator.engine.get_all_downloads(exclude=('soulseek',))
)
except Exception:
pass
for download in all_downloads:
key = _make_context_key(download.username, download.filename)
# Convert DownloadStatus to transfer dict format
live_transfers_lookup[key] = {
'id': download.id,
'filename': download.filename,
'username': download.username,
'state': download.state,
'percentComplete': download.progress,
'size': download.size,
'bytesTransferred': download.transferred,
'averageSpeed': download.speed,
}
key = _make_context_key(download.username, download.filename)
# Convert DownloadStatus to transfer dict format
live_transfers_lookup[key] = {
'id': download.id,
'filename': download.filename,
'username': download.username,
'state': download.state,
'percentComplete': download.progress,
'size': download.size,
'bytesTransferred': download.transferred,
'averageSpeed': download.speed,
}
except Exception as e:
logger.error(f"Could not fetch streaming source downloads: {e}")
@ -3059,7 +3060,7 @@ def _build_prepare_stream_deps():
return _streaming_prepare.PrepareStreamDeps(
config_manager=config_manager,
soulseek_client=soulseek_client,
download_orchestrator=download_orchestrator,
stream_lock=stream_lock,
project_root=os.path.dirname(os.path.abspath(__file__)),
docker_resolve_path=docker_resolve_path,
@ -3506,10 +3507,10 @@ def get_status():
if is_serverless:
soulseek_status = True
soulseek_response_time = 0
elif soulseek_relevant and soulseek_client:
elif soulseek_relevant and download_orchestrator:
soulseek_start = time.time()
try:
soulseek_status = run_async(soulseek_client.check_connection())
soulseek_status = run_async(download_orchestrator.check_connection())
except Exception:
soulseek_status = False
soulseek_response_time = (time.time() - soulseek_start) * 1000
@ -3922,7 +3923,7 @@ def _build_system_stats():
if soulseek_active and not soulseek_known_down:
try:
transfers_data = run_async(soulseek_client._make_request('GET', 'transfers/downloads'))
transfers_data = run_async(download_orchestrator._make_request('GET', 'transfers/downloads'))
if transfers_data:
for user_data in transfers_data:
if 'directories' in user_data:
@ -4148,11 +4149,12 @@ def handle_settings():
if navidrome_client:
navidrome_client.reload_config()
# Reload orchestrator settings (download source mode, hybrid_primary, etc.)
if soulseek_client:
soulseek_client.reload_settings()
if download_orchestrator:
download_orchestrator.reload_settings()
# Reload YouTube client settings (rate limiting, cookies)
if hasattr(soulseek_client, 'youtube'):
soulseek_client.youtube.reload_settings()
_yt = download_orchestrator.client("youtube")
if _yt:
_yt.reload_settings()
# FIX: Re-instantiate the global tidal_client to pick up new settings
try:
tidal_client = TidalClient()
@ -4184,7 +4186,7 @@ def handle_settings():
data = dict(config_manager.config_data)
# Include which download sources are configured so the UI can auto-disable unconfigured ones
try:
data['_source_status'] = soulseek_client.get_source_status()
data['_source_status'] = download_orchestrator.get_source_status()
except Exception:
pass
return jsonify(data)
@ -6735,7 +6737,7 @@ def _build_search_deps():
spotify_client=spotify_client,
hydrabase_client=hydrabase_client,
hydrabase_worker=hydrabase_worker,
soulseek_client=soulseek_client,
download_orchestrator=download_orchestrator,
fix_artist_image_url=fix_artist_image_url,
is_hydrabase_active=_is_hydrabase_active,
get_metadata_fallback_source=_get_metadata_fallback_source,
@ -6761,7 +6763,7 @@ def search_music():
add_activity_item("", "Search Started", f"'{query}'", "Now")
try:
results = _search_basic.run_basic_soulseek_search(query, soulseek_client, run_async)
results = _search_basic.run_basic_soulseek_search(query, download_orchestrator, run_async)
add_activity_item("", "Search Complete", f"'{query}' - {len(results)} results", "Now")
return jsonify({"results": results})
except Exception as e:
@ -6816,7 +6818,7 @@ def enhanced_search_source(source_name):
When the requested source's client isn't available (Spotify unauthed,
Discogs missing token, Hydrabase disconnected, MusicBrainz import
failure, soulseek_client.youtube missing), returns plain JSON
failure, download_orchestrator.client("youtube") missing), returns plain JSON
`{"artists":[],"albums":[],"tracks":[],"available":false}` to match
the original endpoint contract.
"""
@ -6901,7 +6903,7 @@ def stream_enhanced_search_track():
album_name=album_name,
duration_ms=duration_ms,
config_manager=config_manager,
soulseek_client=soulseek_client,
download_orchestrator=download_orchestrator,
matching_engine=matching_engine,
run_async=run_async,
)
@ -7040,7 +7042,7 @@ def download_music_video():
def _progress(pct):
_music_video_downloads[video_id]['progress'] = round(pct, 1)
final_path = soulseek_client.youtube.download_music_video(video_url, output_path, progress_callback=_progress)
final_path = download_orchestrator.client("youtube").download_music_video(video_url, output_path, progress_callback=_progress)
if final_path and os.path.exists(final_path):
_music_video_downloads[video_id]['status'] = 'completed'
@ -7099,7 +7101,7 @@ def start_download():
filename = track_data.get('filename')
file_size = track_data.get('size', 0)
download_id = run_async(soulseek_client.download(
download_id = run_async(download_orchestrator.download(
username,
filename,
file_size
@ -7148,7 +7150,7 @@ def start_download():
if not username or not filename:
return jsonify({"error": "Missing username or filename."}), 400
download_id = run_async(soulseek_client.download(username, filename, file_size))
download_id = run_async(download_orchestrator.download(username, filename, file_size))
logger.info(f"Download ID returned: {download_id}")
if download_id:
@ -7344,7 +7346,7 @@ def get_download_status():
A robust status checker that correctly finds completed files by searching
the entire download directory with fuzzy matching, mirroring the logic from downloads.py.
"""
if not soulseek_client:
if not download_orchestrator:
return jsonify({"transfers": []})
try:
@ -7353,7 +7355,7 @@ def get_download_status():
soulseek_known_down = not _status_cache.get('soulseek', {}).get('connected', True)
transfers_data = None
if not soulseek_known_down:
transfers_data = run_async(soulseek_client._make_request('GET', 'transfers/downloads'))
transfers_data = run_async(download_orchestrator._make_request('GET', 'transfers/downloads'))
# Don't return early if no Soulseek transfers - YouTube/Tidal downloads need to be checked too!
all_transfers = []
@ -7421,7 +7423,7 @@ def get_download_status():
transfer_id = file_info.get('id')
if transfer_id:
try:
run_async(soulseek_client.cancel_download(str(transfer_id), username, remove=True))
run_async(download_orchestrator.cancel_download(str(transfer_id), username, remove=True))
except Exception:
pass
_orphaned_download_keys.discard(context_key)
@ -7533,7 +7535,7 @@ def get_download_status():
# Also include YouTube/Tidal downloads in the response
try:
all_streaming_downloads = run_async(soulseek_client.get_all_downloads())
all_streaming_downloads = run_async(download_orchestrator.get_all_downloads())
for download in all_streaming_downloads:
if download.username in ('youtube', 'tidal', 'qobuz', 'hifi', 'deezer_dl', 'lidarr', 'soundcloud'):
@ -7665,7 +7667,7 @@ def cancel_download():
return jsonify({"success": False, "error": "Missing download_id or username."}), 400
try:
success = _downloads_cancel.cancel_single_download(soulseek_client, run_async, download_id, username)
success = _downloads_cancel.cancel_single_download(download_orchestrator, run_async, download_id, username)
if success:
return jsonify({"success": True, "message": "Download cancelled."})
return jsonify({"success": False, "error": "Failed to cancel download via slskd."}), 500
@ -7679,7 +7681,7 @@ def cancel_all_downloads():
"""Cancel all active downloads from slskd, then clear completed ones."""
try:
success, msg = _downloads_cancel.cancel_all_active(
soulseek_client, run_async, _sweep_empty_download_directories,
download_orchestrator, run_async, _sweep_empty_download_directories,
)
if success:
return jsonify({"success": True, "message": msg})
@ -7694,7 +7696,7 @@ def clear_finished_downloads():
"""Clear all terminal (completed, cancelled, failed) downloads from slskd."""
try:
success = _downloads_cancel.clear_finished_active(
soulseek_client, run_async, _sweep_empty_download_directories,
download_orchestrator, run_async, _sweep_empty_download_directories,
)
if success:
return jsonify({"success": True, "message": "Finished downloads cleared."})
@ -7803,7 +7805,7 @@ def download_selected_candidate(task_id):
batch['active_count'] = batch.get('active_count', 0) + 1
# Build a TrackResult-like candidate object
from core.soulseek_client import TrackResult
from core.download_plugins.types import TrackResult
candidate = TrackResult(
username=username,
filename=filename,
@ -8081,7 +8083,7 @@ def clear_all_searches():
Clear all searches from slskd search history.
"""
try:
success = run_async(soulseek_client.clear_all_searches())
success = run_async(download_orchestrator.clear_all_searches())
if success:
add_activity_item("", "Search Cleanup", "All search history cleared manually", "Now")
return jsonify({"success": True, "message": "All searches cleared."})
@ -8101,7 +8103,7 @@ def maintain_search_history():
keep_searches = data.get('keep_searches', 50)
trigger_threshold = data.get('trigger_threshold', 200)
success = run_async(soulseek_client.maintain_search_history_with_buffer(
success = run_async(download_orchestrator.maintain_search_history_with_buffer(
keep_searches=keep_searches, trigger_threshold=trigger_threshold
))
if success:
@ -11678,34 +11680,19 @@ def redownload_search_sources(track_id):
candidates = []
database = get_database()
# Get all available download source clients
# Get all available download source clients via the orchestrator's
# generic accessor — replaces the old per-source if/hasattr chain
# that Cin called out as defeating the purpose of the registry refactor.
download_clients = {}
try:
orch = soulseek_client # The download orchestrator
if hasattr(orch, 'soulseek') and orch.soulseek:
if not (hasattr(orch.soulseek, 'is_configured') and not orch.soulseek.is_configured()):
download_clients['soulseek'] = orch.soulseek
if hasattr(orch, 'youtube') and orch.youtube:
if not (hasattr(orch.youtube, 'is_configured') and not orch.youtube.is_configured()):
download_clients['youtube'] = orch.youtube
if hasattr(orch, 'tidal') and orch.tidal:
if not (hasattr(orch.tidal, 'is_configured') and not orch.tidal.is_configured()):
download_clients['tidal'] = orch.tidal
if hasattr(orch, 'qobuz') and orch.qobuz:
if not (hasattr(orch.qobuz, 'is_configured') and not orch.qobuz.is_configured()):
download_clients['qobuz'] = orch.qobuz
if hasattr(orch, 'hifi') and orch.hifi:
if not (hasattr(orch.hifi, 'is_configured') and not orch.hifi.is_configured()):
download_clients['hifi'] = orch.hifi
if hasattr(orch, 'deezer_dl') and orch.deezer_dl:
if not (hasattr(orch.deezer_dl, 'is_configured') and not orch.deezer_dl.is_configured()):
download_clients['deezer_dl'] = orch.deezer_dl
if download_orchestrator and hasattr(download_orchestrator, 'configured_clients'):
download_clients = dict(download_orchestrator.configured_clients())
except Exception as e:
logger.warning(f"[Redownload] Error getting download clients: {e}")
if not download_clients:
# Fallback: use orchestrator directly
download_clients = {'default': soulseek_client}
download_clients = {'default': download_orchestrator}
logger.info(f"[Redownload] Streaming search across {len(download_clients)} sources: {list(download_clients.keys())}")
@ -12667,7 +12654,7 @@ def _start_enhanced_album_download(enhanced_tracks, unmatched_tracks, spotify_ar
continue
# Start download
download_id = run_async(soulseek_client.download(username, filename, size))
download_id = run_async(download_orchestrator.download(username, filename, size))
if download_id:
context_key = _make_context_key(username, filename)
@ -12715,7 +12702,7 @@ def _start_enhanced_album_download(enhanced_tracks, unmatched_tracks, spotify_ar
if not username or not filename:
continue
download_id = run_async(soulseek_client.download(username, filename, size))
download_id = run_async(download_orchestrator.download(username, filename, size))
if download_id:
context_key = _make_context_key(username, filename)
@ -12823,7 +12810,7 @@ def _start_album_download_tasks(album_result, spotify_artist, spotify_album):
'disc_number': corrected_meta.get('disc_number', 1)
}
download_id = run_async(soulseek_client.download(username, filename, size))
download_id = run_async(download_orchestrator.download(username, filename, size))
if download_id:
context_key = _make_context_key(username, filename)
@ -12890,7 +12877,7 @@ def start_matched_download():
if not username or not filename:
return jsonify({"success": False, "error": "Missing username or filename"}), 400
download_id = run_async(soulseek_client.download(username, filename, size))
download_id = run_async(download_orchestrator.download(username, filename, size))
if download_id:
context_key = _make_context_key(username, filename)
@ -12950,7 +12937,7 @@ def start_matched_download():
download_payload['title'] = parsed_meta.get('title') or download_payload.get('title')
download_payload['artist'] = parsed_meta.get('artist') or download_payload.get('artist')
download_id = run_async(soulseek_client.download(username, filename, size))
download_id = run_async(download_orchestrator.download(username, filename, size))
if download_id:
context_key = _make_context_key(username, filename)
@ -12979,7 +12966,7 @@ def start_matched_download():
def _parse_filename_metadata(filename: str) -> dict:
"""
A direct port of the metadata parsing logic from the GUI's soulseek_client.py.
A direct port of the metadata parsing logic from the GUI's download_orchestrator.py.
This is the crucial missing step that cleans filenames BEFORE Spotify matching.
"""
return parse_filename_metadata(filename)
@ -14155,7 +14142,7 @@ def _enhance_file_metadata(file_path: str, context: dict, artist: dict, album_in
genius_worker=genius_worker,
spotify_enrichment_worker=spotify_enrichment_worker,
itunes_enrichment_worker=itunes_enrichment_worker,
hifi_client=soulseek_client.hifi if soulseek_client else None,
hifi_client=download_orchestrator.client("hifi") if download_orchestrator else None,
),
)
@ -14260,7 +14247,7 @@ def _post_process_matched_download_with_verification(context_key, context, file_
genius_worker=genius_worker,
spotify_enrichment_worker=spotify_enrichment_worker,
itunes_enrichment_worker=itunes_enrichment_worker,
hifi_client=soulseek_client.hifi if soulseek_client else None,
hifi_client=download_orchestrator.client("hifi") if download_orchestrator else None,
),
)
@ -14384,7 +14371,7 @@ def _post_process_matched_download(context_key, context, file_path):
genius_worker=genius_worker,
spotify_enrichment_worker=spotify_enrichment_worker,
itunes_enrichment_worker=itunes_enrichment_worker,
hifi_client=soulseek_client.hifi if soulseek_client else None,
hifi_client=download_orchestrator.client("hifi") if download_orchestrator else None,
),
)
@ -16828,7 +16815,7 @@ def _build_master_deps():
return _downloads_master.MasterDeps(
config_manager=config_manager,
soulseek_client=soulseek_client,
download_orchestrator=download_orchestrator,
run_async=run_async,
mb_worker=mb_worker,
mb_release_cache=mb_release_cache,
@ -16866,7 +16853,7 @@ def _build_post_processing_deps():
"""Build the PostProcessDeps bundle from web_server.py globals on each call."""
return _downloads_post_processing.PostProcessDeps(
config_manager=config_manager,
soulseek_client=soulseek_client,
download_orchestrator=download_orchestrator,
run_async=run_async,
docker_resolve_path=docker_resolve_path,
extract_filename=extract_filename,
@ -16893,7 +16880,7 @@ from core.downloads import task_worker as _downloads_task_worker
def _build_task_worker_deps():
"""Build TaskWorkerDeps bundle from web_server.py globals on each call."""
return _downloads_task_worker.TaskWorkerDeps(
soulseek_client=soulseek_client,
download_orchestrator=download_orchestrator,
matching_engine=matching_engine,
run_async=run_async,
try_source_reuse=_try_source_reuse,
@ -16919,7 +16906,7 @@ from core.downloads import candidates as _downloads_candidates
def _build_candidates_deps():
"""Build the CandidatesDeps bundle from web_server.py globals on each call."""
return _downloads_candidates.CandidatesDeps(
soulseek_client=soulseek_client,
download_orchestrator=download_orchestrator,
spotify_client=spotify_client,
run_async=run_async,
get_database=get_database,
@ -17084,7 +17071,7 @@ def _try_source_reuse(task_id, batch_id, track):
# Sort by confidence, filter by quality preference
candidates.sort(key=lambda c: c.confidence, reverse=True)
_sr.info(f"Found {len(candidates)} candidates above 0.70, best={candidates[0].confidence:.3f} ({candidates[0].filename})")
slsk = soulseek_client.soulseek if hasattr(soulseek_client, 'soulseek') else soulseek_client
slsk = download_orchestrator.client("soulseek") if hasattr(download_orchestrator, 'client') else download_orchestrator
filtered = slsk.filter_results_by_quality_preference(candidates)
if not filtered:
_sr.info(f"Quality filter rejected all candidates for task {task_id}")
@ -17163,8 +17150,8 @@ def _store_batch_source(batch_id, username, filename):
return
try:
# Access SoulseekClient directly (soulseek_client is DownloadOrchestrator)
slsk = soulseek_client.soulseek if hasattr(soulseek_client, 'soulseek') else soulseek_client
# Access SoulseekClient directly (download_orchestrator is DownloadOrchestrator)
slsk = download_orchestrator.client("soulseek") if hasattr(download_orchestrator, 'client') else download_orchestrator
_sr.info(f"Browsing {username}:{folder_path}...")
files = run_async(slsk.browse_user_directory(username, folder_path))
if not files:
@ -17495,7 +17482,7 @@ def cancel_download_task():
if download_id and username:
try:
# This is an async call, so we run it and wait
run_async(soulseek_client.cancel_download(download_id, username, remove=True))
run_async(download_orchestrator.cancel_download(download_id, username, remove=True))
logger.warning(f"Successfully cancelled Soulseek download {download_id} for task {task_id}")
except Exception as e:
logger.error(f"Failed to cancel download on slskd, but worker already moved on: {e}")
@ -17740,14 +17727,14 @@ def cancel_task_v2():
# username: youtube/tidal/qobuz/hifi/deezer_dl/lidarr go to
# their streaming clients, anything else goes to Soulseek.
#
# Replaces an older block that assumed soulseek_client was a
# Replaces an older block that assumed download_orchestrator was a
# raw SoulseekClient and accessed .base_url / ._make_request
# directly — crashed with AttributeError on the orchestrator
# and silently left streaming downloads running in background.
try:
logger.info(f"[Atomic Cancel] Dispatching cancel to orchestrator: username={username} download_id={download_id}")
cancel_success = run_async(
soulseek_client.cancel_download(download_id, username, remove=True)
download_orchestrator.cancel_download(download_id, username, remove=True)
)
if cancel_success:
logger.info(f"[Atomic Cancel] Orchestrator cancelled download: {download_id}")
@ -19732,7 +19719,7 @@ def get_discover_album(source, album_id):
def hifi_status():
"""Check if HiFi API instances are reachable."""
try:
hifi = soulseek_client.hifi
hifi = download_orchestrator.client("hifi")
available = hifi.is_available()
version = hifi.get_version() if available else None
return jsonify({
@ -19755,9 +19742,7 @@ def soundcloud_status():
of just verifying the import succeeded.
"""
try:
sc = None
if soulseek_client and hasattr(soulseek_client, 'soundcloud'):
sc = soulseek_client.soundcloud
sc = download_orchestrator.client("soundcloud") if download_orchestrator and hasattr(download_orchestrator, 'client') else None
if not sc:
return jsonify({
"available": False,
@ -19785,7 +19770,7 @@ def hifi_instances():
"""Check availability of all HiFi API instances."""
import requests as req
try:
hifi = soulseek_client.hifi
hifi = download_orchestrator.client("hifi")
instances = list(hifi._instances)
results = []
for url in instances:
@ -19840,9 +19825,9 @@ def hifi_add_instance():
added = db.add_hifi_instance(url, priority)
if not added:
return jsonify({'success': False, 'error': 'Instance already exists'}), 400
# Reload the client
if soulseek_client and hasattr(soulseek_client, 'hifi') and soulseek_client.hifi:
soulseek_client.hifi.reload_instances()
# Reload the HiFi client
if download_orchestrator:
download_orchestrator.reload_instances('hifi')
return jsonify({'success': True, 'url': url})
except Exception as e:
logger.error(f"Error adding HiFi instance: {e}")
@ -19862,9 +19847,9 @@ def hifi_remove_instance():
removed = db.remove_hifi_instance(url)
if not removed:
return jsonify({'success': False, 'error': 'Instance not found'}), 404
# Reload the client
if soulseek_client and hasattr(soulseek_client, 'hifi') and soulseek_client.hifi:
soulseek_client.hifi.reload_instances()
# Reload the HiFi client
if download_orchestrator:
download_orchestrator.reload_instances('hifi')
return jsonify({'success': True, 'url': url})
except Exception as e:
logger.error(f"Error removing HiFi instance: {e}")
@ -19884,8 +19869,8 @@ def hifi_toggle_instance():
from database.music_database import get_database
db = get_database()
db.toggle_hifi_instance(url, enabled)
if soulseek_client and hasattr(soulseek_client, 'hifi') and soulseek_client.hifi:
soulseek_client.hifi.reload_instances()
if download_orchestrator:
download_orchestrator.reload_instances('hifi')
return jsonify({'success': True})
except Exception as e:
logger.error(f"Error toggling HiFi instance: {e}")
@ -19905,9 +19890,9 @@ def hifi_reorder_instances():
db = get_database()
if not db.reorder_hifi_instances(urls):
return jsonify({'success': False, 'error': 'One or more URLs not found'}), 400
# Reload the client
if soulseek_client and hasattr(soulseek_client, 'hifi') and soulseek_client.hifi:
soulseek_client.hifi.reload_instances()
# Reload the HiFi client
if download_orchestrator:
download_orchestrator.reload_instances('hifi')
return jsonify({'success': True})
except Exception as e:
logger.error(f"Error reordering HiFi instances: {e}")
@ -20059,11 +20044,12 @@ def deezer_download_test_download():
def _get_tidal_download_client():
"""Get Tidal download client from the orchestrator, with helpful error if unavailable."""
if not soulseek_client:
if not download_orchestrator:
raise RuntimeError("Download orchestrator not initialized — check startup logs for errors")
if not hasattr(soulseek_client, 'tidal') or not soulseek_client.tidal:
tidal = download_orchestrator.client("tidal") if hasattr(download_orchestrator, 'client') else None
if not tidal:
raise RuntimeError("Tidal download client not available — ensure tidalapi is installed")
return soulseek_client.tidal
return tidal
@app.route('/api/tidal/download/auth/start', methods=['POST'])
def tidal_download_auth_start():
@ -20132,7 +20118,7 @@ def qobuz_auth_login():
if not email or not password:
return jsonify({"success": False, "error": "Email and password required"}), 400
qobuz = soulseek_client.qobuz
qobuz = download_orchestrator.client("qobuz")
result = qobuz.login(email, password)
if result['status'] == 'success':
@ -20155,7 +20141,7 @@ def qobuz_auth_token():
if not token:
return jsonify({"success": False, "error": "Auth token required"}), 400
qobuz = soulseek_client.qobuz
qobuz = download_orchestrator.client("qobuz")
result = qobuz.login_with_token(token)
if result['status'] == 'success':
@ -20172,7 +20158,7 @@ def qobuz_auth_token():
def qobuz_auth_status():
"""Check if Qobuz client is authenticated."""
try:
qobuz = soulseek_client.qobuz
qobuz = download_orchestrator.client("qobuz")
authenticated = qobuz.is_authenticated()
user_info = {}
if authenticated and qobuz.user_info:
@ -20189,7 +20175,7 @@ def qobuz_auth_status():
def qobuz_auth_logout():
"""Logout from Qobuz."""
try:
soulseek_client.qobuz.logout()
download_orchestrator.client("qobuz").logout()
_sync_qobuz_credentials_to_worker()
return jsonify({"success": True})
except Exception as e:
@ -21131,9 +21117,7 @@ def _get_metadata_fallback_client():
def get_deezer_arl_status():
"""Check if Deezer ARL is configured and authenticated."""
try:
deezer_dl = None
if soulseek_client and hasattr(soulseek_client, 'deezer_dl') and soulseek_client.deezer_dl:
deezer_dl = soulseek_client.deezer_dl
deezer_dl = download_orchestrator.client("deezer_dl") if download_orchestrator and hasattr(download_orchestrator, 'client') else None
if deezer_dl and deezer_dl.is_authenticated():
user_data = deezer_dl._user_data or {}
return jsonify({
@ -21150,9 +21134,7 @@ def get_deezer_arl_status():
def get_deezer_arl_playlists():
"""Fetch user playlists via Deezer ARL authentication (like /api/spotify/playlists)."""
try:
deezer_dl = None
if soulseek_client and hasattr(soulseek_client, 'deezer_dl') and soulseek_client.deezer_dl:
deezer_dl = soulseek_client.deezer_dl
deezer_dl = download_orchestrator.client("deezer_dl") if download_orchestrator and hasattr(download_orchestrator, 'client') else None
if not deezer_dl or not deezer_dl.is_authenticated():
return jsonify({'error': 'Deezer ARL not authenticated. Configure your ARL token in Settings > Downloads.'}), 401
@ -21180,9 +21162,7 @@ def get_deezer_arl_playlists():
def get_deezer_arl_playlist_tracks(playlist_id):
"""Fetch full playlist with tracks via ARL (like /api/spotify/playlist/<id>)."""
try:
deezer_dl = None
if soulseek_client and hasattr(soulseek_client, 'deezer_dl') and soulseek_client.deezer_dl:
deezer_dl = soulseek_client.deezer_dl
deezer_dl = download_orchestrator.client("deezer_dl") if download_orchestrator and hasattr(download_orchestrator, 'client') else None
if not deezer_dl or not deezer_dl.is_authenticated():
return jsonify({'error': 'Deezer ARL not authenticated.'}), 401
@ -27374,8 +27354,8 @@ def get_your_artists_sources():
try:
deezer_cl = _get_deezer_client()
deezer_oauth = deezer_cl and hasattr(deezer_cl, 'is_user_authenticated') and deezer_cl.is_user_authenticated()
deezer_arl = (hasattr(soulseek_client, 'deezer_dl') and soulseek_client.deezer_dl
and soulseek_client.deezer_dl.is_authenticated())
deezer_arl = (hasattr(download_orchestrator, 'client') and download_orchestrator.client("deezer_dl")
and download_orchestrator.client("deezer_dl").is_authenticated())
if deezer_oauth or deezer_arl:
connected.append('deezer')
except Exception:
@ -27496,10 +27476,10 @@ def _fetch_and_match_liked_artists(profile_id: int):
if deezer_cl and hasattr(deezer_cl, 'is_user_authenticated') and deezer_cl.is_user_authenticated():
logger.info("[Your Artists] Fetching favorite artists from Deezer (OAuth)...")
artists = deezer_cl.get_user_favorite_artists(limit=200)
elif (hasattr(soulseek_client, 'deezer_dl') and soulseek_client.deezer_dl
and soulseek_client.deezer_dl.is_authenticated()):
elif (hasattr(download_orchestrator, 'client') and download_orchestrator.client("deezer_dl")
and download_orchestrator.client("deezer_dl").is_authenticated()):
logger.info("[Your Artists] Fetching favorite artists from Deezer (ARL)...")
artists = soulseek_client.deezer_dl.get_user_favorite_artists(limit=200)
artists = download_orchestrator.client("deezer_dl").get_user_favorite_artists(limit=200)
for a in artists:
database.upsert_liked_artist(
artist_name=a['name'], source_service='deezer',
@ -27646,8 +27626,8 @@ def get_your_albums_sources():
try:
deezer_cl = _get_deezer_client()
deezer_oauth = deezer_cl and hasattr(deezer_cl, 'is_user_authenticated') and deezer_cl.is_user_authenticated()
deezer_arl = (hasattr(soulseek_client, 'deezer_dl') and soulseek_client.deezer_dl
and soulseek_client.deezer_dl.is_authenticated())
deezer_arl = (hasattr(download_orchestrator, 'client') and download_orchestrator.client("deezer_dl")
and download_orchestrator.client("deezer_dl").is_authenticated())
if deezer_oauth or deezer_arl:
connected.append('deezer')
except Exception:
@ -27754,10 +27734,10 @@ def _fetch_liked_albums(profile_id: int):
if deezer_cl and hasattr(deezer_cl, 'is_user_authenticated') and deezer_cl.is_user_authenticated():
logger.info("[Your Albums] Fetching favorite albums from Deezer (OAuth)...")
albums = deezer_cl.get_user_favorite_albums(limit=500)
elif (hasattr(soulseek_client, 'deezer_dl') and soulseek_client.deezer_dl
and soulseek_client.deezer_dl.is_authenticated()):
elif (hasattr(download_orchestrator, 'client') and download_orchestrator.client("deezer_dl")
and download_orchestrator.client("deezer_dl").is_authenticated()):
logger.info("[Your Albums] Fetching favorite albums from Deezer (ARL)...")
albums = soulseek_client.deezer_dl.get_user_favorite_albums(limit=500)
albums = download_orchestrator.client("deezer_dl").get_user_favorite_albums(limit=500)
for a in albums:
database.upsert_liked_album(
album_name=a['album_name'], artist_name=a['artist_name'],
@ -32678,7 +32658,7 @@ register_runtime_clients(
)
_init_connection_test(
soulseek_client_obj=soulseek_client,
download_orchestrator_obj=download_orchestrator,
qobuz_worker=qobuz_enrichment_worker,
hydrabase_client_obj=hydrabase_client,
docker_resolve_url_fn=docker_resolve_url,
@ -32691,12 +32671,12 @@ _init_discover_hero(get_metadata_fallback_client_fn=_get_metadata_fallback_clien
_init_download_validation(
matching_engine_obj=matching_engine,
soulseek_client_obj=soulseek_client,
download_orchestrator_obj=download_orchestrator,
)
_init_wishlist_failed(
engine=automation_engine,
soulseek_client_obj=soulseek_client,
download_orchestrator_obj=download_orchestrator,
sweep_fn=_sweep_empty_download_directories,
)
@ -32715,7 +32695,7 @@ _init_debug_info(
sync_states_dict=sync_states,
youtube_playlist_states_dict=youtube_playlist_states,
tidal_discovery_states_dict=tidal_discovery_states,
soulseek_client_obj=soulseek_client,
download_orchestrator_obj=download_orchestrator,
log_path=_log_path,
log_dir=_log_dir,
flask_app=app,
@ -32734,7 +32714,7 @@ _init_download_monitor(
start_next_batch_of_downloads=_start_next_batch_of_downloads,
orphaned_download_keys=_orphaned_download_keys,
missing_download_executor_obj=missing_download_executor,
soulseek_client_obj=soulseek_client,
download_orchestrator_obj=download_orchestrator,
)
# --- Hydrabase Auto-Reconnect ---

View file

@ -3432,10 +3432,17 @@ const WHATS_NEW = {
'2.4.2': [
// --- post-2.4.1 dev work — entries hidden by _getLatestWhatsNewVersion until the build version bumps ---
{ date: 'Unreleased — 2.4.2 dev cycle' },
{ title: 'Drop SoundCloud Preview Snippets at Search Time', desc: 'soundcloud serves a ~30s preview clip for tracks that are gated behind go+ / login (very common for major-label uploads — official content basically doesn\'t exist on soundcloud, so what shows up is bootlegs, fan uploads, type beats, and these 30s previews). yt-dlp accepts the preview as the download payload, the post-download integrity check catches the duration mismatch and quarantines the file, but the user just sees "all candidates failed" with no explanation. previews also showed up in the candidate-review modal where clicking one bypassed validation and downloaded the same broken file. now `filter_soundcloud_previews` drops these candidates at every entry point — auto-search scoring, modal-cache fallback, AND the not-found raw-results path — so previews never reach the matcher OR the user. drops candidates < 35s or below half the expected duration, gated on expected being non-trivially long (>60s) so genuine short tracks still pass. also fixed a silent regression in the hybrid-fallback path where the per-source attribute removal left `getattr(orch, \'youtube\', None)` returning None for every source — fallback never fired. now resolves through the orchestrator\'s `client(name)` accessor.', page: 'downloads' },
{ title: 'Internal: Move Shared Download Dataclasses + Singleton Boot Path', desc: 'internal — two architectural cleanups on top of the download engine refactor. (1) `TrackResult`, `AlbumResult`, `DownloadStatus`, `SearchResult` lived in `core/soulseek_client.py` for historical reasons (they grew up there as the soulseek-only types and got exported when other download sources were added). every plugin imported these from the soulseek module just to satisfy the contract — coupling 8 clients to a sibling source for type imports only. moved them to `core/download_plugins/types.py` (the neutral plugin package) and updated all 14 import sites across deezer/hifi/lidarr/qobuz/soundcloud/tidal/youtube clients + the engine + matching engine + redownload + tests. clean break, no backward-compat re-export. (2) `web_server.py` now boots the orchestrator via `set_download_orchestrator(DownloadOrchestrator())` so the singleton factory + boot path share state — `get_download_orchestrator()` returns the same instance the global handle points at instead of lazily building a separate one. matches cin\'s `get_metadata_engine()` pattern.' },
{ title: 'Internal: Rename `soulseek_client` Global → `download_orchestrator`', desc: 'internal — followup cleanup. the global handle in web_server.py was named `soulseek_client` for historical reasons (the orchestrator was originally just the soulseek client and grew downstream sources around it), but the type has long been `DownloadOrchestrator` not `SoulseekClient`. renamed the global + every parameter/attribute that carried the legacy name across web_server.py, api/, core/downloads/*, core/search/*, core/streaming/*, services/sync_service.py, and the test fixtures (`MasterDeps.soulseek_client` → `download_orchestrator`, `init(soulseek_client_obj)` → `init(download_orchestrator_obj)`, etc). module path `core.soulseek_client` and class `SoulseekClient` (the actual soulseek-only client) are unchanged — only the orchestrator handle renamed. ~250 references touched, suite green.' },
{ title: 'Internal: Drop Backward-Compat Per-Source Attrs', desc: 'internal — followup to cin\'s download engine review. removed the `orchestrator.soulseek` / `.youtube` / `.tidal` / `.qobuz` / `.hifi` / `.deezer_dl` / `.lidarr` / `.soundcloud` attribute aliases that were preserved for backward compat. external callers (core/downloads/, core/search/, web_server.py) all migrated to the generic `orchestrator.client(\'<source>\')` accessor — alias-aware (legacy `deezer_dl` resolves to canonical `deezer`), single source of truth via the registry. the orchestrator\'s own internal `self.soulseek` / `self.deezer_dl` reaches also routed through `client()` so the only place that knows about per-source identity is the registry. test fakes updated to expose `client(name)` instead of stuffing attributes; conformance test pinned to the new accessor contract. zero behavior change — just cleaner shape.' },
{ title: 'Internal: Download Engine Review Followup', desc: 'internal — three correctness fixes on top of the download engine refactor, all flagged in cin\'s pr review. (1) `engine.cancel_download(source_hint=\'deezer_dl\')` was silently routing deezer cancels to soulseek because the legacy alias never made it to the engine\'s plugin map — only the registry knew about it. fix: aliases now flow through `register_plugin` and `get_plugin` / `cancel_download` resolve them to the canonical name. (2) `_resolve_source_chain` filtered hybrid_order against canonical registry names only, so any user with `deezer_dl` in their config quietly dropped deezer from hybrid mode. fix: orchestrator normalizes through `registry.get_spec()` first. (3) the worker\'s terminal write was a read-then-write split — a cancel landing between the snapshot and the update could be overwritten back to errored / completed. fix: new atomic `update_record_unless_state` on the engine holds `state_lock` across the check + write; both `_mark_terminal` AND the success path use it now. also added generic `client(name)` / `configured_clients()` / `reload_instances(name?)` accessors on the orchestrator + a `get/set_download_orchestrator()` singleton matching cin\'s `get_metadata_engine()` shape, and migrated 30 external `soulseek_client.<source>` reaches in web_server.py to `client("<source>")`. 18 new tests pin every fix.' },
{ title: 'Internal: Typed Metadata Foundation', desc: 'internal — first step of a multi-pr migration to give the metadata pipeline a real contract. the codebase historically grew duck-typed extractors (`_extract_lookup_value(album_data, "id", "album_id", "collectionId", "release_id", default=...)`) at every consumer site because each provider returns its own response shape. ~150 of those across the codebase. new `core/metadata/types.py` defines canonical typed `Album` / `Track` / `Artist` dataclasses with strict required fields. per-source classmethod converters (from_spotify_dict, from_itunes_dict, from_deezer_dict, from_discogs_dict, from_musicbrainz_dict, from_hydrabase_dict) are the SINGLE place that knows each provider\'s wire shape. zero behavior changes in this pr — pure additive foundation. follow-up prs migrate consumers one at a time. full migration plan documented at docs/metadata-types-migration.md.', page: 'library' },
{ title: 'Internal: Migrate Album-Info Builders to Typed Path', desc: 'internal — steps 2+3 of the typed metadata migration in one pr. two album-info builders now route through `Album.from_<source>_dict()` when the caller passes a known source: `_build_album_info` (used by every album-tracks lookup) and the embedded album section of `_build_single_import_context_payload` (used by single-track import context resolution). legacy duck-typed extraction stays as the fallback when source is empty/unknown, raw input isn\'t a dict, or the typed converter raises — so a converter bug can\'t break album resolution or import context. caller-provided album_id / album_name / artist_name fallbacks apply on the typed path the same way they did on legacy. zero behavior change for existing callers since they don\'t pass a source yet — opt-in only. 22 new tests pin the typed path, the legacy fallback, and parametrized coverage across registered providers.' },
{ title: 'Internal: Migrate Discography + Quality Scanner to Typed Path', desc: 'internal — next round of the typed metadata migration. three more album-shape consumers now route through `Album.from_<source>_dict()` when the caller passes a known source: `_build_discography_release_dict` (artist discography release cards), `_build_artist_detail_release_card` (artist detail page release cards), and `_normalize_track_album` (quality scanner result normalization). legacy duck-typed extraction stays as the fallback when source is empty/unknown, raw input isn\'t a dict, or the typed converter raises — same safety contract as the prior migration steps. 20 new tests pin the typed path + legacy fallback + parametrized coverage across registered providers.' },
{ title: 'Fix: Maintenance Findings Badge Showed Inflated Count With Empty Findings Tab', desc: 'discord report (husoyo): duplicate detector and cover art filler badges showed "364 findings" / "31 findings" after a scan, but clicking into the findings tab showed nothing. cause: `_create_finding` silently dedup-skipped re-discovered issues (when an equivalent row already existed with status pending/resolved/dismissed) but the caller incremented `result.findings_created` regardless of whether a row was actually inserted. so on a re-scan that found the same problems as a prior scan, the badge snapshot recorded 364 even though zero NEW pending rows hit the db. fix: `_create_finding` now returns a bool (True on insert, False on dedup-skip / db error). all 16 repair jobs updated to only increment `findings_created` on True. new `findings_skipped_dedup` counter added to job results and surfaced in the scan log: "Done: 2791 scanned, 0 fixed, 0 findings (363 already existed), 0 errors" — so re-scans show a real count, and you can see at a glance how many findings were carried over from prior scans. also fixed a missing `job_id` kwarg in the album tag consistency job that was silently breaking finding creation for that scan. companion ux improvement: findings tab now auto-switches its status filter from "pending" to "all status" when 0 pending rows exist but resolved/dismissed/auto-fixed rows do — with a small notice so you can see what carried over instead of staring at an "all clear" empty state.', page: 'library' },
{ title: 'Internal: Download Source Plugin Contract', desc: 'internal — first step of a multi-step refactor on the multi-source download dispatcher. the orchestrator historically had 8 download sources (soulseek/youtube/tidal/qobuz/hifi/deezer/lidarr/soundcloud) hardcoded into 6+ different dispatch sites — `if username == "youtube" elif username == "tidal" elif ...` chains in `__init__`, search, download, get_all_downloads, cancel_download, etc. adding usenet (planned) would have meant 700+ lines of copy-paste across the same files. new `core/download_plugins/` package defines `DownloadSourcePlugin` (Protocol) — the canonical contract every source must satisfy: `is_configured`, `check_connection`, `search`, `download`, `get_all_downloads`, `get_download_status`, `cancel_download`, `clear_all_completed_downloads`. plus `DownloadPluginRegistry` — single source of truth for which sources exist, with name/alias resolution (legacy `deezer_dl` alias preserved). orchestrator now dispatches through the registry instead of hardcoded `[self.soulseek, self.youtube, ...]` lists; backward-compat `self.<source>` attributes still work so external callers reaching for source-specific internals (e.g. `orchestrator.soulseek._make_request`) keep working unchanged. zero behavior change for users — pure additive foundation that lets future PRs extract shared logic (background thread workers, search query normalization, post-processing context) into the contract instead of copy-pasted across all 8 sources. 19 new tests pin every plugin class\'s structural conformance to the contract — drift in any source will fail at the test boundary instead of at runtime against a live download.' },
{ title: 'Internal: Download Engine — Background Worker, State, Fallback', desc: 'internal — followup to the download source plugin contract. lifts the duplicated thread-spawn boilerplate, per-source active_downloads dicts, and hybrid-fallback dispatch into a central `core/download_engine/` package. each streaming source (youtube, tidal, qobuz, hifi, deezer, soundcloud) used to hand-roll the same ~70 LOC of background thread management — semaphore-gated serialization, rate-limit sleep between downloads, state-dict updates for InProgress/Completed/Errored transitions, exception capture. ~490 LOC of copy-paste across 7 files. all of it gone now — `engine.worker.dispatch(source, target_id, impl_callable, ...)` owns thread spawning + semaphore + delay + state lifecycle. plugins provide only `_download_sync(download_id, target_id, display_name) → file_path`, the source-specific atomic download. per-source rate-limit policy declared via `RateLimitPolicy` (concurrency, delay) — engine reads at register time. cross-source state queries (`get_all_downloads`, `get_download_status`, `cancel_download`, `clear_all_completed_downloads`) read engine state directly instead of iterating per-source dicts. hybrid-mode search now goes through `engine.search_with_fallback(chain)` — same ordering / skip-unconfigured / swallow-per-source-exceptions semantics as before. every per-source migration commit gated by phase A pinning tests (54 tests across all 8 sources) so contract drift fails fast at the test boundary instead of at runtime against a live download. net: ~700 LOC removed across 6 client files, ~85 new engine + worker + rate-limit tests, suite green at every commit. zero behavior change for users — same downloads, same lifecycle states, same hybrid mode. backward-compat preserved for everything that reaches into `orchestrator.soulseek._make_request` etc. adding usenet now = one new client class + one registry entry, no orchestrator changes. follow-up: cin\'s metadata engine work may shape further refactors (e.g. extracting search retry / quality filter — left per-source for now since search code is genuinely 90% source-specific).' },
{ title: 'Discogs Collection in "Your Albums"', desc: 'discord request: pull your discogs collection into the your albums section on discover, similar to spotify liked albums. set your discogs personal access token on settings → connections (already there from prior work) and add discogs as one of the configured sources via the gear button on your albums. background fetcher pulls your full collection (all folders, all pages — capped at 5000 releases), normalizes artist names (strips discogs `(N)` disambiguation suffix), dedupes against any spotify/tidal/deezer-saved versions of the same album. clicking a discogs-only album opens with discogs context — full release detail (year, format, label, country, tracklist) from the /releases endpoint. clicking an album that exists in both your spotify saved AND discogs collection prefers spotify (download flow is more direct). discogs is physical-media-first so many releases won\'t have streaming equivalents — those still show in the grid but the modal flow may need to fall back to a name search to find a downloadable digital version.', page: 'discover' },
{ title: 'Drop Redundant "Your Spotify Library" Section on Discover', desc: 'discover page used to show two near-identical sections: "Your Albums" (cross-source aggregator across spotify/deezer/etc) AND "Your Spotify Library" (spotify-only). same UI, same grid, same filter / sort / download-missing controls — the spotify-only one was a strict subset of what your albums already covers. removed it. spotify saved albums still surface via the your albums section with spotify as one of its configured sources (gear button → configure sources). backend collection / storage is unchanged — the watchlist scanner still populates the spotify_library_albums cache for your albums to read.', page: 'discover' },
{ title: 'Library Disk Usage on Stats Page', desc: 'discord request (samuel [KC]): show how much disk space the library takes. new card on stats → system statistics shows total bytes + per-format breakdown (FLAC vs MP3 vs M4A bars). data comes from `tracks.file_size` populated during deep scan from whatever the media server already returns (plex MediaPart.size, jellyfin MediaSources[].Size, navidrome song.size, soulsync standalone os.path.getsize) — zero filesystem walk overhead. existing libraries see "Run a Deep Scan to populate" until the next deep scan fills in sizes; partial coverage shown as "X tracks measured (+Y pending)". migration is additive (NULL on legacy rows) so upgrading users have nothing to do.', page: 'stats' },