Merge pull request #495 from Nezreka/refactor/download-source-plugins
Refactor/download source plugins
This commit is contained in:
commit
b53d6f1b95
64 changed files with 6291 additions and 1995 deletions
|
|
@ -121,7 +121,7 @@ def register_routes(bp):
|
|||
|
||||
try:
|
||||
from utils.async_helpers import run_async
|
||||
soulseek = current_app.soulsync.get("soulseek_client")
|
||||
soulseek = current_app.soulsync.get("download_orchestrator")
|
||||
if not soulseek:
|
||||
return api_error("NOT_AVAILABLE", "Soulseek client not configured.", 503)
|
||||
|
||||
|
|
@ -138,7 +138,7 @@ def register_routes(bp):
|
|||
"""Cancel all active downloads and clear completed ones."""
|
||||
try:
|
||||
from utils.async_helpers import run_async
|
||||
soulseek = current_app.soulsync.get("soulseek_client")
|
||||
soulseek = current_app.soulsync.get("download_orchestrator")
|
||||
if not soulseek:
|
||||
return api_error("NOT_AVAILABLE", "Soulseek client not configured.", 503)
|
||||
|
||||
|
|
|
|||
|
|
@ -104,7 +104,7 @@ def _run_search_and_download(request_id, query, notify_url):
|
|||
if request_id in _pending_requests:
|
||||
_pending_requests[request_id]['status'] = 'searching'
|
||||
|
||||
soulseek = current_app._get_current_object().soulsync.get('soulseek_client')
|
||||
soulseek = current_app._get_current_object().soulsync.get('download_orchestrator')
|
||||
if not soulseek:
|
||||
with _requests_lock:
|
||||
if request_id in _pending_requests:
|
||||
|
|
|
|||
|
|
@ -26,7 +26,7 @@ def register_routes(bp):
|
|||
spotify = ctx.get("spotify_client")
|
||||
spotify_ok = bool(spotify and spotify.is_authenticated())
|
||||
|
||||
soulseek = ctx.get("soulseek_client")
|
||||
soulseek = ctx.get("download_orchestrator")
|
||||
soulseek_ok = bool(soulseek)
|
||||
|
||||
hydrabase = ctx.get("hydrabase_client")
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
"""Service connection test — lifted from web_server.py.
|
||||
|
||||
The function body is byte-identical to the original. soulseek_client,
|
||||
The function body is byte-identical to the original. download_orchestrator,
|
||||
qobuz_enrichment_worker, hydrabase_client, docker_resolve_url, and
|
||||
docker_resolve_path are injected at runtime because they live in
|
||||
web_server.py and are constructed there.
|
||||
|
|
@ -27,7 +27,7 @@ def _get_metadata_fallback_source():
|
|||
|
||||
|
||||
# Injected at runtime via init().
|
||||
soulseek_client = None
|
||||
download_orchestrator = None
|
||||
qobuz_enrichment_worker = None
|
||||
hydrabase_client = None
|
||||
docker_resolve_url = None
|
||||
|
|
@ -35,16 +35,16 @@ docker_resolve_path = None
|
|||
|
||||
|
||||
def init(
|
||||
soulseek_client_obj,
|
||||
download_orchestrator_obj,
|
||||
qobuz_worker,
|
||||
hydrabase_client_obj,
|
||||
docker_resolve_url_fn,
|
||||
docker_resolve_path_fn,
|
||||
):
|
||||
"""Bind web_server-side helpers/globals so the lifted body can resolve them."""
|
||||
global soulseek_client, qobuz_enrichment_worker, hydrabase_client
|
||||
global download_orchestrator, qobuz_enrichment_worker, hydrabase_client
|
||||
global docker_resolve_url, docker_resolve_path
|
||||
soulseek_client = soulseek_client_obj
|
||||
download_orchestrator = download_orchestrator_obj
|
||||
qobuz_enrichment_worker = qobuz_worker
|
||||
hydrabase_client = hydrabase_client_obj
|
||||
docker_resolve_url = docker_resolve_url_fn
|
||||
|
|
@ -177,13 +177,13 @@ def run_service_test(service, test_config):
|
|||
else:
|
||||
return False, f"Output folder not found: {transfer_path}"
|
||||
elif service == "soulseek":
|
||||
if soulseek_client is None:
|
||||
if download_orchestrator is None:
|
||||
return False, "Download orchestrator failed to initialize. Check server logs for startup errors."
|
||||
|
||||
# Test the orchestrator's configured download source (not just Soulseek)
|
||||
download_mode = config_manager.get('download_source.mode', 'hybrid')
|
||||
|
||||
if run_async(soulseek_client.check_connection()):
|
||||
if run_async(download_orchestrator.check_connection()):
|
||||
# Success message based on active mode
|
||||
mode_messages = {
|
||||
'soulseek': "Successfully connected to Soulseek network via slskd.",
|
||||
|
|
|
|||
|
|
@ -64,7 +64,7 @@ download_batches = None
|
|||
sync_states = None
|
||||
youtube_playlist_states = None
|
||||
tidal_discovery_states = None
|
||||
soulseek_client = None
|
||||
download_orchestrator = None
|
||||
_log_path = None
|
||||
_log_dir = None
|
||||
app = None
|
||||
|
|
@ -80,7 +80,7 @@ def init(
|
|||
sync_states_dict,
|
||||
youtube_playlist_states_dict,
|
||||
tidal_discovery_states_dict,
|
||||
soulseek_client_obj,
|
||||
download_orchestrator_obj,
|
||||
log_path,
|
||||
log_dir,
|
||||
flask_app,
|
||||
|
|
@ -90,7 +90,7 @@ def init(
|
|||
"""Bind shared state/helpers from web_server."""
|
||||
global SOULSYNC_VERSION, _DIRECT_RUN, _status_cache, qobuz_enrichment_worker
|
||||
global download_batches, sync_states, youtube_playlist_states
|
||||
global tidal_discovery_states, soulseek_client, _log_path, _log_dir
|
||||
global tidal_discovery_states, download_orchestrator, _log_path, _log_dir
|
||||
global app, get_database, _get_tidal_client
|
||||
SOULSYNC_VERSION = soulsync_version
|
||||
_DIRECT_RUN = direct_run
|
||||
|
|
@ -100,7 +100,7 @@ def init(
|
|||
sync_states = sync_states_dict
|
||||
youtube_playlist_states = youtube_playlist_states_dict
|
||||
tidal_discovery_states = tidal_discovery_states_dict
|
||||
soulseek_client = soulseek_client_obj
|
||||
download_orchestrator = download_orchestrator_obj
|
||||
_log_path = log_path
|
||||
_log_dir = log_dir
|
||||
app = flask_app
|
||||
|
|
@ -292,9 +292,9 @@ def get_debug_info():
|
|||
|
||||
# Download client init failures
|
||||
info['download_client_failures'] = []
|
||||
if soulseek_client and hasattr(soulseek_client, '_init_failures'):
|
||||
info['download_client_failures'] = soulseek_client._init_failures
|
||||
elif not soulseek_client:
|
||||
if download_orchestrator and hasattr(download_orchestrator, '_init_failures'):
|
||||
info['download_client_failures'] = download_orchestrator._init_failures
|
||||
elif not download_orchestrator:
|
||||
info['download_client_failures'] = ['ALL (orchestrator failed to initialize)']
|
||||
|
||||
# API rate monitor — current calls/min, 24h totals, peaks, rate limit events
|
||||
|
|
|
|||
|
|
@ -20,7 +20,7 @@ from typing import Any, Dict, List, Optional, Tuple
|
|||
|
||||
import requests
|
||||
|
||||
from core.soulseek_client import AlbumResult, DownloadStatus, TrackResult
|
||||
from core.download_plugins.types import AlbumResult, DownloadStatus, TrackResult
|
||||
from utils.logging_config import get_logger
|
||||
|
||||
logger = get_logger("deezer_download")
|
||||
|
|
@ -79,7 +79,10 @@ def _decrypt_chunk(chunk: bytes, key: bytes) -> bytes:
|
|||
) from exc
|
||||
|
||||
|
||||
class DeezerDownloadClient:
|
||||
from core.download_plugins.base import DownloadSourcePlugin
|
||||
|
||||
|
||||
class DeezerDownloadClient(DownloadSourcePlugin):
|
||||
"""Deezer download client using ARL token authentication."""
|
||||
|
||||
def __init__(self, download_path: str = None):
|
||||
|
|
@ -91,9 +94,9 @@ class DeezerDownloadClient:
|
|||
self.download_path = Path(download_path)
|
||||
self.download_path.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# Download tracking (same pattern as Tidal/Qobuz/HiFi)
|
||||
self.active_downloads: Dict[str, Dict[str, Any]] = {}
|
||||
self._download_lock = threading.Lock()
|
||||
# Engine reference is populated by set_engine() at registration
|
||||
# time. None until orchestrator wires the registry.
|
||||
self._engine = None
|
||||
|
||||
# Shutdown check callback (set by web_server)
|
||||
self.shutdown_check = None
|
||||
|
|
@ -125,6 +128,10 @@ class DeezerDownloadClient:
|
|||
|
||||
logger.info(f"Deezer download client initialized (download path: {self.download_path})")
|
||||
|
||||
def set_engine(self, engine):
|
||||
"""Engine callback — wires the central thread worker + state store."""
|
||||
self._engine = engine
|
||||
|
||||
# ─── Authentication ──────────────────────────────────────────
|
||||
|
||||
def _authenticate(self, arl: str) -> bool:
|
||||
|
|
@ -605,87 +612,67 @@ class DeezerDownloadClient:
|
|||
if not self._authenticated:
|
||||
logger.error("Deezer not authenticated — cannot download")
|
||||
return None
|
||||
if self._engine is None:
|
||||
# Raise rather than return None so the orchestrator's
|
||||
# download_with_fallback surfaces a real warning + tries
|
||||
# the next source. Returning None silently dropped the
|
||||
# download with no user feedback (per JohnBaumb).
|
||||
raise RuntimeError("Deezer client has no engine reference — cannot dispatch download")
|
||||
|
||||
# Parse filename: "track_id||display_name"
|
||||
parts = filename.split('||', 1)
|
||||
track_id = parts[0]
|
||||
display_name = parts[1] if len(parts) > 1 else f"Track {track_id}"
|
||||
|
||||
download_id = str(uuid.uuid4())
|
||||
|
||||
with self._download_lock:
|
||||
self.active_downloads[download_id] = {
|
||||
'id': download_id,
|
||||
return self._engine.worker.dispatch(
|
||||
source_name='deezer',
|
||||
target_id=track_id,
|
||||
display_name=display_name,
|
||||
original_filename=filename,
|
||||
impl_callable=self._download_sync,
|
||||
extra_record_fields={
|
||||
'track_id': track_id,
|
||||
'display_name': display_name,
|
||||
'filename': filename,
|
||||
'username': 'deezer_dl',
|
||||
'state': 'Initializing',
|
||||
'progress': 0.0,
|
||||
'size': file_size,
|
||||
'transferred': 0,
|
||||
'speed': 0,
|
||||
'file_path': None,
|
||||
'error': None,
|
||||
}
|
||||
|
||||
thread = threading.Thread(
|
||||
target=self._download_thread_worker,
|
||||
args=(download_id, track_id, display_name),
|
||||
daemon=True,
|
||||
name=f'deezer-dl-{track_id}'
|
||||
},
|
||||
# Legacy username slot — frontend status indicators key off
|
||||
# ``deezer_dl``, not the canonical ``deezer``.
|
||||
username_override='deezer_dl',
|
||||
# Diagnostic thread name for multi-thread debugging.
|
||||
thread_name=f'deezer-dl-{track_id}',
|
||||
)
|
||||
thread.start()
|
||||
|
||||
logger.info(f"Started Deezer download {download_id}: {display_name}")
|
||||
return download_id
|
||||
def _set_error(self, download_id: str, message: str) -> None:
|
||||
"""Helper: set the engine record's `error` slot. No-op if
|
||||
engine isn't wired or record was already removed."""
|
||||
if self._engine is None:
|
||||
return
|
||||
self._engine.update_record('deezer', download_id, {'error': message})
|
||||
|
||||
def _download_thread_worker(self, download_id: str, track_id: str, display_name: str):
|
||||
"""Background worker for a single download."""
|
||||
try:
|
||||
result_path = self._download_sync(download_id, track_id, display_name)
|
||||
with self._download_lock:
|
||||
if download_id in self.active_downloads:
|
||||
dl = self.active_downloads[download_id]
|
||||
if dl['state'] == 'Cancelled':
|
||||
return
|
||||
if result_path:
|
||||
dl['state'] = 'Completed, Succeeded'
|
||||
dl['progress'] = 100.0
|
||||
dl['file_path'] = result_path
|
||||
logger.info(f"Deezer download {download_id} completed: {result_path}")
|
||||
else:
|
||||
dl['state'] = 'Errored'
|
||||
logger.error(f"Deezer download {download_id} failed: {dl.get('error', 'unknown')}")
|
||||
except Exception as e:
|
||||
logger.error(f"Deezer download thread error: {e}")
|
||||
with self._download_lock:
|
||||
if download_id in self.active_downloads:
|
||||
self.active_downloads[download_id]['state'] = 'Errored'
|
||||
self.active_downloads[download_id]['error'] = str(e)
|
||||
def _is_cancelled(self, download_id: str) -> bool:
|
||||
if self._engine is None:
|
||||
return False
|
||||
record = self._engine.get_record('deezer', download_id)
|
||||
return record is not None and record.get('state') == 'Cancelled'
|
||||
|
||||
def _download_sync(self, download_id: str, track_id: str, display_name: str) -> Optional[str]:
|
||||
"""Synchronous download: get URL, download, decrypt, save."""
|
||||
# Check for shutdown
|
||||
if self.shutdown_check and self.shutdown_check():
|
||||
with self._download_lock:
|
||||
if download_id in self.active_downloads:
|
||||
self.active_downloads[download_id]['state'] = 'Aborted'
|
||||
if self._engine is not None:
|
||||
self._engine.update_record('deezer', download_id, {'state': 'Aborted'})
|
||||
return None
|
||||
|
||||
# Get track data from private API
|
||||
track_data = self._get_track_data(track_id)
|
||||
if not track_data:
|
||||
with self._download_lock:
|
||||
if download_id in self.active_downloads:
|
||||
self.active_downloads[download_id]['error'] = 'Failed to get track data'
|
||||
self._set_error(download_id, 'Failed to get track data')
|
||||
return None
|
||||
|
||||
track_token = track_data.get('TRACK_TOKEN', '')
|
||||
if not track_token:
|
||||
with self._download_lock:
|
||||
if download_id in self.active_downloads:
|
||||
self.active_downloads[download_id]['error'] = 'No track token available'
|
||||
self._set_error(download_id, 'No track token available')
|
||||
return None
|
||||
|
||||
# Determine quality and get media URL with fallback
|
||||
|
|
@ -695,7 +682,6 @@ class DeezerDownloadClient:
|
|||
|
||||
if allow_fallback:
|
||||
quality_order = _QUALITY_ORDER.copy()
|
||||
# Start from user's preferred quality
|
||||
try:
|
||||
pref_idx = quality_order.index(self._quality)
|
||||
quality_order = quality_order[pref_idx:] + quality_order[:pref_idx]
|
||||
|
|
@ -712,27 +698,16 @@ class DeezerDownloadClient:
|
|||
break
|
||||
|
||||
if not media_url:
|
||||
with self._download_lock:
|
||||
if download_id in self.active_downloads:
|
||||
self.active_downloads[download_id]['error'] = 'No media URL available (may require higher subscription tier)'
|
||||
self._set_error(download_id, 'No media URL available (may require higher subscription tier)')
|
||||
return None
|
||||
|
||||
if actual_quality != self._quality:
|
||||
logger.info(f"Quality fallback: {self._quality} → {actual_quality} for {display_name}")
|
||||
|
||||
# Determine file extension
|
||||
ext = '.flac' if actual_quality == 'flac' else '.mp3'
|
||||
|
||||
# Sanitize filename
|
||||
safe_name = self._sanitize_filename(display_name)
|
||||
out_path = str(self.download_path / f"{safe_name}{ext}")
|
||||
|
||||
# Update state
|
||||
with self._download_lock:
|
||||
if download_id in self.active_downloads:
|
||||
dl = self.active_downloads[download_id]
|
||||
dl['state'] = 'InProgress, Downloading'
|
||||
|
||||
# Download and decrypt
|
||||
try:
|
||||
bf_key = _get_blowfish_key(track_id)
|
||||
|
|
@ -740,9 +715,8 @@ class DeezerDownloadClient:
|
|||
resp.raise_for_status()
|
||||
|
||||
total_size = int(resp.headers.get('content-length', 0))
|
||||
with self._download_lock:
|
||||
if download_id in self.active_downloads:
|
||||
self.active_downloads[download_id]['size'] = total_size
|
||||
if self._engine is not None:
|
||||
self._engine.update_record('deezer', download_id, {'size': total_size})
|
||||
|
||||
downloaded = 0
|
||||
chunk_index = 0
|
||||
|
|
@ -755,23 +729,20 @@ class DeezerDownloadClient:
|
|||
|
||||
# Check for cancellation/shutdown
|
||||
if self.shutdown_check and self.shutdown_check():
|
||||
with self._download_lock:
|
||||
if download_id in self.active_downloads:
|
||||
self.active_downloads[download_id]['state'] = 'Aborted'
|
||||
if self._engine is not None:
|
||||
self._engine.update_record('deezer', download_id, {'state': 'Aborted'})
|
||||
try:
|
||||
os.remove(out_path)
|
||||
except OSError:
|
||||
pass
|
||||
return None
|
||||
|
||||
with self._download_lock:
|
||||
if download_id in self.active_downloads:
|
||||
if self.active_downloads[download_id]['state'] == 'Cancelled':
|
||||
try:
|
||||
os.remove(out_path)
|
||||
except OSError:
|
||||
pass
|
||||
return None
|
||||
if self._is_cancelled(download_id):
|
||||
try:
|
||||
os.remove(out_path)
|
||||
except OSError:
|
||||
pass
|
||||
return None
|
||||
|
||||
# Decrypt every 3rd chunk (Deezer's encryption pattern)
|
||||
if chunk_index % 3 == 0 and len(raw_chunk) == _CHUNK_SIZE:
|
||||
|
|
@ -788,12 +759,12 @@ class DeezerDownloadClient:
|
|||
speed = int(downloaded / elapsed) if elapsed > 0 else 0
|
||||
progress = (downloaded / total_size * 100) if total_size > 0 else 0
|
||||
|
||||
with self._download_lock:
|
||||
if download_id in self.active_downloads:
|
||||
dl = self.active_downloads[download_id]
|
||||
dl['transferred'] = downloaded
|
||||
dl['progress'] = min(progress, 99.9)
|
||||
dl['speed'] = speed
|
||||
if self._engine is not None:
|
||||
self._engine.update_record('deezer', download_id, {
|
||||
'transferred': downloaded,
|
||||
'progress': min(progress, 99.9),
|
||||
'speed': speed,
|
||||
})
|
||||
|
||||
# Validate file size
|
||||
file_size = os.path.getsize(out_path)
|
||||
|
|
@ -803,9 +774,7 @@ class DeezerDownloadClient:
|
|||
os.remove(out_path)
|
||||
except OSError:
|
||||
pass
|
||||
with self._download_lock:
|
||||
if download_id in self.active_downloads:
|
||||
self.active_downloads[download_id]['error'] = f'File too small ({file_size} bytes)'
|
||||
self._set_error(download_id, f'File too small ({file_size} bytes)')
|
||||
return None
|
||||
|
||||
logger.info(f"Deezer download complete: {out_path} ({file_size / 1048576:.1f} MB, {actual_quality})")
|
||||
|
|
@ -817,59 +786,58 @@ class DeezerDownloadClient:
|
|||
os.remove(out_path)
|
||||
except OSError:
|
||||
pass
|
||||
with self._download_lock:
|
||||
if download_id in self.active_downloads:
|
||||
self.active_downloads[download_id]['error'] = str(e)
|
||||
self._set_error(download_id, str(e))
|
||||
return None
|
||||
|
||||
# ─── Download Status ─────────────────────────────────────────
|
||||
|
||||
def _record_to_status(self, record: dict) -> DownloadStatus:
|
||||
return DownloadStatus(
|
||||
id=record['id'],
|
||||
filename=record['filename'],
|
||||
username=record['username'],
|
||||
state=record['state'],
|
||||
progress=record['progress'],
|
||||
size=record.get('size', 0),
|
||||
transferred=record.get('transferred', 0),
|
||||
speed=record.get('speed', 0),
|
||||
file_path=record.get('file_path'),
|
||||
)
|
||||
|
||||
async def get_all_downloads(self) -> List[DownloadStatus]:
|
||||
"""Return all active downloads."""
|
||||
with self._download_lock:
|
||||
return [self._to_status(dl) for dl in self.active_downloads.values()]
|
||||
if self._engine is None:
|
||||
return []
|
||||
return [
|
||||
self._record_to_status(record)
|
||||
for record in self._engine.iter_records_for_source('deezer')
|
||||
]
|
||||
|
||||
async def get_download_status(self, download_id: str) -> Optional[DownloadStatus]:
|
||||
"""Get status of a specific download."""
|
||||
with self._download_lock:
|
||||
dl = self.active_downloads.get(download_id)
|
||||
return self._to_status(dl) if dl else None
|
||||
if self._engine is None:
|
||||
return None
|
||||
record = self._engine.get_record('deezer', download_id)
|
||||
return self._record_to_status(record) if record is not None else None
|
||||
|
||||
async def cancel_download(self, download_id: str, username: str = None,
|
||||
remove: bool = False) -> bool:
|
||||
"""Cancel a download."""
|
||||
with self._download_lock:
|
||||
dl = self.active_downloads.get(download_id)
|
||||
if not dl:
|
||||
return False
|
||||
dl['state'] = 'Cancelled'
|
||||
if remove:
|
||||
del self.active_downloads[download_id]
|
||||
if self._engine is None:
|
||||
return False
|
||||
if self._engine.get_record('deezer', download_id) is None:
|
||||
return False
|
||||
self._engine.update_record('deezer', download_id, {'state': 'Cancelled'})
|
||||
if remove:
|
||||
self._engine.remove_record('deezer', download_id)
|
||||
return True
|
||||
|
||||
async def clear_all_completed_downloads(self) -> bool:
|
||||
"""Remove all terminal downloads."""
|
||||
terminal_states = {'Completed, Succeeded', 'Cancelled', 'Errored', 'Aborted'}
|
||||
with self._download_lock:
|
||||
to_remove = [k for k, v in self.active_downloads.items() if v['state'] in terminal_states]
|
||||
for k in to_remove:
|
||||
del self.active_downloads[k]
|
||||
if self._engine is None:
|
||||
return True
|
||||
terminal = {'Completed, Succeeded', 'Cancelled', 'Errored', 'Aborted'}
|
||||
for record in list(self._engine.iter_records_for_source('deezer')):
|
||||
if record.get('state') in terminal:
|
||||
self._engine.remove_record('deezer', record['id'])
|
||||
return True
|
||||
|
||||
def _to_status(self, dl: dict) -> DownloadStatus:
|
||||
"""Convert internal dict to DownloadStatus."""
|
||||
return DownloadStatus(
|
||||
id=dl['id'],
|
||||
filename=dl['filename'],
|
||||
username=dl['username'],
|
||||
state=dl['state'],
|
||||
progress=dl['progress'],
|
||||
size=dl['size'],
|
||||
transferred=dl['transferred'],
|
||||
speed=dl['speed'],
|
||||
file_path=dl.get('file_path'),
|
||||
)
|
||||
|
||||
# ─── Utilities ───────────────────────────────────────────────
|
||||
|
||||
@staticmethod
|
||||
|
|
|
|||
31
core/download_engine/__init__.py
Normal file
31
core/download_engine/__init__.py
Normal file
|
|
@ -0,0 +1,31 @@
|
|||
"""Download Engine — central owner of cross-source download state,
|
||||
thread workers, search retry, rate-limits, and fallback chains.
|
||||
|
||||
This is the second leg of the multi-source download dispatcher
|
||||
refactor (the first leg, ``core/download_plugins/``, defined the
|
||||
contract). The engine takes ownership of everything that used to
|
||||
be duplicated across the per-source clients (background thread
|
||||
workers, active_downloads dicts, search retry ladders, quality
|
||||
filtering, hybrid fallback). Clients become DUMB — just hit the
|
||||
API for their source, manage their own auth state, and let the
|
||||
engine drive everything else.
|
||||
|
||||
This package is built up in phases (see
|
||||
``docs/download-engine-refactor-plan.md`` for the full plan):
|
||||
|
||||
- Phase B (current) — engine skeleton + state lift.
|
||||
- Phase C — background download worker.
|
||||
- Phase D — search retry + quality filter.
|
||||
- Phase E — rate-limit pool.
|
||||
- Phase F — fallback chain.
|
||||
|
||||
Each phase is purely additive at first (engine grows, clients
|
||||
unchanged). Migration to the new shape happens one source per
|
||||
commit so behavior never breaks across the suite.
|
||||
"""
|
||||
|
||||
from core.download_engine.engine import DownloadEngine
|
||||
from core.download_engine.rate_limit import RateLimitPolicy
|
||||
from core.download_engine.worker import BackgroundDownloadWorker
|
||||
|
||||
__all__ = ["DownloadEngine", "BackgroundDownloadWorker", "RateLimitPolicy"]
|
||||
457
core/download_engine/engine.py
Normal file
457
core/download_engine/engine.py
Normal file
|
|
@ -0,0 +1,457 @@
|
|||
"""DownloadEngine — central owner of cross-source download state.
|
||||
|
||||
Phase B scope: skeleton only. The engine exposes a place for
|
||||
plugins to register, a single ``active_downloads`` dict keyed by
|
||||
``(source, download_id)``, and per-source RLocks that guard mutations
|
||||
without serializing workers across different sources.
|
||||
|
||||
Subsequent phases bolt more capability on top:
|
||||
- ``dispatch_download(plugin, target_id)`` (Phase C — replaces every
|
||||
client's ``_download_thread_worker`` boilerplate).
|
||||
- ``search(query, source_chain)`` (Phase D — replaces every client's
|
||||
retry ladder + quality filter).
|
||||
- ``rate_limit.acquire(source)`` (Phase E — replaces every client's
|
||||
semaphore + last-download-timestamp dance).
|
||||
- ``search_with_fallback`` / ``download_with_fallback`` (Phase F —
|
||||
unifies hybrid mode across search and download).
|
||||
|
||||
The engine is constructed by ``DownloadOrchestrator.__init__`` and
|
||||
each plugin from the registry is registered with it. In Phase B
|
||||
nothing in the existing code paths goes through the engine yet —
|
||||
this commit is pure additive scaffolding so subsequent commits can
|
||||
introduce engine-driven behavior one piece at a time without a
|
||||
big-bang switchover.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import threading
|
||||
from typing import Any, Dict, Iterator, List, Optional, Tuple
|
||||
|
||||
from utils.logging_config import get_logger
|
||||
|
||||
logger = get_logger("download_engine")
|
||||
|
||||
|
||||
# Type alias for the per-download state dict. Today's clients each
|
||||
# define their own slightly-different shape (see Phase A pinning
|
||||
# tests); the engine stores them as opaque dicts and the per-plugin
|
||||
# accessor preserves the source-specific fields.
|
||||
DownloadRecord = Dict[str, Any]
|
||||
|
||||
|
||||
class DownloadEngine:
|
||||
"""Central state for every active download across every source.
|
||||
|
||||
State is keyed by ``(source_name, download_id)`` so the same
|
||||
UUID could hypothetically appear in two sources without
|
||||
collision (in practice each source generates its own UUID4
|
||||
so collisions are negligible — the source qualifier exists
|
||||
so the engine can answer "which plugin owns this download" in
|
||||
O(1) without iterating every plugin).
|
||||
|
||||
Thread safety: per-source lock sharding. Each source gets its own
|
||||
RLock — progress callbacks on Deezer don't block Tidal's worker
|
||||
and vice versa, matching the pre-refactor behavior where each
|
||||
client owned its own download lock. Read-only accessors
|
||||
(``get_record``, ``iter_records_for_source``) take the source's
|
||||
lock briefly and return a SHALLOW COPY so the caller can iterate
|
||||
without holding the lock. Callers that need to mutate a record
|
||||
should use ``update_record`` which takes the lock and applies the
|
||||
patch atomically.
|
||||
"""
|
||||
|
||||
def __init__(self) -> None:
|
||||
# Nested dict: source_name → {download_id → record}. Replaces
|
||||
# the original single-dict composite-key layout so
|
||||
# ``iter_records_for_source`` is O(source_records) instead of
|
||||
# O(total_records).
|
||||
self._records: Dict[str, Dict[str, DownloadRecord]] = {}
|
||||
# Per-source RLocks. Each source gets its own so progress
|
||||
# updates on one source never block writes on another. RLock
|
||||
# so a plugin's worker callback can re-enter while holding the
|
||||
# lock for its own update. Lazily created via ``_source_lock``;
|
||||
# the meta-lock guards creation against the create-race window
|
||||
# where two threads could both miss + both create.
|
||||
self._source_locks: Dict[str, threading.RLock] = {}
|
||||
self._source_locks_lock = threading.Lock()
|
||||
# Plugins that have registered with the engine. Source name
|
||||
# → plugin instance.
|
||||
self._plugins: Dict[str, Any] = {}
|
||||
# Alias → canonical-name map. Lets engine resolve legacy
|
||||
# source-name strings (e.g. ``'deezer_dl'`` for Deezer) to
|
||||
# the canonical key in ``_plugins``. Cin's review caught
|
||||
# that engine.cancel_download(source_hint='deezer_dl')
|
||||
# silently fell through to Soulseek because alias resolution
|
||||
# only existed at the registry, not on the engine.
|
||||
self._aliases: Dict[str, str] = {}
|
||||
# Background download worker — lives on the engine because
|
||||
# it owns the cross-source state the worker mutates. Lazy
|
||||
# import keeps the engine module standalone.
|
||||
from core.download_engine.worker import BackgroundDownloadWorker
|
||||
self.worker = BackgroundDownloadWorker(self)
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Plugin registration
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def register_plugin(self, source_name: str, plugin: Any,
|
||||
aliases: Tuple[str, ...] = ()) -> None:
|
||||
"""Register a plugin under its canonical source name. Called
|
||||
once per source by the orchestrator after the registry's
|
||||
``initialize`` builds the client instances.
|
||||
|
||||
``aliases`` is the list of legacy source-name strings that
|
||||
should resolve to this plugin (e.g. ``'deezer_dl'`` for
|
||||
Deezer). Without alias resolution the engine couldn't route
|
||||
cancel/lookup calls that came in with the legacy name.
|
||||
|
||||
If the plugin exposes ``set_engine(engine)``, the engine
|
||||
passes a self-reference so the plugin can dispatch into
|
||||
``engine.worker`` / read state / etc. Plugins that haven't
|
||||
been migrated to the engine yet simply don't define
|
||||
``set_engine`` — they keep their pre-engine behavior
|
||||
unchanged.
|
||||
|
||||
Also reads the plugin's declared ``RateLimitPolicy`` (via
|
||||
the ``rate_limit_policy()`` method or ``RATE_LIMIT_POLICY``
|
||||
class attribute) and applies it to the worker. Plugins that
|
||||
don't declare a policy get the conservative default
|
||||
(concurrency=1, delay=0).
|
||||
"""
|
||||
if source_name in self._plugins:
|
||||
logger.warning("Plugin %s already registered with engine — overwriting", source_name)
|
||||
self._plugins[source_name] = plugin
|
||||
for alias in aliases:
|
||||
self._aliases[alias] = source_name
|
||||
|
||||
# Apply the plugin's rate-limit policy BEFORE set_engine so
|
||||
# set_engine callbacks can override per-source if they need
|
||||
# config-driven values (e.g. YouTube's user-tunable delay).
|
||||
from core.download_engine.rate_limit import resolve_policy
|
||||
policy = resolve_policy(plugin)
|
||||
self.worker.set_concurrency(source_name, policy.download_concurrency)
|
||||
self.worker.set_delay(source_name, policy.download_delay_seconds)
|
||||
|
||||
set_engine = getattr(plugin, 'set_engine', None)
|
||||
if callable(set_engine):
|
||||
try:
|
||||
set_engine(self)
|
||||
except Exception as exc:
|
||||
logger.warning(
|
||||
"Plugin %s set_engine callback failed: %s", source_name, exc,
|
||||
)
|
||||
|
||||
def get_plugin(self, source_name: str) -> Optional[Any]:
|
||||
"""Return the plugin instance for the given source name.
|
||||
Resolves through aliases — e.g. ``get_plugin('deezer_dl')``
|
||||
returns the same instance as ``get_plugin('deezer')``."""
|
||||
if source_name in self._plugins:
|
||||
return self._plugins[source_name]
|
||||
canonical = self._aliases.get(source_name)
|
||||
if canonical:
|
||||
return self._plugins.get(canonical)
|
||||
return None
|
||||
|
||||
def _resolve_canonical(self, source_name: str) -> Optional[str]:
|
||||
"""Return the canonical source name for an input that may be
|
||||
an alias. Returns None if the input matches neither a
|
||||
canonical name nor an alias."""
|
||||
if source_name in self._plugins:
|
||||
return source_name
|
||||
return self._aliases.get(source_name)
|
||||
|
||||
def registered_sources(self) -> List[str]:
|
||||
return list(self._plugins.keys())
|
||||
|
||||
def _source_lock(self, source_name: str) -> threading.RLock:
|
||||
"""Return the per-source RLock, lazy-creating it on first use.
|
||||
The meta-lock around the cache lookup closes the create-race
|
||||
window where two threads both miss + both create a fresh lock.
|
||||
"""
|
||||
with self._source_locks_lock:
|
||||
lock = self._source_locks.get(source_name)
|
||||
if lock is None:
|
||||
lock = threading.RLock()
|
||||
self._source_locks[source_name] = lock
|
||||
return lock
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Active-downloads state — Phase B core surface
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def add_record(self, source_name: str, download_id: str, record: DownloadRecord) -> None:
|
||||
"""Insert a fresh download record. Used by clients (today
|
||||
directly via their own dicts; Phase B2 routes them through
|
||||
here)."""
|
||||
with self._source_lock(source_name):
|
||||
source_bucket = self._records.setdefault(source_name, {})
|
||||
if download_id in source_bucket:
|
||||
logger.warning("Replacing existing download record for %s/%s", source_name, download_id)
|
||||
source_bucket[download_id] = dict(record)
|
||||
|
||||
def update_record(self, source_name: str, download_id: str, patch: DownloadRecord) -> None:
|
||||
"""Apply a partial patch to an existing record. No-op if the
|
||||
record was already removed (e.g. cancelled mid-update)."""
|
||||
with self._source_lock(source_name):
|
||||
existing = self._records.get(source_name, {}).get(download_id)
|
||||
if existing is None:
|
||||
return
|
||||
existing.update(patch)
|
||||
|
||||
def update_record_unless_state(self, source_name: str, download_id: str,
|
||||
patch: DownloadRecord,
|
||||
skip_if_state_in: Tuple[str, ...] = ()) -> bool:
|
||||
"""Atomically check the record's state and apply ``patch`` only
|
||||
if the current state is NOT in ``skip_if_state_in``. Returns
|
||||
True if the patch was applied, False if it was skipped (or
|
||||
the record didn't exist).
|
||||
|
||||
Used by the background download worker's ``_mark_terminal``
|
||||
to avoid the read-then-write race Cin flagged: a cancel
|
||||
landing between the snapshot and update could be overwritten
|
||||
back to Errored / Completed. Holding the source's lock across
|
||||
the check + write closes the window.
|
||||
"""
|
||||
with self._source_lock(source_name):
|
||||
existing = self._records.get(source_name, {}).get(download_id)
|
||||
if existing is None:
|
||||
return False
|
||||
if existing.get('state') in skip_if_state_in:
|
||||
return False
|
||||
existing.update(patch)
|
||||
return True
|
||||
|
||||
def remove_record(self, source_name: str, download_id: str) -> Optional[DownloadRecord]:
|
||||
"""Delete a record (cancellation cleanup). Returns the
|
||||
removed record or None if not found."""
|
||||
with self._source_lock(source_name):
|
||||
source_bucket = self._records.get(source_name)
|
||||
if not source_bucket:
|
||||
return None
|
||||
removed = source_bucket.pop(download_id, None)
|
||||
# Drop the empty source bucket so iteration / membership
|
||||
# checks don't see a stale source key.
|
||||
if not source_bucket:
|
||||
self._records.pop(source_name, None)
|
||||
return removed
|
||||
|
||||
def get_record(self, source_name: str, download_id: str) -> Optional[DownloadRecord]:
|
||||
"""Return a SHALLOW COPY of the record. Caller mutations
|
||||
don't affect engine state — use ``update_record`` for that."""
|
||||
with self._source_lock(source_name):
|
||||
record = self._records.get(source_name, {}).get(download_id)
|
||||
return dict(record) if record is not None else None
|
||||
|
||||
def iter_records_for_source(self, source_name: str) -> Iterator[DownloadRecord]:
|
||||
"""Yield SHALLOW COPIES of every record owned by a source.
|
||||
Holds the source's lock briefly to snapshot, then yields
|
||||
outside the lock so callers can spend arbitrary time on each
|
||||
record.
|
||||
|
||||
With the nested-dict layout this is O(source_records) — only
|
||||
touches the bucket for the requested source, not every record
|
||||
across every source.
|
||||
"""
|
||||
with self._source_lock(source_name):
|
||||
source_bucket = self._records.get(source_name, {})
|
||||
snapshot = [dict(record) for record in source_bucket.values()]
|
||||
for record in snapshot:
|
||||
yield record
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Cross-source query dispatch — Phase B2 surface
|
||||
# ------------------------------------------------------------------
|
||||
#
|
||||
# The orchestrator historically iterated every plugin in its own
|
||||
# ``get_all_downloads`` / ``get_download_status`` / ``cancel_download``
|
||||
# methods (with hand-maintained client lists, before the registry
|
||||
# came along). That iteration logic moves into the engine here so
|
||||
# the orchestrator becomes a thin pass-through (Phase B3).
|
||||
#
|
||||
# In Phase B these methods iterate the registered plugins and call
|
||||
# their existing ``get_all_downloads`` / ``cancel_download``
|
||||
# methods — same behavior as today, just in a new home. Phase C/D
|
||||
# will replace plugin-iteration with direct engine-state queries
|
||||
# once the thread worker is also lifted.
|
||||
#
|
||||
# All methods are async to match the per-plugin contract.
|
||||
|
||||
async def get_all_downloads(self, exclude: Tuple[str, ...] = ()):
|
||||
"""Aggregated view across every registered plugin's active
|
||||
downloads. Per-plugin exceptions are swallowed (one source
|
||||
failing shouldn't take down cross-source aggregation) but
|
||||
logged at debug level — same defensive shape the legacy
|
||||
orchestrator had.
|
||||
|
||||
``exclude`` skips named sources entirely. The download monitor
|
||||
passes ``('soulseek',)`` so it doesn't double-fetch slskd
|
||||
transfers (it already pulled them via the slskd transfers
|
||||
endpoint earlier in the same loop).
|
||||
"""
|
||||
all_downloads = []
|
||||
for source_name, plugin in self._plugins.items():
|
||||
if plugin is None or source_name in exclude:
|
||||
continue
|
||||
try:
|
||||
all_downloads.extend(await plugin.get_all_downloads())
|
||||
except Exception as exc:
|
||||
logger.debug("%s get_all_downloads failed: %s", source_name, exc)
|
||||
return all_downloads
|
||||
|
||||
async def get_download_status(self, download_id: str):
|
||||
"""Find a download_id across every plugin. Returns the first
|
||||
plugin's response or None if no plugin owns it."""
|
||||
for source_name, plugin in self._plugins.items():
|
||||
if plugin is None:
|
||||
continue
|
||||
try:
|
||||
status = await plugin.get_download_status(download_id)
|
||||
if status:
|
||||
return status
|
||||
except Exception as exc:
|
||||
logger.debug("%s get_download_status failed: %s", source_name, exc)
|
||||
return None
|
||||
|
||||
async def cancel_download(self, download_id: str,
|
||||
source_hint: Optional[str] = None,
|
||||
remove: bool = False) -> bool:
|
||||
"""Cancel a download. ``source_hint`` is the source name (or
|
||||
legacy alias like ``'deezer_dl'``, or a real Soulseek peer
|
||||
username) — when provided, routes directly to that plugin.
|
||||
When omitted, every plugin is asked in turn until one accepts.
|
||||
|
||||
Cin's review caught a bug here: legacy alias strings like
|
||||
``'deezer_dl'`` weren't resolved to the canonical ``'deezer'``
|
||||
plugin name, so the cancel silently fell through to Soulseek.
|
||||
Resolution now goes through ``_resolve_canonical`` first.
|
||||
"""
|
||||
# Direct routing when the caller knows the source.
|
||||
if source_hint:
|
||||
canonical = self._resolve_canonical(source_hint)
|
||||
# Streaming source names (or aliases) resolve to a
|
||||
# registered plugin. Anything else (real Soulseek peer
|
||||
# name not in our registry) routes to Soulseek.
|
||||
if canonical and canonical != 'soulseek':
|
||||
target_plugin = self._plugins.get(canonical)
|
||||
if target_plugin is not None:
|
||||
try:
|
||||
return await target_plugin.cancel_download(
|
||||
download_id, source_hint, remove,
|
||||
)
|
||||
except Exception as exc:
|
||||
logger.debug("%s cancel_download failed: %s", canonical, exc)
|
||||
return False
|
||||
soulseek = self._plugins.get('soulseek')
|
||||
if soulseek is not None:
|
||||
try:
|
||||
return await soulseek.cancel_download(download_id, source_hint, remove)
|
||||
except Exception as exc:
|
||||
logger.debug("soulseek cancel_download failed: %s", exc)
|
||||
return False
|
||||
|
||||
# No hint → ask every plugin until one cancels successfully.
|
||||
for source_name, plugin in self._plugins.items():
|
||||
if plugin is None:
|
||||
continue
|
||||
try:
|
||||
if await plugin.cancel_download(download_id, source_hint, remove):
|
||||
return True
|
||||
except Exception as exc:
|
||||
logger.debug("%s cancel_download failed: %s", source_name, exc)
|
||||
return False
|
||||
|
||||
async def clear_all_completed_downloads(self) -> bool:
|
||||
"""Best-effort cleanup of every plugin's completed-downloads
|
||||
list. Skips plugins that report not-configured (saves API
|
||||
calls + log noise)."""
|
||||
results = []
|
||||
for source_name, plugin in self._plugins.items():
|
||||
if plugin is None:
|
||||
continue
|
||||
if hasattr(plugin, 'is_configured') and not plugin.is_configured():
|
||||
logger.debug("Skipping %s clear_all_completed_downloads (not configured)", source_name)
|
||||
continue
|
||||
try:
|
||||
results.append(await plugin.clear_all_completed_downloads())
|
||||
except Exception as exc:
|
||||
logger.warning("%s clear_all_completed_downloads failed: %s", source_name, exc)
|
||||
results.append(False)
|
||||
return all(results) if results else True
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Hybrid fallback — Phase F surface
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
async def search_with_fallback(self, query: str, source_chain,
|
||||
timeout=None, progress_callback=None):
|
||||
"""Try each source in ``source_chain`` until one returns
|
||||
tracks. Skips unconfigured / unregistered sources, swallows
|
||||
per-source exceptions. Returns the first non-empty
|
||||
(tracks, albums) tuple, or ``([], [])`` when every source
|
||||
in the chain is exhausted.
|
||||
|
||||
Replaces orchestrator's hand-rolled hybrid search loop. The
|
||||
chain is ordered (most-preferred first).
|
||||
"""
|
||||
for i, source_name in enumerate(source_chain):
|
||||
plugin = self._plugins.get(source_name)
|
||||
if plugin is None:
|
||||
logger.info(f"Skipping {source_name} (not available)")
|
||||
continue
|
||||
if hasattr(plugin, 'is_configured') and not plugin.is_configured():
|
||||
logger.info(f"Skipping {source_name} (not configured)")
|
||||
continue
|
||||
|
||||
try:
|
||||
logger.info(f"Trying {source_name} (priority {i+1}): {query}")
|
||||
tracks, albums = await plugin.search(query, timeout, progress_callback)
|
||||
if tracks:
|
||||
logger.info(f"{source_name} found {len(tracks)} tracks")
|
||||
return (tracks, albums)
|
||||
except Exception as e:
|
||||
logger.warning(f"{source_name} search failed: {e}")
|
||||
|
||||
logger.warning(
|
||||
"Hybrid search: all sources (%s) found nothing for: %s",
|
||||
', '.join(source_chain), query,
|
||||
)
|
||||
return ([], [])
|
||||
|
||||
async def download_with_fallback(self, username: str, filename: str,
|
||||
file_size: int, source_chain) -> Optional[str]:
|
||||
"""Try each source in ``source_chain`` until one accepts the
|
||||
download (returns a non-None download_id). Fixes the legacy
|
||||
bug where hybrid mode silently routed to a single source via
|
||||
the username hint with no retry on failure.
|
||||
|
||||
``username`` is treated as a hint when it matches a source
|
||||
name in the chain — that source is tried FIRST regardless of
|
||||
chain order. Anything else (e.g. a real Soulseek peer name)
|
||||
routes through the chain in declared order.
|
||||
"""
|
||||
# Promote a matching source-name hint to the head of the chain.
|
||||
ordered_chain = list(source_chain)
|
||||
if username and username in ordered_chain:
|
||||
ordered_chain.remove(username)
|
||||
ordered_chain.insert(0, username)
|
||||
|
||||
for source_name in ordered_chain:
|
||||
plugin = self._plugins.get(source_name)
|
||||
if plugin is None:
|
||||
continue
|
||||
if hasattr(plugin, 'is_configured') and not plugin.is_configured():
|
||||
continue
|
||||
try:
|
||||
download_id = await plugin.download(username, filename, file_size)
|
||||
if download_id is not None:
|
||||
return download_id
|
||||
logger.info(f"{source_name} declined download — trying next in chain")
|
||||
except Exception as e:
|
||||
logger.warning(f"{source_name} download raised — trying next in chain: {e}")
|
||||
|
||||
logger.warning(
|
||||
"Hybrid download: every source in chain (%s) refused %r",
|
||||
', '.join(ordered_chain), filename,
|
||||
)
|
||||
return None
|
||||
73
core/download_engine/rate_limit.py
Normal file
73
core/download_engine/rate_limit.py
Normal file
|
|
@ -0,0 +1,73 @@
|
|||
"""Per-source rate-limit policy declarations.
|
||||
|
||||
Today's per-source download throttling is scattered:
|
||||
|
||||
- YouTube: ``self._download_delay = config_manager.get('youtube.download_delay', 3)``
|
||||
set in ``__init__``, applied in ``set_engine`` via worker.set_delay.
|
||||
- Qobuz: module-level ``_qobuz_api_lock`` + ``_QOBUZ_MIN_INTERVAL`` for
|
||||
search-side throttling, no download-side throttle.
|
||||
- Other sources: no explicit declarations — default to 0s delay /
|
||||
concurrency=1, which works because the streaming APIs have their
|
||||
own gateway-level rate limits.
|
||||
|
||||
Phase E centralizes this into one place: each plugin declares a
|
||||
``RateLimitPolicy`` (either as a class attribute or returned from a
|
||||
``rate_limit_policy()`` method), and the engine reads + applies the
|
||||
policy to ``engine.worker`` at ``register_plugin`` time.
|
||||
|
||||
Adding a new source = declaring its policy alongside the rest of
|
||||
the source's auth/config — no longer a hidden line in __init__ or a
|
||||
module-level constant in the client file.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class RateLimitPolicy:
|
||||
"""Per-source download throttling policy.
|
||||
|
||||
Attributes:
|
||||
download_concurrency: Max number of concurrent downloads
|
||||
from this source. Default 1 (serial). Most streaming
|
||||
APIs prefer serial transfers because parallel just
|
||||
trades rate-limit errors for thread overhead.
|
||||
download_delay_seconds: Minimum gap between successive
|
||||
downloads from this source. YouTube uses 3s today
|
||||
(legacy ``_download_delay`` config key) to avoid
|
||||
yt-dlp 429s. Most other sources use 0.
|
||||
"""
|
||||
|
||||
download_concurrency: int = 1
|
||||
download_delay_seconds: float = 0.0
|
||||
|
||||
|
||||
# Sentinel default — most plugins want this. Plugins that need
|
||||
# tighter throttling override by exposing ``RATE_LIMIT_POLICY`` as
|
||||
# a class attribute or returning a custom one from
|
||||
# ``rate_limit_policy()``.
|
||||
DEFAULT_POLICY = RateLimitPolicy()
|
||||
|
||||
|
||||
def resolve_policy(plugin) -> RateLimitPolicy:
|
||||
"""Read a plugin's declared rate-limit policy. Checks (in order):
|
||||
1. ``plugin.rate_limit_policy()`` method (returns a RateLimitPolicy)
|
||||
2. ``plugin.RATE_LIMIT_POLICY`` class attribute
|
||||
3. ``DEFAULT_POLICY``
|
||||
"""
|
||||
method = getattr(plugin, 'rate_limit_policy', None)
|
||||
if callable(method):
|
||||
try:
|
||||
policy = method()
|
||||
if isinstance(policy, RateLimitPolicy):
|
||||
return policy
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
declared = getattr(plugin, 'RATE_LIMIT_POLICY', None)
|
||||
if isinstance(declared, RateLimitPolicy):
|
||||
return declared
|
||||
|
||||
return DEFAULT_POLICY
|
||||
315
core/download_engine/worker.py
Normal file
315
core/download_engine/worker.py
Normal file
|
|
@ -0,0 +1,315 @@
|
|||
"""BackgroundDownloadWorker — engine-owned thread spawning + state
|
||||
lifecycle for downloads.
|
||||
|
||||
Today every streaming download client (YouTube, Tidal, Qobuz, HiFi,
|
||||
Deezer, SoundCloud) hand-rolls the same thread-spawn pattern:
|
||||
|
||||
```python
|
||||
async def download(self, ...):
|
||||
download_id = str(uuid.uuid4())
|
||||
with self._download_lock:
|
||||
self.active_downloads[download_id] = {...initial state...}
|
||||
threading.Thread(
|
||||
target=self._download_thread_worker,
|
||||
args=(download_id, target_id, display_name, ...),
|
||||
daemon=True,
|
||||
).start()
|
||||
return download_id
|
||||
|
||||
def _download_thread_worker(self, download_id, target_id, display_name, ...):
|
||||
with self._download_semaphore:
|
||||
# rate-limit sleep
|
||||
# update state to 'InProgress, Downloading'
|
||||
file_path = self._download_sync(...) # the source-specific atomic op
|
||||
# update state to 'Completed, Succeeded' / 'Errored'
|
||||
```
|
||||
|
||||
That pattern is duplicated 6+ times across the codebase (~70 LOC
|
||||
each, ~490 total). The worker class lifts it into the engine — each
|
||||
plugin only has to provide the atomic op (``impl_callable``) and
|
||||
declare its rate-limit policy. Adding a new download source becomes
|
||||
a much smaller patch.
|
||||
|
||||
Phase C1 scope: introduce the worker. No client migrated yet — the
|
||||
worker just exists for C2–C7 to migrate sources one at a time, each
|
||||
under a passing pinning test.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import threading
|
||||
import time
|
||||
import uuid
|
||||
from typing import Any, Callable, Dict, Optional
|
||||
|
||||
from utils.logging_config import get_logger
|
||||
|
||||
logger = get_logger("download_engine.worker")
|
||||
|
||||
|
||||
# Type aliases for clarity. ``ImplCallable`` is the per-plugin
|
||||
# atomic download operation — synchronous, returns a file path on
|
||||
# success or raises (or returns None) on failure.
|
||||
ImplCallable = Callable[[str, Any, str], Optional[str]]
|
||||
|
||||
|
||||
class BackgroundDownloadWorker:
|
||||
"""Engine-owned thread spawner for per-source downloads.
|
||||
|
||||
State-machine semantics (preserved verbatim from the legacy
|
||||
per-client workers so consumers reading these fields keep
|
||||
working):
|
||||
|
||||
- ``Initializing`` — set on dispatch, before the thread starts.
|
||||
- ``InProgress, Downloading`` — set when the worker thread
|
||||
acquires the semaphore and is about to call the impl.
|
||||
- ``Completed, Succeeded`` — set when impl returns a non-None
|
||||
file path. ``progress=100.0`` and ``file_path=<the path>``
|
||||
also written.
|
||||
- ``Errored`` — set when impl returns None OR raises. The
|
||||
record is left in place so downstream consumers can inspect
|
||||
what failed.
|
||||
|
||||
Per-source serialization: each source gets a ``threading.Semaphore``
|
||||
(default size 1, configurable per-source via ``set_concurrency``).
|
||||
Same shape the existing clients use today (each source defines
|
||||
its own semaphore). Engine owning them centrally lets a future
|
||||
Phase E rate-limiter swap the semaphore for a smarter pool.
|
||||
|
||||
Per-source delay-between-downloads: default 0 seconds (most
|
||||
sources don't need it). YouTube currently uses 3s, Qobuz uses
|
||||
1s — the legacy values get configured in via ``set_delay``
|
||||
when the source registers.
|
||||
"""
|
||||
|
||||
def __init__(self, engine: Any) -> None:
|
||||
self._engine = engine
|
||||
# Per-source semaphores + delay state. The first dispatch
|
||||
# for a source auto-creates a semaphore with concurrency=1
|
||||
# if the source hasn't been configured explicitly.
|
||||
self._semaphores: Dict[str, threading.Semaphore] = {}
|
||||
self._delays: Dict[str, float] = {}
|
||||
self._last_download_at: Dict[str, float] = {}
|
||||
self._config_lock = threading.Lock()
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Per-source rate-limit configuration
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def set_concurrency(self, source_name: str, max_concurrent: int) -> None:
|
||||
"""Set the max number of concurrent downloads for a source.
|
||||
Default is 1 (serial). Most sources will keep the default —
|
||||
the streaming APIs all rate-limit at the API gateway level
|
||||
anyway, parallel downloads just trade rate-limit errors for
|
||||
thread overhead."""
|
||||
with self._config_lock:
|
||||
self._semaphores[source_name] = threading.Semaphore(max_concurrent)
|
||||
|
||||
def set_delay(self, source_name: str, seconds: float) -> None:
|
||||
"""Set a minimum delay between successive downloads from the
|
||||
same source. YouTube uses 3s today (avoid yt-dlp 429s),
|
||||
Qobuz uses 1s. Other sources use 0 (no delay)."""
|
||||
with self._config_lock:
|
||||
self._delays[source_name] = float(seconds)
|
||||
|
||||
def _get_semaphore(self, source_name: str) -> threading.Semaphore:
|
||||
with self._config_lock:
|
||||
sem = self._semaphores.get(source_name)
|
||||
if sem is None:
|
||||
sem = threading.Semaphore(1)
|
||||
self._semaphores[source_name] = sem
|
||||
return sem
|
||||
|
||||
def _get_delay(self, source_name: str) -> float:
|
||||
with self._config_lock:
|
||||
return self._delays.get(source_name, 0.0)
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Dispatch — public API
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def dispatch(
|
||||
self,
|
||||
source_name: str,
|
||||
target_id: Any,
|
||||
display_name: str,
|
||||
original_filename: str,
|
||||
impl_callable: ImplCallable,
|
||||
extra_record_fields: Optional[Dict[str, Any]] = None,
|
||||
username_override: Optional[str] = None,
|
||||
thread_name: Optional[str] = None,
|
||||
) -> str:
|
||||
"""Kick off a background download.
|
||||
|
||||
Args:
|
||||
source_name: Canonical source name (e.g. 'youtube',
|
||||
'tidal'). Used as the engine state key + the
|
||||
username slot in the record (unless overridden).
|
||||
target_id: Source-specific identifier (track_id, video_id,
|
||||
permalink_url, album_foreign_id, etc.). Passed
|
||||
verbatim to ``impl_callable``.
|
||||
display_name: Human-readable label for logs / UI.
|
||||
original_filename: The encoded filename the orchestrator
|
||||
received (e.g. ``'12345||Song Title'``). Stored in
|
||||
the record's ``filename`` slot for context-key lookups.
|
||||
impl_callable: Synchronous function that performs the
|
||||
actual download. Signature:
|
||||
``impl_callable(download_id, target_id, display_name) -> Optional[str]``.
|
||||
Returns the final file path on success or None /
|
||||
raises on failure.
|
||||
extra_record_fields: Per-source extras to merge into the
|
||||
initial record (e.g. ``{'video_id': '...', 'url':
|
||||
'...', 'title': '...'}`` for YouTube). Used to
|
||||
preserve source-specific slots that downstream
|
||||
consumers + status APIs read.
|
||||
username_override: Use this instead of ``source_name``
|
||||
in the record's ``username`` slot. Required for
|
||||
Deezer (legacy ``'deezer_dl'``) — every other source
|
||||
uses the canonical name.
|
||||
thread_name: Optional thread name for diagnostics. Deezer
|
||||
uses ``'deezer-dl-<track_id>'`` — Phase A pinning
|
||||
tests catch any drift in this convention.
|
||||
|
||||
Returns:
|
||||
download_id (UUID4 string). The orchestrator polls via
|
||||
``engine.get_download_status(download_id)`` for progress.
|
||||
"""
|
||||
download_id = str(uuid.uuid4())
|
||||
|
||||
record: Dict[str, Any] = {
|
||||
'id': download_id,
|
||||
'filename': original_filename,
|
||||
'username': username_override or source_name,
|
||||
'state': 'Initializing',
|
||||
'progress': 0.0,
|
||||
'size': 0,
|
||||
'transferred': 0,
|
||||
'speed': 0,
|
||||
'time_remaining': None,
|
||||
'file_path': None,
|
||||
}
|
||||
if extra_record_fields:
|
||||
record.update(extra_record_fields)
|
||||
|
||||
self._engine.add_record(source_name, download_id, record)
|
||||
|
||||
thread = threading.Thread(
|
||||
target=self._worker_loop,
|
||||
args=(source_name, download_id, target_id, display_name, impl_callable),
|
||||
daemon=True,
|
||||
name=thread_name,
|
||||
)
|
||||
thread.start()
|
||||
|
||||
return download_id
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Worker thread — the lifted boilerplate
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def _worker_loop(
|
||||
self,
|
||||
source_name: str,
|
||||
download_id: str,
|
||||
target_id: Any,
|
||||
display_name: str,
|
||||
impl_callable: ImplCallable,
|
||||
) -> None:
|
||||
"""Runs on the spawned daemon thread. Handles semaphore
|
||||
acquisition, rate-limit sleep, state lifecycle, exception
|
||||
capture. The plugin-specific work happens entirely inside
|
||||
``impl_callable``."""
|
||||
try:
|
||||
with self._get_semaphore(source_name):
|
||||
# Rate-limit delay against the LAST download from
|
||||
# this source (not just this worker — semaphore
|
||||
# ensures serial access while delay is configured).
|
||||
delay = self._get_delay(source_name)
|
||||
if delay > 0:
|
||||
last_at = self._last_download_at.get(source_name, 0.0)
|
||||
elapsed = time.time() - last_at
|
||||
if last_at > 0 and elapsed < delay:
|
||||
wait_time = delay - elapsed
|
||||
logger.info(
|
||||
"Rate-limit delay for %s: waiting %.1fs before next download",
|
||||
source_name, wait_time,
|
||||
)
|
||||
time.sleep(wait_time)
|
||||
|
||||
self._engine.update_record(source_name, download_id, {
|
||||
'state': 'InProgress, Downloading',
|
||||
})
|
||||
|
||||
try:
|
||||
file_path = impl_callable(download_id, target_id, display_name)
|
||||
except Exception as exc:
|
||||
logger.error(
|
||||
"%s download %s failed (impl raised): %s",
|
||||
source_name, download_id, exc,
|
||||
)
|
||||
self._mark_terminal(
|
||||
source_name, download_id,
|
||||
success=False, error=str(exc),
|
||||
)
|
||||
return
|
||||
|
||||
self._last_download_at[source_name] = time.time()
|
||||
|
||||
if file_path:
|
||||
# Atomic write — preserve Cancelled if user cancelled
|
||||
# between impl returning and this write. Same guard
|
||||
# _mark_terminal uses; Cin flagged both split sites.
|
||||
self._engine.update_record_unless_state(
|
||||
source_name, download_id,
|
||||
{
|
||||
'state': 'Completed, Succeeded',
|
||||
'progress': 100.0,
|
||||
'file_path': file_path,
|
||||
},
|
||||
skip_if_state_in=('Cancelled',),
|
||||
)
|
||||
logger.info(
|
||||
"%s download %s completed: %s",
|
||||
source_name, download_id, file_path,
|
||||
)
|
||||
else:
|
||||
self._mark_terminal(source_name, download_id, success=False)
|
||||
logger.error(
|
||||
"%s download %s failed (impl returned None)",
|
||||
source_name, download_id,
|
||||
)
|
||||
|
||||
except Exception as exc:
|
||||
# Defensive — semaphore / sleep shouldn't blow up the
|
||||
# thread, but if they do the record needs SOME terminal
|
||||
# state or it sits at 'Initializing' forever.
|
||||
logger.exception(
|
||||
"%s worker_loop crashed for download %s: %s",
|
||||
source_name, download_id, exc,
|
||||
)
|
||||
self._mark_terminal(
|
||||
source_name, download_id,
|
||||
success=False, error=f'worker crash: {exc}',
|
||||
)
|
||||
|
||||
def _mark_terminal(self, source_name: str, download_id: str,
|
||||
success: bool, error: Optional[str] = None) -> None:
|
||||
"""Write a terminal state, but DON'T clobber an explicit
|
||||
'Cancelled' state set by the user via cancel_download.
|
||||
Mirrors the legacy per-client guard
|
||||
(``if state != 'Cancelled': state = 'Errored'``) every
|
||||
client used to hand-roll inside its thread worker.
|
||||
|
||||
Uses ``update_record_unless_state`` so the check + write are
|
||||
atomic under the engine's per-source lock. Cin caught a race
|
||||
where a cancel landing between the read-snapshot + write
|
||||
could overwrite Cancelled back to Errored / Completed.
|
||||
"""
|
||||
patch: Dict[str, Any] = {
|
||||
'state': 'Completed, Succeeded' if success else 'Errored',
|
||||
}
|
||||
if error is not None:
|
||||
patch['error'] = error
|
||||
self._engine.update_record_unless_state(
|
||||
source_name, download_id, patch, skip_if_state_in=('Cancelled',),
|
||||
)
|
||||
|
|
@ -11,6 +11,12 @@ Supports eight modes:
|
|||
- Deezer Only: Deezer downloads via ARL authentication
|
||||
- SoundCloud Only: Anonymous SoundCloud downloads (DJ mixes, removed/exclusive tracks)
|
||||
- Hybrid: Try primary source first, fallback to others
|
||||
|
||||
The orchestrator dispatches through ``core.download_plugins.registry``
|
||||
instead of hardcoded per-source ``[self.soulseek, self.youtube, ...]``
|
||||
lists. External callers reach individual clients via the generic
|
||||
``orchestrator.client('<name>')`` accessor (alias-aware), not direct
|
||||
attribute access.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
|
|
@ -19,14 +25,9 @@ from pathlib import Path
|
|||
|
||||
from utils.logging_config import get_logger
|
||||
from config.settings import config_manager
|
||||
from core.soulseek_client import SoulseekClient, TrackResult, AlbumResult, DownloadStatus
|
||||
from core.youtube_client import YouTubeClient
|
||||
from core.tidal_download_client import TidalDownloadClient
|
||||
from core.qobuz_client import QobuzClient
|
||||
from core.hifi_client import HiFiClient
|
||||
from core.deezer_download_client import DeezerDownloadClient
|
||||
from core.lidarr_download_client import LidarrDownloadClient
|
||||
from core.soundcloud_client import SoundcloudClient
|
||||
from core.download_engine import DownloadEngine
|
||||
from core.download_plugins.registry import DownloadPluginRegistry, build_default_registry
|
||||
from core.download_plugins.types import TrackResult, AlbumResult, DownloadStatus
|
||||
|
||||
logger = get_logger("download_orchestrator")
|
||||
|
||||
|
|
@ -39,19 +40,31 @@ class DownloadOrchestrator:
|
|||
Routes requests to the appropriate client(s) based on configured mode.
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
"""Initialize orchestrator with all clients.
|
||||
Each client is initialized independently — one failing client doesn't prevent others from working."""
|
||||
self._init_failures = []
|
||||
def __init__(self, registry: Optional[DownloadPluginRegistry] = None,
|
||||
engine: Optional[DownloadEngine] = None):
|
||||
"""Initialize orchestrator with a plugin registry. Each plugin
|
||||
is built and registered independently — one failing plugin
|
||||
doesn't prevent others from working. The ``registry`` arg
|
||||
exists so tests can inject a registry with mock plugins; in
|
||||
production callers leave it None and get the default.
|
||||
|
||||
self.soulseek = self._safe_init('Soulseek', SoulseekClient)
|
||||
self.youtube = self._safe_init('YouTube', YouTubeClient)
|
||||
self.tidal = self._safe_init('Tidal', TidalDownloadClient)
|
||||
self.qobuz = self._safe_init('Qobuz', QobuzClient)
|
||||
self.hifi = self._safe_init('HiFi', HiFiClient)
|
||||
self.deezer_dl = self._safe_init('Deezer', DeezerDownloadClient)
|
||||
self.lidarr = self._safe_init('Lidarr', LidarrDownloadClient)
|
||||
self.soundcloud = self._safe_init('SoundCloud', SoundcloudClient)
|
||||
``engine`` is the cross-source state owner. Phase B introduces
|
||||
it as a held reference; it isn't on any code path yet — Phase
|
||||
C/D/E/F migrate behavior into it incrementally.
|
||||
"""
|
||||
self.registry = registry if registry is not None else build_default_registry()
|
||||
self.registry.initialize()
|
||||
self._init_failures = self.registry.init_failures
|
||||
|
||||
# Engine — owns cross-source state, threading, search retry,
|
||||
# rate-limits, fallback. Built in subsequent phases. For Phase
|
||||
# B it's just an empty registry of plugins so future phases
|
||||
# can route through it without further orchestrator changes.
|
||||
self.engine = engine if engine is not None else DownloadEngine()
|
||||
for source_name, plugin in self.registry.all_plugins():
|
||||
spec = self.registry.get_spec(source_name)
|
||||
aliases = spec.aliases if spec else ()
|
||||
self.engine.register_plugin(source_name, plugin, aliases=aliases)
|
||||
|
||||
if self._init_failures:
|
||||
logger.warning(f"Download clients failed to initialize: {', '.join(self._init_failures)}")
|
||||
|
|
@ -71,15 +84,6 @@ class DownloadOrchestrator:
|
|||
self.hybrid_secondary,
|
||||
)
|
||||
|
||||
def _safe_init(self, name, cls):
|
||||
"""Initialize a download client, returning None on failure instead of crashing."""
|
||||
try:
|
||||
return cls()
|
||||
except Exception as e:
|
||||
logger.error(f"{name} download client failed to initialize: {e}")
|
||||
self._init_failures.append(name)
|
||||
return None
|
||||
|
||||
def reload_settings(self):
|
||||
"""Reload settings from config (call after settings change)"""
|
||||
self.mode = config_manager.get('download_source.mode', 'soulseek')
|
||||
|
|
@ -88,20 +92,28 @@ class DownloadOrchestrator:
|
|||
self.hybrid_order = config_manager.get('download_source.hybrid_order', ['hifi', 'youtube', 'soulseek'])
|
||||
|
||||
# Reload underlying client configs (SLSKD URL, API key, etc.)
|
||||
if self.soulseek:
|
||||
self.soulseek._setup_client()
|
||||
soulseek = self.client('soulseek')
|
||||
if soulseek:
|
||||
soulseek._setup_client()
|
||||
logger.info("Soulseek client config reloaded")
|
||||
|
||||
# Reconnect Deezer if ARL changed
|
||||
deezer_arl = config_manager.get('deezer_download.arl', '')
|
||||
if deezer_arl and self.deezer_dl:
|
||||
self.deezer_dl.reconnect(deezer_arl)
|
||||
self.deezer_dl._quality = config_manager.get('deezer_download.quality', 'flac')
|
||||
deezer_dl = self.client('deezer_dl')
|
||||
if deezer_arl and deezer_dl:
|
||||
deezer_dl.reconnect(deezer_arl)
|
||||
deezer_dl._quality = config_manager.get('deezer_download.quality', 'flac')
|
||||
|
||||
# Reload download path for all clients that cache it
|
||||
# Reload download path for all clients that cache it.
|
||||
# Soulseek owns the path config and is reloaded above; every
|
||||
# other source mirrors that path so files all land in one
|
||||
# tree. Sources without a `download_path` attribute (e.g.
|
||||
# Lidarr — pulls into Lidarr's own tree) silently skip.
|
||||
new_path = Path(config_manager.get('soulseek.download_path', './downloads'))
|
||||
for client in [self.youtube, self.tidal, self.qobuz, self.hifi, self.deezer_dl, self.soundcloud]:
|
||||
if client and hasattr(client, 'download_path') and client.download_path != new_path:
|
||||
for name, client in self.registry.all_plugins():
|
||||
if name == 'soulseek':
|
||||
continue
|
||||
if hasattr(client, 'download_path') and client.download_path != new_path:
|
||||
client.download_path = new_path
|
||||
client.download_path.mkdir(parents=True, exist_ok=True)
|
||||
# YouTube also caches path in yt-dlp opts
|
||||
|
|
@ -111,11 +123,61 @@ class DownloadOrchestrator:
|
|||
|
||||
logger.info(f"Download Orchestrator settings reloaded - Mode: {self.mode}")
|
||||
|
||||
def _client(self, name):
|
||||
"""Get a client by name, returning None if not initialized."""
|
||||
return {'soulseek': self.soulseek, 'youtube': self.youtube, 'tidal': self.tidal,
|
||||
'qobuz': self.qobuz, 'hifi': self.hifi, 'deezer_dl': self.deezer_dl,
|
||||
'lidarr': self.lidarr, 'soundcloud': self.soundcloud}.get(name)
|
||||
def client(self, name):
|
||||
"""Generic accessor for a download source client by name.
|
||||
|
||||
Cin's review feedback: external callers should reach into
|
||||
per-source clients via this method (``orch.client('hifi')``)
|
||||
instead of attribute access (``orch.hifi``). Resolves both
|
||||
canonical names (``deezer``) and legacy aliases (``deezer_dl``)
|
||||
via the registry. Returns None if the source isn't registered
|
||||
or failed to initialize.
|
||||
"""
|
||||
return self.registry.get(name)
|
||||
|
||||
# Internal alias kept for legacy callers inside this file.
|
||||
_client = client
|
||||
|
||||
def configured_clients(self) -> dict:
|
||||
"""Return ``{source_name: client}`` for every download source
|
||||
that's both initialized AND reports is_configured() == True.
|
||||
|
||||
Replaces the legacy per-source iteration pattern Cin called
|
||||
out — `if hasattr(orch, 'soulseek') and orch.soulseek and
|
||||
orch.soulseek.is_configured(): download_clients['soulseek']
|
||||
= orch.soulseek` repeated for each source.
|
||||
"""
|
||||
result = {}
|
||||
for name, client in self.registry.all_plugins():
|
||||
try:
|
||||
if not hasattr(client, 'is_configured') or client.is_configured():
|
||||
result[name] = client
|
||||
except Exception as exc:
|
||||
logger.debug("%s is_configured raised: %s", name, exc)
|
||||
return result
|
||||
|
||||
def reload_instances(self, source: str = None) -> bool:
|
||||
"""Reload a source's instance config (e.g. HiFi instance list,
|
||||
Qobuz session restore). Generic dispatch — caller passes the
|
||||
source name instead of reaching for ``orch.hifi.reload_instances()``.
|
||||
|
||||
When ``source`` is None, reloads every source that has a
|
||||
``reload_instances`` method.
|
||||
"""
|
||||
sources = [source] if source else list(self.registry.names())
|
||||
ok = True
|
||||
for name in sources:
|
||||
client = self.client(name)
|
||||
if client is None:
|
||||
continue
|
||||
if not hasattr(client, 'reload_instances'):
|
||||
continue
|
||||
try:
|
||||
client.reload_instances()
|
||||
except Exception as exc:
|
||||
logger.warning("%s reload_instances failed: %s", name, exc)
|
||||
ok = False
|
||||
return ok
|
||||
|
||||
def is_configured(self) -> bool:
|
||||
"""
|
||||
|
|
@ -132,12 +194,18 @@ class DownloadOrchestrator:
|
|||
return False
|
||||
|
||||
def get_source_status(self) -> dict:
|
||||
"""Return configured status for each download source."""
|
||||
return {name: (c.is_configured() if c else False)
|
||||
for name, c in [('soulseek', self.soulseek), ('youtube', self.youtube),
|
||||
('tidal', self.tidal), ('qobuz', self.qobuz),
|
||||
('hifi', self.hifi), ('deezer_dl', self.deezer_dl),
|
||||
('lidarr', self.lidarr), ('soundcloud', self.soundcloud)]}
|
||||
"""Return configured status for each download source.
|
||||
|
||||
Keys preserve the legacy ``deezer_dl`` alias used by the
|
||||
frontend status indicators and per-source dispatch strings,
|
||||
so callers reading specific keys keep working unchanged.
|
||||
"""
|
||||
status = {}
|
||||
for name in self.registry.names():
|
||||
client = self.registry.get(name)
|
||||
key = 'deezer_dl' if name == 'deezer' else name
|
||||
status[key] = client.is_configured() if client else False
|
||||
return status
|
||||
|
||||
async def check_connection(self) -> bool:
|
||||
"""
|
||||
|
|
@ -149,7 +217,7 @@ class DownloadOrchestrator:
|
|||
if client and self.mode != 'hybrid':
|
||||
return await client.check_connection()
|
||||
elif self.mode == 'hybrid':
|
||||
sources_to_check = self.hybrid_order if self.hybrid_order else ['soulseek', 'youtube', 'tidal', 'qobuz', 'hifi', 'deezer_dl', 'lidarr', 'soundcloud']
|
||||
sources_to_check = self.hybrid_order if self.hybrid_order else self.registry.names()
|
||||
results = {}
|
||||
for source in sources_to_check:
|
||||
client = self._client(source)
|
||||
|
|
@ -168,77 +236,62 @@ class DownloadOrchestrator:
|
|||
|
||||
return False
|
||||
|
||||
def _normalize_source_name(self, name: str) -> Optional[str]:
|
||||
"""Convert a possibly-aliased source name (e.g. legacy
|
||||
``'deezer_dl'``) to the canonical registry name (``'deezer'``).
|
||||
Returns None if the input matches neither a canonical name
|
||||
nor an alias.
|
||||
|
||||
Cin's review caught a bug where legacy alias values from
|
||||
config (hybrid_order containing ``'deezer_dl'``) silently
|
||||
dropped Deezer from hybrid mode because the canonical-name
|
||||
membership check rejected the alias.
|
||||
"""
|
||||
spec = self.registry.get_spec(name) if name else None
|
||||
return spec.name if spec else None
|
||||
|
||||
def _resolve_source_chain(self) -> List[str]:
|
||||
"""Order the configured sources for hybrid mode. Prefers
|
||||
``hybrid_order`` config; falls back to legacy
|
||||
primary/secondary pair when no order set. Normalizes alias
|
||||
names through the registry so legacy ``deezer_dl`` config
|
||||
values resolve correctly to the canonical ``deezer`` plugin."""
|
||||
if self.hybrid_order:
|
||||
chain = []
|
||||
seen = set()
|
||||
for raw in self.hybrid_order:
|
||||
canonical = self._normalize_source_name(raw)
|
||||
if canonical and canonical not in seen:
|
||||
chain.append(canonical)
|
||||
seen.add(canonical)
|
||||
return chain
|
||||
primary = self._normalize_source_name(self.hybrid_primary) or 'soulseek'
|
||||
secondary = self._normalize_source_name(self.hybrid_secondary) or 'soulseek'
|
||||
if secondary == primary:
|
||||
secondary = next(
|
||||
(name for name in self.registry.names() if name != primary),
|
||||
'soulseek',
|
||||
)
|
||||
chain = [primary, secondary]
|
||||
if not chain:
|
||||
chain = ['soulseek']
|
||||
return chain
|
||||
|
||||
async def search(self, query: str, timeout: int = None, progress_callback=None) -> Tuple[List[TrackResult], List[AlbumResult]]:
|
||||
"""
|
||||
Search for tracks using configured source(s).
|
||||
|
||||
Args:
|
||||
query: Search query
|
||||
timeout: Search timeout (for Soulseek)
|
||||
progress_callback: Progress callback (for Soulseek)
|
||||
|
||||
Returns:
|
||||
Tuple of (track_results, album_results)
|
||||
"""
|
||||
source_names = {'soulseek': 'Soulseek', 'youtube': 'YouTube', 'tidal': 'Tidal',
|
||||
'qobuz': 'Qobuz', 'hifi': 'HiFi', 'deezer_dl': 'Deezer', 'lidarr': 'Lidarr',
|
||||
'soundcloud': 'SoundCloud'}
|
||||
|
||||
"""Search for tracks using configured source(s). Single-source
|
||||
modes route directly; hybrid mode delegates to
|
||||
``engine.search_with_fallback`` which tries the chain in order."""
|
||||
if self.mode != 'hybrid':
|
||||
client = self._client(self.mode)
|
||||
if not client:
|
||||
logger.error(f"{source_names.get(self.mode, self.mode)} client not available (failed to initialize)")
|
||||
logger.error(f"{self.registry.display_name(self.mode)} client not available (failed to initialize)")
|
||||
return [], []
|
||||
logger.info(f"Searching {source_names.get(self.mode, self.mode)}: {query}")
|
||||
logger.info(f"Searching {self.registry.display_name(self.mode)}: {query}")
|
||||
return await client.search(query, timeout, progress_callback)
|
||||
|
||||
elif self.mode == 'hybrid':
|
||||
clients = {name: self._client(name) for name in source_names}
|
||||
|
||||
# Build ordered source list: prefer hybrid_order, fall back to legacy primary/secondary
|
||||
if self.hybrid_order:
|
||||
source_order = [s for s in self.hybrid_order if s in clients]
|
||||
else:
|
||||
primary = self.hybrid_primary if self.hybrid_primary in clients else 'soulseek'
|
||||
secondary = self.hybrid_secondary if self.hybrid_secondary in clients else 'soulseek'
|
||||
if secondary == primary:
|
||||
secondary = next((name for name in clients if name != primary), 'soulseek')
|
||||
source_order = [primary, secondary]
|
||||
|
||||
if not source_order:
|
||||
source_order = ['soulseek']
|
||||
|
||||
logger.info(f"Hybrid search ({' → '.join(source_order)}): {query}")
|
||||
|
||||
# Try each source in priority order (skip unconfigured/unavailable ones)
|
||||
for i, source_name in enumerate(source_order):
|
||||
client = clients.get(source_name)
|
||||
if not client:
|
||||
logger.info(f"Skipping {source_name} (not available)")
|
||||
continue
|
||||
if hasattr(client, 'is_configured') and not client.is_configured():
|
||||
logger.info(f"Skipping {source_name} (not configured)")
|
||||
continue
|
||||
|
||||
try:
|
||||
if i == 0:
|
||||
logger.info(f"Trying {source_name} (priority {i+1}): {query}")
|
||||
else:
|
||||
logger.info(f"Trying {source_name} (priority {i+1}): {query}")
|
||||
|
||||
tracks, albums = await client.search(query, timeout, progress_callback)
|
||||
if tracks:
|
||||
logger.info(f"{source_name} found {len(tracks)} tracks")
|
||||
return (tracks, albums)
|
||||
except Exception as e:
|
||||
logger.warning(f"{source_name} search failed: {e}")
|
||||
|
||||
# Nothing found from any source
|
||||
logger.warning(f"Hybrid search: all sources ({', '.join(source_order)}) found nothing for: {query}")
|
||||
return ([], [])
|
||||
|
||||
# Fallback: empty results
|
||||
return ([], [])
|
||||
chain = self._resolve_source_chain()
|
||||
logger.info(f"Hybrid search ({' → '.join(chain)}): {query}")
|
||||
return await self.engine.search_with_fallback(query, chain, timeout, progress_callback)
|
||||
|
||||
async def search_and_download_best(self, query: str, expected_track=None) -> Optional[str]:
|
||||
"""
|
||||
|
|
@ -316,7 +369,8 @@ class DownloadOrchestrator:
|
|||
elif is_streaming:
|
||||
filtered_results = tracks
|
||||
else:
|
||||
filtered_results = self.soulseek.filter_results_by_quality_preference(tracks) if self.soulseek else tracks
|
||||
soulseek = self.client('soulseek')
|
||||
filtered_results = soulseek.filter_results_by_quality_preference(tracks) if soulseek else tracks
|
||||
|
||||
if not filtered_results:
|
||||
logger.warning(f"No suitable quality results found for: {query}")
|
||||
|
|
@ -339,105 +393,47 @@ class DownloadOrchestrator:
|
|||
Download a track using the appropriate client.
|
||||
|
||||
Args:
|
||||
username: Username (or "youtube" for YouTube)
|
||||
filename: Filename or YouTube video ID
|
||||
username: Source-name string for streaming sources
|
||||
(e.g. ``'youtube'``, ``'tidal'``, ``'deezer_dl'``)
|
||||
OR the actual slskd peer username for Soulseek.
|
||||
filename: Filename / video ID / track ID encoding (source-specific)
|
||||
file_size: File size estimate
|
||||
|
||||
Returns:
|
||||
download_id: Unique download ID for tracking
|
||||
"""
|
||||
# Detect which client to use based on username
|
||||
source_map = {'youtube': self.youtube, 'tidal': self.tidal, 'qobuz': self.qobuz,
|
||||
'hifi': self.hifi, 'deezer_dl': self.deezer_dl, 'lidarr': self.lidarr,
|
||||
'soundcloud': self.soundcloud}
|
||||
source_names = {'youtube': 'YouTube', 'tidal': 'Tidal', 'qobuz': 'Qobuz',
|
||||
'hifi': 'HiFi', 'deezer_dl': 'Deezer', 'lidarr': 'Lidarr',
|
||||
'soundcloud': 'SoundCloud'}
|
||||
|
||||
if username in source_map:
|
||||
client = source_map[username]
|
||||
# Streaming sources are dispatched by name match; anything
|
||||
# unrecognized falls through to Soulseek (peer username case).
|
||||
spec = self.registry.get_spec(username) if username else None
|
||||
if spec is not None and spec.name != 'soulseek':
|
||||
client = self.registry.get(spec.name)
|
||||
if not client:
|
||||
raise RuntimeError(f"{source_names[username]} download client not available (failed to initialize)")
|
||||
logger.info(f"Downloading from {source_names[username]}: {filename}")
|
||||
raise RuntimeError(f"{spec.display_name} download client not available (failed to initialize)")
|
||||
logger.info(f"Downloading from {spec.display_name}: {filename}")
|
||||
return await client.download(username, filename, file_size)
|
||||
else:
|
||||
if not self.soulseek:
|
||||
raise RuntimeError("Soulseek client not available (failed to initialize)")
|
||||
logger.info(f"Downloading from Soulseek: {filename}")
|
||||
return await self.soulseek.download(username, filename, file_size)
|
||||
|
||||
soulseek = self.registry.get('soulseek')
|
||||
if not soulseek:
|
||||
raise RuntimeError("Soulseek client not available (failed to initialize)")
|
||||
logger.info(f"Downloading from Soulseek: {filename}")
|
||||
return await soulseek.download(username, filename, file_size)
|
||||
|
||||
async def get_all_downloads(self) -> List[DownloadStatus]:
|
||||
"""
|
||||
Get all active downloads from all sources.
|
||||
|
||||
Returns:
|
||||
List of DownloadStatus objects
|
||||
"""
|
||||
# Get downloads from all available sources
|
||||
all_downloads = []
|
||||
for client in [self.soulseek, self.youtube, self.tidal, self.qobuz, self.hifi, self.deezer_dl, self.lidarr, self.soundcloud]:
|
||||
if client:
|
||||
try:
|
||||
all_downloads.extend(await client.get_all_downloads())
|
||||
except Exception:
|
||||
pass
|
||||
return all_downloads
|
||||
"""Aggregated view across every source. Delegates to the
|
||||
engine, which iterates registered plugins."""
|
||||
return await self.engine.get_all_downloads()
|
||||
|
||||
async def get_download_status(self, download_id: str) -> Optional[DownloadStatus]:
|
||||
"""
|
||||
Get status of a specific download.
|
||||
|
||||
Args:
|
||||
download_id: Download ID to query
|
||||
|
||||
Returns:
|
||||
DownloadStatus object or None if not found
|
||||
"""
|
||||
# Try each source until we find the download
|
||||
for client in [self.soulseek, self.youtube, self.tidal, self.qobuz, self.hifi, self.deezer_dl, self.lidarr, self.soundcloud]:
|
||||
if not client:
|
||||
continue
|
||||
try:
|
||||
status = await client.get_download_status(download_id)
|
||||
if status:
|
||||
return status
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
return None
|
||||
"""Find a download by id across every source. Delegates to
|
||||
the engine."""
|
||||
return await self.engine.get_download_status(download_id)
|
||||
|
||||
async def cancel_download(self, download_id: str, username: str = None, remove: bool = False) -> bool:
|
||||
"""
|
||||
Cancel an active download.
|
||||
|
||||
Args:
|
||||
download_id: Download ID to cancel
|
||||
username: Username hint (optional)
|
||||
remove: Whether to remove from active downloads
|
||||
|
||||
Returns:
|
||||
True if cancelled successfully
|
||||
"""
|
||||
# If username is provided, route directly to that source
|
||||
source_map = {'youtube': self.youtube, 'tidal': self.tidal, 'qobuz': self.qobuz,
|
||||
'hifi': self.hifi, 'deezer_dl': self.deezer_dl, 'lidarr': self.lidarr,
|
||||
'soundcloud': self.soundcloud}
|
||||
if username in source_map:
|
||||
client = source_map[username]
|
||||
return await client.cancel_download(download_id, username, remove) if client else False
|
||||
elif username:
|
||||
return await self.soulseek.cancel_download(download_id, username, remove) if self.soulseek else False
|
||||
|
||||
# Otherwise, try all available sources
|
||||
for client in [self.soulseek, self.youtube, self.tidal, self.qobuz, self.hifi, self.deezer_dl, self.lidarr, self.soundcloud]:
|
||||
if not client:
|
||||
continue
|
||||
try:
|
||||
if await client.cancel_download(download_id, username, remove):
|
||||
return True
|
||||
except Exception:
|
||||
pass
|
||||
return False
|
||||
"""Cancel an active download. Delegates to the engine, which
|
||||
handles source-hint routing (streaming source name → direct
|
||||
plugin, unknown name → Soulseek as peer username, no hint →
|
||||
try every plugin)."""
|
||||
return await self.engine.cancel_download(download_id, username, remove)
|
||||
|
||||
async def signal_download_completion(self, download_id: str, username: str, remove: bool = True) -> bool:
|
||||
"""
|
||||
|
|
@ -452,40 +448,16 @@ class DownloadOrchestrator:
|
|||
True if successful
|
||||
"""
|
||||
# This is Soulseek-specific, so only call on Soulseek client
|
||||
if not self.soulseek:
|
||||
soulseek = self.client('soulseek')
|
||||
if not soulseek:
|
||||
return False
|
||||
return await self.soulseek.signal_download_completion(download_id, username, remove)
|
||||
return await soulseek.signal_download_completion(download_id, username, remove)
|
||||
|
||||
async def clear_all_completed_downloads(self) -> bool:
|
||||
"""
|
||||
Clear all completed downloads from both sources.
|
||||
|
||||
Returns:
|
||||
True if successful
|
||||
"""
|
||||
results = []
|
||||
for name, client in [
|
||||
("soulseek", self.soulseek),
|
||||
("youtube", self.youtube),
|
||||
("tidal", self.tidal),
|
||||
("qobuz", self.qobuz),
|
||||
("hifi", self.hifi),
|
||||
("deezer_dl", self.deezer_dl),
|
||||
("lidarr", self.lidarr),
|
||||
("soundcloud", self.soundcloud),
|
||||
]:
|
||||
if not client:
|
||||
continue
|
||||
if hasattr(client, "is_configured") and not client.is_configured():
|
||||
logger.debug("Skipping %s clear_all_completed_downloads (not configured)", name)
|
||||
continue
|
||||
try:
|
||||
results.append(await client.clear_all_completed_downloads())
|
||||
except Exception as exc:
|
||||
logger.warning("%s clear_all_completed_downloads failed: %s", name, exc)
|
||||
results.append(False)
|
||||
|
||||
return all(results) if results else True
|
||||
"""Clear completed downloads from every source. Delegates
|
||||
to the engine, which skips unconfigured plugins and treats
|
||||
per-plugin failures as False (not an exception)."""
|
||||
return await self.engine.clear_all_completed_downloads()
|
||||
|
||||
# ===== Soulseek-specific methods (for backwards compatibility) =====
|
||||
# These are internal methods that some parts of the codebase use directly
|
||||
|
|
@ -503,9 +475,10 @@ class DownloadOrchestrator:
|
|||
Returns:
|
||||
API response
|
||||
"""
|
||||
if not self.soulseek:
|
||||
soulseek = self.client('soulseek')
|
||||
if not soulseek:
|
||||
raise RuntimeError("Soulseek client not available (failed to initialize)")
|
||||
return await self.soulseek._make_request(method, endpoint, **kwargs)
|
||||
return await soulseek._make_request(method, endpoint, **kwargs)
|
||||
|
||||
async def _make_direct_request(self, method: str, endpoint: str, **kwargs):
|
||||
"""
|
||||
|
|
@ -520,9 +493,10 @@ class DownloadOrchestrator:
|
|||
Returns:
|
||||
API response
|
||||
"""
|
||||
if not self.soulseek:
|
||||
soulseek = self.client('soulseek')
|
||||
if not soulseek:
|
||||
raise RuntimeError("Soulseek client not available (failed to initialize)")
|
||||
return await self.soulseek._make_direct_request(method, endpoint, **kwargs)
|
||||
return await soulseek._make_direct_request(method, endpoint, **kwargs)
|
||||
|
||||
async def clear_all_searches(self) -> bool:
|
||||
"""
|
||||
|
|
@ -531,7 +505,8 @@ class DownloadOrchestrator:
|
|||
Returns:
|
||||
True if successful
|
||||
"""
|
||||
return await self.soulseek.clear_all_searches() if self.soulseek else True
|
||||
soulseek = self.client('soulseek')
|
||||
return await soulseek.clear_all_searches() if soulseek else True
|
||||
|
||||
async def maintain_search_history_with_buffer(self, keep_searches: int = 50, trigger_threshold: int = 200) -> bool:
|
||||
"""
|
||||
|
|
@ -544,15 +519,54 @@ class DownloadOrchestrator:
|
|||
Returns:
|
||||
True if successful
|
||||
"""
|
||||
return await self.soulseek.maintain_search_history_with_buffer(keep_searches, trigger_threshold) if self.soulseek else True
|
||||
soulseek = self.client('soulseek')
|
||||
return await soulseek.maintain_search_history_with_buffer(keep_searches, trigger_threshold) if soulseek else True
|
||||
|
||||
async def cancel_all_downloads(self) -> bool:
|
||||
"""Cancel and remove all downloads from all sources."""
|
||||
"""Cancel and remove all downloads from all sources.
|
||||
|
||||
Note: YouTube is intentionally excluded from this loop in the
|
||||
legacy implementation — preserved here. (yt-dlp downloads
|
||||
run as detached subprocesses and don't share the
|
||||
``cancel_all_downloads`` semantics the streaming sources
|
||||
use.) Sources without ``cancel_all_downloads`` fall back to
|
||||
``clear_all_completed_downloads``.
|
||||
"""
|
||||
ok = True
|
||||
for client in [self.soulseek, self.tidal, self.qobuz, self.hifi, self.deezer_dl, self.lidarr, self.soundcloud]:
|
||||
if client:
|
||||
try:
|
||||
await client.cancel_all_downloads() if hasattr(client, 'cancel_all_downloads') else await client.clear_all_completed_downloads()
|
||||
except Exception:
|
||||
ok = False
|
||||
for name, client in self.registry.all_plugins():
|
||||
if name == 'youtube':
|
||||
continue
|
||||
try:
|
||||
await client.cancel_all_downloads() if hasattr(client, 'cancel_all_downloads') else await client.clear_all_completed_downloads()
|
||||
except Exception:
|
||||
ok = False
|
||||
return ok
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Singleton accessor — mirrors Cin's metadata engine pattern
|
||||
# (``get_metadata_engine()``). Callers that don't need a custom
|
||||
# registry use this instead of instantiating DownloadOrchestrator
|
||||
# directly. web_server.py constructs the singleton at startup and
|
||||
# exposes it via the ``download_orchestrator`` global.
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
_default_orchestrator: Optional['DownloadOrchestrator'] = None
|
||||
|
||||
|
||||
def get_download_orchestrator() -> 'DownloadOrchestrator':
|
||||
"""Return (lazily creating) the process-wide DownloadOrchestrator
|
||||
singleton. Mirrors the ``get_metadata_engine()`` pattern Cin used
|
||||
for the metadata engine refactor."""
|
||||
global _default_orchestrator
|
||||
if _default_orchestrator is None:
|
||||
_default_orchestrator = DownloadOrchestrator()
|
||||
return _default_orchestrator
|
||||
|
||||
|
||||
def set_download_orchestrator(orchestrator: 'DownloadOrchestrator') -> None:
|
||||
"""Set the process-wide singleton. Used by web_server.py at boot
|
||||
to install the orchestrator it constructs as the default for
|
||||
callers that grab via ``get_download_orchestrator()``."""
|
||||
global _default_orchestrator
|
||||
_default_orchestrator = orchestrator
|
||||
|
|
|
|||
34
core/download_plugins/__init__.py
Normal file
34
core/download_plugins/__init__.py
Normal file
|
|
@ -0,0 +1,34 @@
|
|||
"""Download source plugin contract + registry.
|
||||
|
||||
This package defines the canonical interface every download source
|
||||
(Soulseek, YouTube, Tidal, Qobuz, HiFi, Deezer, Lidarr, SoundCloud,
|
||||
and future additions like Usenet) must satisfy. The orchestrator
|
||||
dispatches through this contract instead of hardcoded
|
||||
`if self.youtube ... elif self.tidal ...` chains.
|
||||
|
||||
This is the foundation step of a multi-commit refactor. Subsequent
|
||||
commits extract shared logic (background download worker, search
|
||||
query normalization, post-processing context building) into the
|
||||
contract so adding a new source becomes a one-class plugin instead
|
||||
of a 700+ LOC copy-paste loop.
|
||||
|
||||
See `core/download_plugins/base.py` for the protocol contract and
|
||||
`core/download_plugins/registry.py` for the dispatch entry point.
|
||||
"""
|
||||
|
||||
from core.download_plugins.base import DownloadSourcePlugin
|
||||
|
||||
# NOTE: DownloadPluginRegistry is intentionally NOT re-exported here.
|
||||
# Importing the registry triggers eager imports of every client class
|
||||
# (see registry.py for why eager — test fixtures inject mock modules
|
||||
# at collection time and we need real bindings before that). Clients
|
||||
# inherit from DownloadSourcePlugin (Cin's review feedback — visible
|
||||
# contract conformance), so importing the package via ``from
|
||||
# core.download_plugins import DownloadSourcePlugin`` from a client
|
||||
# file would create a circular import if registry came along for the
|
||||
# ride. Callers that need the registry import it directly:
|
||||
# from core.download_plugins.registry import DownloadPluginRegistry
|
||||
|
||||
__all__ = [
|
||||
"DownloadSourcePlugin",
|
||||
]
|
||||
133
core/download_plugins/base.py
Normal file
133
core/download_plugins/base.py
Normal file
|
|
@ -0,0 +1,133 @@
|
|||
"""Canonical contract every download source plugin must satisfy.
|
||||
|
||||
`DownloadSourcePlugin` is a structural Protocol — any class that
|
||||
implements these methods with matching signatures is automatically
|
||||
treated as a download source. No inheritance required, no manual
|
||||
registration required beyond the registry entry.
|
||||
|
||||
The protocol is intentionally narrow — only the methods the
|
||||
orchestrator dispatches generically across all sources. Source-
|
||||
specific extras (Soulseek's slskd internals, Lidarr's album-only
|
||||
flow, etc.) stay on the underlying client and are accessed through
|
||||
the registry's typed accessor.
|
||||
|
||||
This file is the FOUNDATION step. Existing client classes
|
||||
(SoulseekClient, YouTubeClient, TidalDownloadClient, etc.) already
|
||||
conform structurally — they grew the same shape independently
|
||||
because every consumer site needed the same calls. This file just
|
||||
makes the implicit contract explicit so:
|
||||
|
||||
- Type checkers can flag drift if a new source forgets a method.
|
||||
- The orchestrator can iterate plugins generically instead of
|
||||
hardcoding `[self.soulseek, self.youtube, ...]` everywhere.
|
||||
- Future PRs can move shared logic INTO the contract (a base
|
||||
class with default implementations) without changing the
|
||||
signature surface every consumer already depends on.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import TYPE_CHECKING, List, Optional, Protocol, Tuple, runtime_checkable
|
||||
|
||||
# Soulseek client owns the canonical TrackResult / AlbumResult /
|
||||
# DownloadStatus dataclasses — every other source already imports
|
||||
# from there. We only need them for type annotations on this
|
||||
# protocol; using TYPE_CHECKING avoids a circular import once the
|
||||
# clients themselves inherit from DownloadSourcePlugin (Cin's
|
||||
# review feedback — clients explicitly declare conformance instead
|
||||
# of relying on structural typing).
|
||||
if TYPE_CHECKING:
|
||||
from core.download_plugins.types import AlbumResult, DownloadStatus, TrackResult
|
||||
|
||||
|
||||
@runtime_checkable
|
||||
class DownloadSourcePlugin(Protocol):
|
||||
"""Structural contract for a download source.
|
||||
|
||||
`runtime_checkable` lets `isinstance(client, DownloadSourcePlugin)`
|
||||
work for the conformance test, but it ONLY checks method names —
|
||||
not signatures or async-ness. The conformance test in
|
||||
``tests/test_download_plugin_conformance.py`` does the deeper
|
||||
signature check.
|
||||
"""
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Configuration / lifecycle
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def is_configured(self) -> bool:
|
||||
"""Return True iff this source has the credentials / settings
|
||||
it needs to function. Used by the orchestrator to skip
|
||||
unconfigured sources during hybrid fallback."""
|
||||
...
|
||||
|
||||
async def check_connection(self) -> bool:
|
||||
"""Probe the source's API / endpoint. Return True if the
|
||||
source is reachable. May make a live network call."""
|
||||
...
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Search
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
async def search(
|
||||
self,
|
||||
query: str,
|
||||
timeout: Optional[int] = None,
|
||||
progress_callback=None,
|
||||
) -> Tuple[List[TrackResult], List[AlbumResult]]:
|
||||
"""Search the source for tracks (and albums where supported).
|
||||
|
||||
Returns a tuple of (track_results, album_results). Either
|
||||
list may be empty. Sources that don't expose album-level
|
||||
search return ``[]`` as the second element.
|
||||
"""
|
||||
...
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Download
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
async def download(
|
||||
self,
|
||||
username: str,
|
||||
filename: str,
|
||||
file_size: int = 0,
|
||||
) -> Optional[str]:
|
||||
"""Kick off a download. Returns a download_id string the
|
||||
caller can poll via ``get_download_status``. Returns ``None``
|
||||
if the source can't / won't handle this download.
|
||||
|
||||
``username`` is the source-name string for streaming sources
|
||||
(e.g. ``'youtube'``, ``'tidal'``) and the actual slskd peer
|
||||
username for Soulseek. ``filename`` is source-specific —
|
||||
Soulseek file path, YouTube ``video_id||title``, Tidal /
|
||||
Qobuz ``track_id||display``, etc.
|
||||
"""
|
||||
...
|
||||
|
||||
async def get_all_downloads(self) -> List[DownloadStatus]:
|
||||
"""Return live status of all downloads currently tracked by
|
||||
this source. The orchestrator concatenates results from
|
||||
every plugin to build the global download list."""
|
||||
...
|
||||
|
||||
async def get_download_status(self, download_id: str) -> Optional[DownloadStatus]:
|
||||
"""Return status for a single download or ``None`` if this
|
||||
source doesn't know about that download_id."""
|
||||
...
|
||||
|
||||
async def cancel_download(
|
||||
self,
|
||||
download_id: str,
|
||||
username: Optional[str] = None,
|
||||
remove: bool = False,
|
||||
) -> bool:
|
||||
"""Cancel an active download. ``remove=True`` also drops
|
||||
the row from the source's active-downloads tracking."""
|
||||
...
|
||||
|
||||
async def clear_all_completed_downloads(self) -> bool:
|
||||
"""Drop completed downloads from active tracking. Sources
|
||||
that don't keep completed history return True with no-op."""
|
||||
...
|
||||
192
core/download_plugins/registry.py
Normal file
192
core/download_plugins/registry.py
Normal file
|
|
@ -0,0 +1,192 @@
|
|||
"""Plugin registry — single source of truth for which download
|
||||
sources exist, what their canonical names are, and which client
|
||||
class implements each.
|
||||
|
||||
Replaces the orchestrator's hardcoded ``[self.soulseek,
|
||||
self.youtube, self.tidal, ...]`` lists and ``source_map`` dicts
|
||||
that historically had to be touched in 6+ places to add a source.
|
||||
With the registry:
|
||||
|
||||
- One ``register()`` call adds a source to every dispatch path.
|
||||
- Iteration helpers replace hand-maintained lists.
|
||||
- The orchestrator stays oblivious to source-specific quirks.
|
||||
- Adding Usenet (planned) becomes a one-line registry entry plus
|
||||
the new client class — no orchestrator changes.
|
||||
|
||||
This is the foundation step. Subsequent commits move shared logic
|
||||
(thread workers, search query normalization, post-processing
|
||||
context building) out of the orchestrator and the per-source
|
||||
clients into helpers the registry exposes.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Callable, Dict, Iterator, List, Optional, Tuple
|
||||
|
||||
from utils.logging_config import get_logger
|
||||
|
||||
from core.download_plugins.base import DownloadSourcePlugin
|
||||
|
||||
# Eager client imports — keep the import-order behavior the orchestrator
|
||||
# had before this refactor. Some integration tests inject mock modules
|
||||
# into ``sys.modules`` at collection time (see
|
||||
# ``tests/test_tidal_search_shortening.py``); making the registry's
|
||||
# factories lazy-import would cause tidalapi-dependent code to bind to
|
||||
# whichever ``tidalapi`` object happens to be in ``sys.modules`` at the
|
||||
# moment ``DownloadOrchestrator()`` is constructed — which is later
|
||||
# than the legacy module-top imports here. Importing everything at
|
||||
# registry-load time pins the bindings the same way the legacy
|
||||
# orchestrator did.
|
||||
from core.deezer_download_client import DeezerDownloadClient
|
||||
from core.hifi_client import HiFiClient
|
||||
from core.lidarr_download_client import LidarrDownloadClient
|
||||
from core.qobuz_client import QobuzClient
|
||||
from core.soulseek_client import SoulseekClient
|
||||
from core.soundcloud_client import SoundcloudClient
|
||||
from core.tidal_download_client import TidalDownloadClient
|
||||
from core.youtube_client import YouTubeClient
|
||||
|
||||
logger = get_logger("download_plugins.registry")
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class PluginSpec:
|
||||
"""Static descriptor for a download source. The ``factory`` is
|
||||
a zero-arg callable that builds the client instance — kept as a
|
||||
callable rather than a class so each source can do its own
|
||||
setup (e.g. SoulseekClient calls ``_setup_client`` after init,
|
||||
Deezer reads ARL from config). ``aliases`` lets the registry
|
||||
accept multiple historical names (e.g. ``deezer_dl`` is the
|
||||
legacy alias for ``deezer``)."""
|
||||
|
||||
name: str
|
||||
factory: Callable[[], DownloadSourcePlugin]
|
||||
display_name: str
|
||||
aliases: Tuple[str, ...] = field(default_factory=tuple)
|
||||
|
||||
|
||||
class DownloadPluginRegistry:
|
||||
"""Holds the live plugin instances + name → instance lookup.
|
||||
|
||||
Construction is two-phase:
|
||||
1. Specs are registered (cheap — just stores callable refs).
|
||||
2. ``initialize()`` calls each factory once and stores the
|
||||
resulting client. Failures are caught and logged so one
|
||||
broken source doesn't take down the orchestrator (mirrors
|
||||
the existing ``_safe_init`` behavior).
|
||||
|
||||
Iteration helpers (``all_plugins``, ``configured_plugins``)
|
||||
replace the hand-maintained lists scattered across the
|
||||
orchestrator's ``get_all_downloads``, ``cancel_all_downloads``,
|
||||
etc. so adding a source touches the registry alone.
|
||||
"""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self._specs: Dict[str, PluginSpec] = {}
|
||||
self._instances: Dict[str, Optional[DownloadSourcePlugin]] = {}
|
||||
self._init_failures: List[str] = []
|
||||
|
||||
def register(self, spec: PluginSpec) -> None:
|
||||
"""Register a plugin spec under its canonical name + each alias.
|
||||
Aliases all resolve to the same instance after ``initialize``."""
|
||||
if spec.name in self._specs:
|
||||
raise ValueError(f"Plugin already registered: {spec.name}")
|
||||
self._specs[spec.name] = spec
|
||||
|
||||
def initialize(self) -> None:
|
||||
"""Build every registered plugin's instance. Failures captured
|
||||
in ``init_failures`` and the slot is set to None so the
|
||||
orchestrator can skip unavailable sources without crashing."""
|
||||
for spec in self._specs.values():
|
||||
try:
|
||||
instance = spec.factory()
|
||||
self._instances[spec.name] = instance
|
||||
except Exception as exc:
|
||||
logger.error("%s download client failed to initialize: %s", spec.display_name, exc)
|
||||
self._init_failures.append(spec.display_name)
|
||||
self._instances[spec.name] = None
|
||||
|
||||
@property
|
||||
def init_failures(self) -> List[str]:
|
||||
return list(self._init_failures)
|
||||
|
||||
def get(self, name: str) -> Optional[DownloadSourcePlugin]:
|
||||
"""Resolve a name (or alias) to its plugin instance, or
|
||||
None if the source failed to initialize / isn't registered."""
|
||||
if not name:
|
||||
return None
|
||||
# Direct hit
|
||||
if name in self._instances:
|
||||
return self._instances[name]
|
||||
# Alias lookup
|
||||
for spec in self._specs.values():
|
||||
if name in spec.aliases:
|
||||
return self._instances.get(spec.name)
|
||||
return None
|
||||
|
||||
def get_spec(self, name: str) -> Optional[PluginSpec]:
|
||||
if name in self._specs:
|
||||
return self._specs[name]
|
||||
for spec in self._specs.values():
|
||||
if name in spec.aliases:
|
||||
return spec
|
||||
return None
|
||||
|
||||
def display_name(self, name: str) -> str:
|
||||
spec = self.get_spec(name)
|
||||
return spec.display_name if spec else name
|
||||
|
||||
def names(self) -> List[str]:
|
||||
"""Canonical names of every registered source (regardless of
|
||||
whether it initialized successfully)."""
|
||||
return list(self._specs.keys())
|
||||
|
||||
def all_plugins(self) -> Iterator[Tuple[str, DownloadSourcePlugin]]:
|
||||
"""Yield (name, plugin) for every successfully-initialized
|
||||
plugin. Replaces the orchestrator's hand-maintained client
|
||||
lists in get_all_downloads / cancel_all_downloads / etc."""
|
||||
for name, instance in self._instances.items():
|
||||
if instance is not None:
|
||||
yield name, instance
|
||||
|
||||
def configured_plugins(self) -> Iterator[Tuple[str, DownloadSourcePlugin]]:
|
||||
"""Yield (name, plugin) for every initialized AND configured
|
||||
plugin. Useful for hybrid mode and any operation that should
|
||||
skip sources the user hasn't set up."""
|
||||
for name, instance in self.all_plugins():
|
||||
try:
|
||||
if instance.is_configured():
|
||||
yield name, instance
|
||||
except Exception:
|
||||
continue
|
||||
|
||||
|
||||
def build_default_registry() -> DownloadPluginRegistry:
|
||||
"""Construct the registry with SoulSync's eight built-in download
|
||||
sources. Called once during orchestrator init.
|
||||
|
||||
Adding a new source (Usenet, etc.) means adding one ``register``
|
||||
call here — no orchestrator changes required.
|
||||
|
||||
The factory itself is just the class constructor — clients are
|
||||
imported eagerly at the top of this module so they bind to the
|
||||
real third-party libs (tidalapi, etc.) at import time, not at
|
||||
factory-call time. See the import-block comment above for why.
|
||||
"""
|
||||
registry = DownloadPluginRegistry()
|
||||
|
||||
registry.register(PluginSpec(name='soulseek', factory=SoulseekClient, display_name='Soulseek'))
|
||||
registry.register(PluginSpec(name='youtube', factory=YouTubeClient, display_name='YouTube'))
|
||||
registry.register(PluginSpec(name='tidal', factory=TidalDownloadClient, display_name='Tidal'))
|
||||
registry.register(PluginSpec(name='qobuz', factory=QobuzClient, display_name='Qobuz'))
|
||||
registry.register(PluginSpec(name='hifi', factory=HiFiClient, display_name='HiFi'))
|
||||
# 'deezer_dl' is the legacy name used in config + per-source dispatch
|
||||
# strings (e.g. orchestrator's ``source_map``). Canonical name is
|
||||
# ``deezer`` so future-facing code reads naturally.
|
||||
registry.register(PluginSpec(name='deezer', factory=DeezerDownloadClient, display_name='Deezer',
|
||||
aliases=('deezer_dl',)))
|
||||
registry.register(PluginSpec(name='lidarr', factory=LidarrDownloadClient, display_name='Lidarr'))
|
||||
registry.register(PluginSpec(name='soundcloud',factory=SoundcloudClient, display_name='SoundCloud'))
|
||||
|
||||
return registry
|
||||
199
core/download_plugins/types.py
Normal file
199
core/download_plugins/types.py
Normal file
|
|
@ -0,0 +1,199 @@
|
|||
"""Shared dataclasses for the download-plugin contract.
|
||||
|
||||
Every download source returns the same shape for search hits and
|
||||
download status — the four classes here are the canonical types
|
||||
that the ``DownloadSourcePlugin`` Protocol exchanges. Living in
|
||||
this neutral module (rather than ``core/soulseek_client.py`` where
|
||||
they grew up by accident) means a new plugin doesn't have to import
|
||||
from a sibling source just to satisfy the contract.
|
||||
|
||||
Move history: extracted from ``core.soulseek_client`` so plugins
|
||||
import from a neutral package per Cin's contract-first standard.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
from core.imports.filename import parse_filename_metadata
|
||||
|
||||
|
||||
@dataclass
|
||||
class SearchResult:
|
||||
"""Base class for search results"""
|
||||
username: str
|
||||
filename: str
|
||||
size: int
|
||||
bitrate: Optional[int]
|
||||
duration: Optional[int] # Duration in milliseconds (converted from slskd's seconds)
|
||||
quality: str
|
||||
free_upload_slots: int
|
||||
upload_speed: int
|
||||
queue_length: int
|
||||
result_type: str = "track" # "track" or "album"
|
||||
|
||||
@property
|
||||
def quality_score(self) -> float:
|
||||
quality_weights = {
|
||||
'flac': 1.0,
|
||||
'mp3': 0.8,
|
||||
'ogg': 0.7,
|
||||
'aac': 0.6,
|
||||
'wma': 0.5
|
||||
}
|
||||
|
||||
base_score = quality_weights.get(self.quality.lower(), 0.3)
|
||||
|
||||
if self.bitrate:
|
||||
if self.bitrate >= 320:
|
||||
base_score += 0.2
|
||||
elif self.bitrate >= 256:
|
||||
base_score += 0.1
|
||||
elif self.bitrate < 128:
|
||||
base_score -= 0.2
|
||||
|
||||
# Free upload slots
|
||||
if self.free_upload_slots == 0:
|
||||
base_score -= 0.15
|
||||
elif self.free_upload_slots > 0:
|
||||
base_score += 0.05
|
||||
|
||||
# Upload speed in bytes/sec (tiered)
|
||||
if self.upload_speed >= 5_000_000: # ~5 MB/s / 40 Mbps
|
||||
base_score += 0.15
|
||||
elif self.upload_speed >= 1_000_000: # ~1 MB/s / 8 Mbps
|
||||
base_score += 0.10
|
||||
elif self.upload_speed >= 500_000: # ~500 KB/s / 4 Mbps
|
||||
base_score += 0.05
|
||||
elif self.upload_speed < 100_000: # ~100 KB/s / 800 kbps
|
||||
base_score -= 0.05
|
||||
|
||||
# Queue length (graduated penalty)
|
||||
if self.queue_length > 50:
|
||||
base_score -= 0.25
|
||||
elif self.queue_length > 20:
|
||||
base_score -= 0.15
|
||||
elif self.queue_length > 10:
|
||||
base_score -= 0.10
|
||||
|
||||
return min(base_score, 1.0)
|
||||
|
||||
|
||||
@dataclass
|
||||
class TrackResult(SearchResult):
|
||||
"""Individual track search result"""
|
||||
artist: Optional[str] = None
|
||||
title: Optional[str] = None
|
||||
album: Optional[str] = None
|
||||
track_number: Optional[int] = None
|
||||
_source_metadata: Optional[Dict[str, Any]] = None
|
||||
|
||||
def __post_init__(self):
|
||||
self.result_type = "track"
|
||||
# Try to extract metadata from filename if not provided
|
||||
if not self.title or not self.artist:
|
||||
self._parse_filename_metadata()
|
||||
|
||||
def _parse_filename_metadata(self):
|
||||
"""Extract artist, title, album from filename patterns"""
|
||||
parsed = parse_filename_metadata(self.filename)
|
||||
if not self.artist and parsed.get("artist"):
|
||||
self.artist = parsed["artist"]
|
||||
if not self.title and parsed.get("title"):
|
||||
self.title = parsed["title"]
|
||||
if not self.album and parsed.get("album"):
|
||||
self.album = parsed["album"]
|
||||
if self.track_number is None:
|
||||
track_number = parsed.get("track_number")
|
||||
if track_number is not None:
|
||||
self.track_number = track_number
|
||||
|
||||
|
||||
@dataclass
|
||||
class AlbumResult:
|
||||
"""Album/folder search result containing multiple tracks"""
|
||||
username: str
|
||||
album_path: str # Directory path
|
||||
album_title: str
|
||||
artist: Optional[str]
|
||||
track_count: int
|
||||
total_size: int
|
||||
tracks: List[TrackResult]
|
||||
dominant_quality: str # Most common quality in album
|
||||
year: Optional[str] = None
|
||||
free_upload_slots: int = 0
|
||||
upload_speed: int = 0
|
||||
queue_length: int = 0
|
||||
result_type: str = "album"
|
||||
|
||||
@property
|
||||
def quality_score(self) -> float:
|
||||
"""Calculate album quality score based on dominant quality and track count"""
|
||||
quality_weights = {
|
||||
'flac': 1.0,
|
||||
'mp3': 0.8,
|
||||
'ogg': 0.7,
|
||||
'aac': 0.6,
|
||||
'wma': 0.5
|
||||
}
|
||||
|
||||
base_score = quality_weights.get(self.dominant_quality.lower(), 0.3)
|
||||
|
||||
# Bonus for complete albums (typically 8-15 tracks)
|
||||
if 8 <= self.track_count <= 20:
|
||||
base_score += 0.1
|
||||
elif self.track_count > 20:
|
||||
base_score += 0.05
|
||||
|
||||
# Free upload slots
|
||||
if self.free_upload_slots == 0:
|
||||
base_score -= 0.15
|
||||
elif self.free_upload_slots > 0:
|
||||
base_score += 0.05
|
||||
|
||||
# Upload speed in bytes/sec (tiered)
|
||||
if self.upload_speed >= 5_000_000: # ~5 MB/s / 40 Mbps
|
||||
base_score += 0.15
|
||||
elif self.upload_speed >= 1_000_000: # ~1 MB/s / 8 Mbps
|
||||
base_score += 0.10
|
||||
elif self.upload_speed >= 500_000: # ~500 KB/s / 4 Mbps
|
||||
base_score += 0.05
|
||||
elif self.upload_speed < 100_000: # ~100 KB/s / 800 kbps
|
||||
base_score -= 0.05
|
||||
|
||||
# Queue length (graduated penalty)
|
||||
if self.queue_length > 50:
|
||||
base_score -= 0.25
|
||||
elif self.queue_length > 20:
|
||||
base_score -= 0.15
|
||||
elif self.queue_length > 10:
|
||||
base_score -= 0.10
|
||||
|
||||
return min(base_score, 1.0)
|
||||
|
||||
@property
|
||||
def size_mb(self) -> int:
|
||||
"""Album size in MB"""
|
||||
return self.total_size // (1024 * 1024)
|
||||
|
||||
@property
|
||||
def average_track_size_mb(self) -> float:
|
||||
"""Average track size in MB"""
|
||||
if self.track_count > 0:
|
||||
return self.size_mb / self.track_count
|
||||
return 0
|
||||
|
||||
|
||||
@dataclass
|
||||
class DownloadStatus:
|
||||
id: str
|
||||
filename: str
|
||||
username: str
|
||||
state: str
|
||||
progress: float
|
||||
size: int
|
||||
transferred: int
|
||||
speed: int
|
||||
time_remaining: Optional[int] = None
|
||||
file_path: Optional[str] = None
|
||||
|
|
@ -42,31 +42,31 @@ _TERMINAL_STATUSES = {
|
|||
}
|
||||
|
||||
|
||||
def cancel_single_download(soulseek_client, run_async: Callable,
|
||||
def cancel_single_download(download_orchestrator, run_async: Callable,
|
||||
download_id: str, username: str) -> bool:
|
||||
"""Cancel one specific slskd download (with `remove=True`)."""
|
||||
return run_async(soulseek_client.cancel_download(download_id, username, remove=True))
|
||||
return run_async(download_orchestrator.cancel_download(download_id, username, remove=True))
|
||||
|
||||
|
||||
def cancel_all_active(soulseek_client, run_async: Callable,
|
||||
def cancel_all_active(download_orchestrator, run_async: Callable,
|
||||
sweep_callback: Callable[[], None]) -> tuple[bool, str]:
|
||||
"""Cancel every active slskd download, clear the resulting ones, sweep dirs.
|
||||
|
||||
Returns `(success, message)` so the route can map to the right HTTP shape.
|
||||
"""
|
||||
cancel_success = run_async(soulseek_client.cancel_all_downloads())
|
||||
cancel_success = run_async(download_orchestrator.cancel_all_downloads())
|
||||
if not cancel_success:
|
||||
return False, "Failed to cancel active downloads."
|
||||
|
||||
run_async(soulseek_client.clear_all_completed_downloads())
|
||||
run_async(download_orchestrator.clear_all_completed_downloads())
|
||||
sweep_callback()
|
||||
return True, "All downloads cancelled and cleared."
|
||||
|
||||
|
||||
def clear_finished_active(soulseek_client, run_async: Callable,
|
||||
def clear_finished_active(download_orchestrator, run_async: Callable,
|
||||
sweep_callback: Callable[[], None]) -> bool:
|
||||
"""Clear all terminal transfers from slskd, sweep dirs on success."""
|
||||
success = run_async(soulseek_client.clear_all_completed_downloads())
|
||||
success = run_async(download_orchestrator.clear_all_completed_downloads())
|
||||
if success:
|
||||
sweep_callback()
|
||||
return success
|
||||
|
|
|
|||
|
|
@ -25,7 +25,7 @@ and notifies the lifecycle via `on_download_completed(success=False)` so the
|
|||
worker slot frees up.
|
||||
|
||||
Lifted verbatim from web_server.py. Wide dependency surface
|
||||
(soulseek_client, spotify_client, lifecycle callback, context-key helper,
|
||||
(download_orchestrator, spotify_client, lifecycle callback, context-key helper,
|
||||
status updater, DB) all injected via `CandidatesDeps`.
|
||||
"""
|
||||
|
||||
|
|
@ -49,7 +49,7 @@ logger = logging.getLogger(__name__)
|
|||
@dataclass
|
||||
class CandidatesDeps:
|
||||
"""Bundle of cross-cutting deps the candidate-fallback logic needs."""
|
||||
soulseek_client: Any
|
||||
download_orchestrator: Any
|
||||
spotify_client: Any
|
||||
run_async: Callable[..., Any]
|
||||
get_database: Callable[[], Any]
|
||||
|
|
@ -209,7 +209,7 @@ def attempt_download_with_candidates(task_id, candidates, track, batch_id=None,
|
|||
|
||||
# Initiate download
|
||||
logger.info(f"[Modal Worker] Starting download: {username} / {os.path.basename(filename)}")
|
||||
download_id = deps.run_async(deps.soulseek_client.download(username, filename, size))
|
||||
download_id = deps.run_async(deps.download_orchestrator.download(username, filename, size))
|
||||
|
||||
if download_id:
|
||||
# Store context for post-processing with complete Spotify metadata (GUI PARITY)
|
||||
|
|
@ -330,7 +330,7 @@ def attempt_download_with_candidates(task_id, candidates, track, batch_id=None,
|
|||
logger.warning(f"[Modal Worker] Task {task_id} cancelled after download {download_id} started - attempting to cancel download")
|
||||
# Try to cancel the download immediately
|
||||
try:
|
||||
deps.run_async(deps.soulseek_client.cancel_download(download_id, username, remove=True))
|
||||
deps.run_async(deps.download_orchestrator.cancel_download(download_id, username, remove=True))
|
||||
logger.warning(f"Successfully cancelled active download {download_id}")
|
||||
except Exception as cancel_error:
|
||||
logger.error(f"Failed to cancel active download {download_id}: {cancel_error}")
|
||||
|
|
@ -340,8 +340,8 @@ def attempt_download_with_candidates(task_id, candidates, track, batch_id=None,
|
|||
deps.on_download_completed(batch_id, task_id, success=False)
|
||||
return False
|
||||
|
||||
# Store download information - use real download ID from soulseek_client
|
||||
# CRITICAL FIX: Trust the download ID returned by soulseek_client.download()
|
||||
# Store download information - use real download ID from download_orchestrator
|
||||
# CRITICAL FIX: Trust the download ID returned by download_orchestrator.download()
|
||||
download_tasks[task_id]['download_id'] = download_id
|
||||
|
||||
download_tasks[task_id]['username'] = username
|
||||
|
|
|
|||
|
|
@ -42,7 +42,7 @@ logger = logging.getLogger(__name__)
|
|||
class MasterDeps:
|
||||
"""Bundle of cross-cutting deps the master worker needs."""
|
||||
config_manager: Any
|
||||
soulseek_client: Any
|
||||
download_orchestrator: Any
|
||||
run_async: Callable[..., Any]
|
||||
mb_worker: Any
|
||||
mb_release_cache: dict
|
||||
|
|
@ -372,7 +372,7 @@ def run_full_missing_tracks_process(batch_id, playlist_id, tracks_json, deps: Ma
|
|||
_sr.info(f"[Album Pre-flight] Searching for '{artist_name} {album_name}'")
|
||||
logger.info(f"[Album Pre-flight] Searching Soulseek for complete album: '{artist_name} - {album_name}'")
|
||||
|
||||
slsk = deps.soulseek_client.soulseek if hasattr(deps.soulseek_client, 'soulseek') else deps.soulseek_client
|
||||
slsk = deps.download_orchestrator.client('soulseek') if hasattr(deps.download_orchestrator, 'client') else deps.download_orchestrator
|
||||
|
||||
# Try multiple query variations (banned keywords in artist/album name can return 0 results)
|
||||
album_queries = [f"{artist_name} {album_name}"]
|
||||
|
|
|
|||
|
|
@ -33,7 +33,7 @@ _run_post_processing_worker = None
|
|||
_start_next_batch_of_downloads = None
|
||||
_orphaned_download_keys = None
|
||||
missing_download_executor = None
|
||||
soulseek_client = None
|
||||
download_orchestrator = None
|
||||
|
||||
|
||||
def init(
|
||||
|
|
@ -44,12 +44,12 @@ def init(
|
|||
start_next_batch_of_downloads,
|
||||
orphaned_download_keys,
|
||||
missing_download_executor_obj,
|
||||
soulseek_client_obj,
|
||||
download_orchestrator_obj,
|
||||
):
|
||||
"""Bind web_server-side helpers/globals so the class body can resolve them."""
|
||||
global _make_context_key, _on_download_completed, _download_track_worker
|
||||
global _run_post_processing_worker, _start_next_batch_of_downloads
|
||||
global _orphaned_download_keys, missing_download_executor, soulseek_client
|
||||
global _orphaned_download_keys, missing_download_executor, download_orchestrator
|
||||
_make_context_key = make_context_key
|
||||
_on_download_completed = on_download_completed
|
||||
_download_track_worker = download_track_worker
|
||||
|
|
@ -57,7 +57,7 @@ def init(
|
|||
_start_next_batch_of_downloads = start_next_batch_of_downloads
|
||||
_orphaned_download_keys = orphaned_download_keys
|
||||
missing_download_executor = missing_download_executor_obj
|
||||
soulseek_client = soulseek_client_obj
|
||||
download_orchestrator = download_orchestrator_obj
|
||||
|
||||
|
||||
class WebUIDownloadMonitor:
|
||||
|
|
@ -199,7 +199,7 @@ class WebUIDownloadMonitor:
|
|||
if op[0] == 'cancel_download':
|
||||
_, download_id, username = op
|
||||
logger.debug(f"[Deferred] Cancelling download: {download_id} from {username}")
|
||||
run_async(soulseek_client.cancel_download(download_id, username, remove=True))
|
||||
run_async(download_orchestrator.cancel_download(download_id, username, remove=True))
|
||||
logger.debug(f"[Deferred] Successfully cancelled download {download_id}")
|
||||
elif op[0] == 'cleanup_orphan':
|
||||
_, context_key = op
|
||||
|
|
@ -254,8 +254,9 @@ class WebUIDownloadMonitor:
|
|||
|
||||
# Get Soulseek downloads from API
|
||||
transfers_data = None
|
||||
if soulseek_active and soulseek_client and getattr(soulseek_client, 'soulseek', None) and soulseek_client.soulseek.base_url:
|
||||
transfers_data = run_async(soulseek_client._make_request('GET', 'transfers/downloads'))
|
||||
_slsk = download_orchestrator.client('soulseek') if download_orchestrator and hasattr(download_orchestrator, 'client') else None
|
||||
if soulseek_active and _slsk and _slsk.base_url:
|
||||
transfers_data = run_async(download_orchestrator._make_request('GET', 'transfers/downloads'))
|
||||
if transfers_data:
|
||||
for user_data in transfers_data:
|
||||
username = user_data.get('username', 'Unknown')
|
||||
|
|
@ -266,31 +267,34 @@ class WebUIDownloadMonitor:
|
|||
key = _make_context_key(username, file_info.get('filename', ''))
|
||||
live_transfers[key] = file_info
|
||||
|
||||
# Also get non-Soulseek downloads (YouTube/Tidal/Qobuz/HiFi/Deezer/Lidarr)
|
||||
# Call each client directly to avoid redundant slskd API call through orchestrator
|
||||
# Also get non-Soulseek downloads via the engine — single
|
||||
# cross-source aggregation, no per-source iteration.
|
||||
try:
|
||||
all_downloads = []
|
||||
for _dl_client in [soulseek_client.youtube, soulseek_client.tidal, soulseek_client.qobuz,
|
||||
soulseek_client.hifi, soulseek_client.deezer_dl, soulseek_client.lidarr,
|
||||
getattr(soulseek_client, 'soundcloud', None)]:
|
||||
if _dl_client:
|
||||
try:
|
||||
all_downloads.extend(run_async(_dl_client.get_all_downloads()))
|
||||
except Exception:
|
||||
pass
|
||||
if download_orchestrator and hasattr(download_orchestrator, 'engine'):
|
||||
try:
|
||||
# Exclude soulseek — slskd transfers were already
|
||||
# pulled via the transfers/downloads endpoint above.
|
||||
# Without the exclude both fetch paths run, doubling
|
||||
# the per-tick slskd API hit.
|
||||
all_downloads = run_async(
|
||||
download_orchestrator.engine.get_all_downloads(exclude=('soulseek',))
|
||||
)
|
||||
except Exception:
|
||||
pass
|
||||
for download in all_downloads:
|
||||
key = _make_context_key(download.username, download.filename)
|
||||
# Convert DownloadStatus to transfer dict format for monitor compatibility
|
||||
live_transfers[key] = {
|
||||
'id': download.id,
|
||||
'filename': download.filename,
|
||||
'username': download.username,
|
||||
'state': download.state,
|
||||
'percentComplete': download.progress,
|
||||
'size': download.size,
|
||||
'bytesTransferred': download.transferred,
|
||||
'averageSpeed': download.speed,
|
||||
}
|
||||
key = _make_context_key(download.username, download.filename)
|
||||
# Convert DownloadStatus to transfer dict format for monitor compatibility
|
||||
live_transfers[key] = {
|
||||
'id': download.id,
|
||||
'filename': download.filename,
|
||||
'username': download.username,
|
||||
'state': download.state,
|
||||
'percentComplete': download.progress,
|
||||
'size': download.size,
|
||||
'bytesTransferred': download.transferred,
|
||||
'averageSpeed': download.speed,
|
||||
}
|
||||
except Exception as yt_error:
|
||||
logger.error(f"Monitor: Could not fetch streaming source downloads: {yt_error}")
|
||||
|
||||
|
|
|
|||
|
|
@ -54,7 +54,7 @@ class PostProcessDeps:
|
|||
always live (no caching of pre-init Spotify clients etc).
|
||||
"""
|
||||
config_manager: Any
|
||||
soulseek_client: Any
|
||||
download_orchestrator: Any
|
||||
run_async: Callable
|
||||
docker_resolve_path: Callable[[str], str]
|
||||
extract_filename: Callable[[str], str]
|
||||
|
|
@ -196,7 +196,7 @@ def run_post_processing_worker(task_id: str, batch_id: str, deps: PostProcessDep
|
|||
# Query the download orchestrator for the status which contains the real file path
|
||||
# CRITICAL FIX: Use the actual download_id designated by the client, not the internal task_id
|
||||
actual_download_id = task.get('download_id') or task_id
|
||||
status = deps.run_async(deps.soulseek_client.get_download_status(actual_download_id))
|
||||
status = deps.run_async(deps.download_orchestrator.get_download_status(actual_download_id))
|
||||
if status and status.file_path:
|
||||
real_path = status.file_path
|
||||
if os.path.exists(real_path):
|
||||
|
|
|
|||
|
|
@ -34,7 +34,7 @@ logger = logging.getLogger(__name__)
|
|||
@dataclass
|
||||
class TaskWorkerDeps:
|
||||
"""Bundle of cross-cutting deps the per-task download worker needs."""
|
||||
soulseek_client: Any
|
||||
download_orchestrator: Any
|
||||
matching_engine: Any
|
||||
run_async: Callable
|
||||
try_source_reuse: Callable # (task_id, batch_id, track) -> bool
|
||||
|
|
@ -210,7 +210,7 @@ def download_track_worker(task_id: str, batch_id: Optional[str], deps: TaskWorke
|
|||
|
||||
try:
|
||||
# Perform search with timeout
|
||||
tracks_result, _ = deps.run_async(deps.soulseek_client.search(query, timeout=30))
|
||||
tracks_result, _ = deps.run_async(deps.download_orchestrator.search(query, timeout=30))
|
||||
logger.debug(f"Search completed for task {task_id}, got {len(tracks_result) if tracks_result else 0} results")
|
||||
|
||||
# CRITICAL: Check cancellation immediately after search returns
|
||||
|
|
@ -260,7 +260,13 @@ def download_track_worker(task_id: str, batch_id: Optional[str], deps: TaskWorke
|
|||
search_diagnostics.append(f'"{query}": {result_count} results, {len(candidates)} passed filters but download failed to start')
|
||||
else:
|
||||
search_diagnostics.append(f'"{query}": {result_count} results but none passed quality/artist filters')
|
||||
all_raw_results.extend(tracks_result[:20]) # Keep top results for review
|
||||
# Strip SoundCloud preview snippets before caching for the
|
||||
# review modal — the user can't pick something useful from
|
||||
# a 30s preview clip, and clicking one bypasses validation
|
||||
# and downloads it anyway.
|
||||
from core.downloads.validation import filter_soundcloud_previews
|
||||
_filtered_raw = filter_soundcloud_previews(tracks_result[:20], track)
|
||||
all_raw_results.extend(_filtered_raw)
|
||||
else:
|
||||
search_diagnostics.append(f'"{query}": no results found')
|
||||
|
||||
|
|
@ -272,24 +278,23 @@ def download_track_worker(task_id: str, batch_id: Optional[str], deps: TaskWorke
|
|||
# === HYBRID FALLBACK: If primary source failed, try remaining sources directly ===
|
||||
# The orchestrator's hybrid search stops at the first source with results, even if
|
||||
# those results all fail quality filtering. Try remaining sources individually.
|
||||
if getattr(deps.soulseek_client, 'mode', '') == 'hybrid':
|
||||
if getattr(deps.download_orchestrator, 'mode', '') == 'hybrid':
|
||||
try:
|
||||
orch = deps.soulseek_client
|
||||
orch = deps.download_orchestrator
|
||||
hybrid_order = getattr(orch, 'hybrid_order', None) or []
|
||||
if not hybrid_order:
|
||||
primary = getattr(orch, 'hybrid_primary', 'soulseek')
|
||||
secondary = getattr(orch, 'hybrid_secondary', '')
|
||||
hybrid_order = [primary, secondary] if secondary and secondary != primary else [primary]
|
||||
|
||||
# Resolve via the orchestrator's generic accessor — the
|
||||
# legacy per-source attrs were dropped in the registry
|
||||
# refactor, so getattr(orch, 'soulseek', None) etc. all
|
||||
# silently returned None and the fallback never fired.
|
||||
source_clients = {
|
||||
'soulseek': getattr(orch, 'soulseek', None),
|
||||
'youtube': getattr(orch, 'youtube', None),
|
||||
'tidal': getattr(orch, 'tidal', None),
|
||||
'qobuz': getattr(orch, 'qobuz', None),
|
||||
'hifi': getattr(orch, 'hifi', None),
|
||||
'deezer_dl': getattr(orch, 'deezer_dl', None),
|
||||
'lidarr': getattr(orch, 'lidarr', None),
|
||||
'soundcloud': getattr(orch, 'soundcloud', None),
|
||||
name: orch.client(name)
|
||||
for name in ('soulseek', 'youtube', 'tidal', 'qobuz',
|
||||
'hifi', 'deezer_dl', 'lidarr', 'soundcloud')
|
||||
}
|
||||
|
||||
# The orchestrator tried sources in order but stopped at the first with results.
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
"""Soulseek/streaming candidate validation — lifted from web_server.py.
|
||||
|
||||
Body is byte-identical to the original. ``matching_engine`` and
|
||||
``soulseek_client`` are injected via init() because both are
|
||||
``download_orchestrator`` are injected via init() because both are
|
||||
constructed in web_server.py and referenced by name throughout
|
||||
the body.
|
||||
"""
|
||||
|
|
@ -14,14 +14,51 @@ logger = logging.getLogger(__name__)
|
|||
|
||||
# Injected at runtime via init().
|
||||
matching_engine = None
|
||||
soulseek_client = None
|
||||
download_orchestrator = None
|
||||
|
||||
|
||||
def init(matching_engine_obj, soulseek_client_obj):
|
||||
def init(matching_engine_obj, download_orchestrator_obj):
|
||||
"""Bind the matching engine and download orchestrator from web_server."""
|
||||
global matching_engine, soulseek_client
|
||||
global matching_engine, download_orchestrator
|
||||
matching_engine = matching_engine_obj
|
||||
soulseek_client = soulseek_client_obj
|
||||
download_orchestrator = download_orchestrator_obj
|
||||
|
||||
|
||||
def filter_soundcloud_previews(results, expected_track):
|
||||
"""Drop SoundCloud preview snippets so they never reach the cache,
|
||||
the modal, or the auto-download attempt.
|
||||
|
||||
SoundCloud serves a ~30s preview clip for tracks gated behind Go+ /
|
||||
login. yt-dlp accepts the preview as the download payload, the
|
||||
integrity check catches the truncated file, but the user just sees
|
||||
"all candidates failed" with previews still listed in the modal
|
||||
(and clickable for manual retry, which downloads another preview).
|
||||
|
||||
Filter at every spot raw search results enter the task: validation
|
||||
scoring, modal-cache fallback when validation drops everything,
|
||||
and the not-found raw-results cache. Keep candidates that genuinely
|
||||
are short (intros, sound effects) when the expected track is also
|
||||
short.
|
||||
"""
|
||||
if not results or not expected_track:
|
||||
return results
|
||||
expected_ms = getattr(expected_track, 'duration_ms', 0) or 0
|
||||
if expected_ms <= 0:
|
||||
return results
|
||||
expected_secs = expected_ms / 1000.0
|
||||
if expected_secs <= 60:
|
||||
return results
|
||||
|
||||
def _is_preview(r):
|
||||
if getattr(r, 'username', None) != 'soundcloud':
|
||||
return False
|
||||
cand_ms = getattr(r, 'duration', None) or 0
|
||||
if cand_ms <= 0:
|
||||
return False
|
||||
cand_secs = cand_ms / 1000.0
|
||||
return cand_secs < 35 or cand_secs < expected_secs * 0.5
|
||||
|
||||
return [r for r in results if not _is_preview(r)]
|
||||
|
||||
|
||||
def get_valid_candidates(results, spotify_track, query):
|
||||
|
|
@ -33,6 +70,13 @@ def get_valid_candidates(results, spotify_track, query):
|
|||
if not results:
|
||||
return []
|
||||
|
||||
# Pre-filter: drop SoundCloud preview snippets when expected
|
||||
# duration is non-trivially long. Same helper is also applied at
|
||||
# the modal-cache fallback path so previews never reach the UI.
|
||||
results = filter_soundcloud_previews(results, spotify_track)
|
||||
if not results:
|
||||
return []
|
||||
|
||||
# Streaming sources (YouTube, Tidal, Qobuz, HiFi, Deezer, SoundCloud) return structured API results
|
||||
# with proper artist/title metadata — score using the same matching engine as Soulseek
|
||||
_streaming_sources = ("youtube", "tidal", "qobuz", "hifi", "deezer_dl", "soundcloud")
|
||||
|
|
@ -45,7 +89,12 @@ def get_valid_candidates(results, spotify_track, query):
|
|||
# Detect if the expected track is a specific version (live, remix, acoustic, etc.)
|
||||
expected_title_lower = (expected_title or '').lower()
|
||||
_version_keywords = ['remix', 'live', 'acoustic', 'instrumental', 'radio edit',
|
||||
'extended', 'slowed', 'sped up', 'reverb', 'karaoke']
|
||||
'extended', 'slowed', 'sped up', 'reverb', 'karaoke',
|
||||
# Producer-tag noise common on SoundCloud — "type
|
||||
# beat" is an instrumental track produced in
|
||||
# someone's style, tagged with the artist name to
|
||||
# game search. NEVER the real song.
|
||||
'type beat']
|
||||
expected_is_version = any(kw in expected_title_lower for kw in _version_keywords)
|
||||
|
||||
scored = []
|
||||
|
|
@ -151,8 +200,8 @@ def get_valid_candidates(results, spotify_track, query):
|
|||
quality_filtered_candidates = initial_candidates
|
||||
else:
|
||||
# Filter by user's quality profile before artist verification (Soulseek only)
|
||||
# Use existing soulseek_client to avoid re-initializing (which accesses download_path filesystem)
|
||||
quality_filtered_candidates = soulseek_client.soulseek.filter_results_by_quality_preference(initial_candidates)
|
||||
# Use existing download_orchestrator to avoid re-initializing (which accesses download_path filesystem)
|
||||
quality_filtered_candidates = download_orchestrator.client('soulseek').filter_results_by_quality_preference(initial_candidates)
|
||||
|
||||
# IMPORTANT: Respect empty results from quality filter
|
||||
# If user has strict quality requirements (e.g., FLAC-only with fallback disabled),
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@
|
|||
|
||||
Body is byte-identical to the original. Wishlist helpers are
|
||||
direct imports from core.wishlist.*; runtime state comes from
|
||||
core.runtime_state; automation_engine, soulseek_client, and the
|
||||
core.runtime_state; automation_engine, download_orchestrator, and the
|
||||
sweep helper are injected via init() because they are constructed
|
||||
in web_server.py.
|
||||
"""
|
||||
|
|
@ -29,15 +29,15 @@ logger = logging.getLogger(__name__)
|
|||
|
||||
# Injected at runtime via init().
|
||||
automation_engine = None
|
||||
soulseek_client = None
|
||||
download_orchestrator = None
|
||||
_sweep_empty_download_directories = None
|
||||
|
||||
|
||||
def init(engine, soulseek_client_obj, sweep_fn):
|
||||
def init(engine, download_orchestrator_obj, sweep_fn):
|
||||
"""Bind shared singletons + the sweep helper from web_server."""
|
||||
global automation_engine, soulseek_client, _sweep_empty_download_directories
|
||||
global automation_engine, download_orchestrator, _sweep_empty_download_directories
|
||||
automation_engine = engine
|
||||
soulseek_client = soulseek_client_obj
|
||||
download_orchestrator = download_orchestrator_obj
|
||||
_sweep_empty_download_directories = sweep_fn
|
||||
|
||||
|
||||
|
|
@ -185,7 +185,7 @@ def _process_failed_tracks_to_wishlist_exact(batch_id):
|
|||
# Auto-cleanup: Clear completed downloads from slskd
|
||||
try:
|
||||
logger.info(f"[Auto-Cleanup] Clearing completed downloads from slskd after batch {batch_id}")
|
||||
run_async(soulseek_client.clear_all_completed_downloads())
|
||||
run_async(download_orchestrator.clear_all_completed_downloads())
|
||||
logger.info("[Auto-Cleanup] Completed downloads cleared from slskd")
|
||||
except Exception as cleanup_error:
|
||||
logger.warning(f"[Auto-Cleanup] Failed to clear completed downloads: {cleanup_error}")
|
||||
|
|
|
|||
|
|
@ -33,7 +33,7 @@ import requests as http_requests
|
|||
|
||||
from utils.logging_config import get_logger
|
||||
from config.settings import config_manager
|
||||
from core.soulseek_client import TrackResult, AlbumResult, DownloadStatus
|
||||
from core.download_plugins.types import TrackResult, AlbumResult, DownloadStatus
|
||||
|
||||
logger = get_logger("hifi_client")
|
||||
|
||||
|
|
@ -86,7 +86,10 @@ DEFAULT_INSTANCES = [
|
|||
]
|
||||
|
||||
|
||||
class HiFiClient:
|
||||
from core.download_plugins.base import DownloadSourcePlugin
|
||||
|
||||
|
||||
class HiFiClient(DownloadSourcePlugin):
|
||||
"""
|
||||
HiFi API client for searching and downloading lossless music.
|
||||
Uses public hifi-api instances (Tidal backend) — no auth required.
|
||||
|
|
@ -110,9 +113,7 @@ class HiFiClient:
|
|||
'Accept': 'application/json',
|
||||
})
|
||||
|
||||
self.active_downloads: Dict[str, Dict[str, Any]] = {}
|
||||
self._download_lock = threading.Lock()
|
||||
|
||||
self._engine = None
|
||||
self.shutdown_check = None
|
||||
|
||||
self._last_api_call = 0
|
||||
|
|
@ -125,6 +126,9 @@ class HiFiClient:
|
|||
def set_shutdown_check(self, check_callable):
|
||||
self.shutdown_check = check_callable
|
||||
|
||||
def set_engine(self, engine):
|
||||
self._engine = engine
|
||||
|
||||
def _load_instances_from_db(self):
|
||||
try:
|
||||
from database.music_database import get_database
|
||||
|
|
@ -571,71 +575,34 @@ class HiFiClient:
|
|||
)
|
||||
|
||||
async def download(self, username: str, filename: str, file_size: int = 0) -> Optional[str]:
|
||||
if '||' not in filename:
|
||||
logger.error(f"Invalid filename format: {filename}")
|
||||
return None
|
||||
if self._engine is None:
|
||||
# Raise rather than return None so the orchestrator's
|
||||
# download_with_fallback surfaces a real warning + tries
|
||||
# the next source. Returning None silently dropped the
|
||||
# download with no user feedback (per JohnBaumb).
|
||||
raise RuntimeError("HiFi client has no engine reference — cannot dispatch download")
|
||||
|
||||
track_id_str, display_name = filename.split('||', 1)
|
||||
try:
|
||||
if '||' not in filename:
|
||||
logger.error(f"Invalid filename format: {filename}")
|
||||
return None
|
||||
|
||||
track_id_str, display_name = filename.split('||', 1)
|
||||
try:
|
||||
track_id = int(track_id_str)
|
||||
except ValueError:
|
||||
logger.error(f"Invalid track ID: {track_id_str}")
|
||||
return None
|
||||
|
||||
download_id = str(uuid.uuid4())
|
||||
|
||||
with self._download_lock:
|
||||
self.active_downloads[download_id] = {
|
||||
'id': download_id,
|
||||
'filename': filename,
|
||||
'username': 'hifi',
|
||||
'state': 'Initializing',
|
||||
'progress': 0.0,
|
||||
'size': 0,
|
||||
'transferred': 0,
|
||||
'speed': 0,
|
||||
'time_remaining': None,
|
||||
'track_id': track_id,
|
||||
'display_name': display_name,
|
||||
'file_path': None,
|
||||
}
|
||||
|
||||
thread = threading.Thread(
|
||||
target=self._download_worker,
|
||||
args=(download_id, track_id, display_name),
|
||||
daemon=True,
|
||||
)
|
||||
thread.start()
|
||||
|
||||
return download_id
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to start HiFi download: {e}")
|
||||
track_id = int(track_id_str)
|
||||
except ValueError:
|
||||
logger.error(f"Invalid track ID: {track_id_str}")
|
||||
return None
|
||||
|
||||
def _download_worker(self, download_id: str, track_id: int, display_name: str):
|
||||
try:
|
||||
with self._download_lock:
|
||||
if download_id in self.active_downloads:
|
||||
self.active_downloads[download_id]['state'] = 'InProgress, Downloading'
|
||||
|
||||
file_path = self._download_sync(download_id, track_id, display_name)
|
||||
|
||||
with self._download_lock:
|
||||
if download_id in self.active_downloads:
|
||||
if file_path:
|
||||
self.active_downloads[download_id]['state'] = 'Completed, Succeeded'
|
||||
self.active_downloads[download_id]['progress'] = 100.0
|
||||
self.active_downloads[download_id]['file_path'] = file_path
|
||||
else:
|
||||
self.active_downloads[download_id]['state'] = 'Errored'
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"HiFi download worker failed for {download_id}: {e}")
|
||||
with self._download_lock:
|
||||
if download_id in self.active_downloads:
|
||||
self.active_downloads[download_id]['state'] = 'Errored'
|
||||
return self._engine.worker.dispatch(
|
||||
source_name='hifi',
|
||||
target_id=track_id,
|
||||
display_name=display_name,
|
||||
original_filename=filename,
|
||||
impl_callable=self._download_sync,
|
||||
extra_record_fields={
|
||||
'track_id': track_id,
|
||||
'display_name': display_name,
|
||||
},
|
||||
)
|
||||
|
||||
def _download_sync(self, download_id: str, track_id: int, display_name: str) -> Optional[str]:
|
||||
quality_key = config_manager.get('hifi_download.quality', 'lossless')
|
||||
|
|
@ -676,9 +643,8 @@ class HiFiClient:
|
|||
speed_start = time.time()
|
||||
segments_completed = 0
|
||||
|
||||
with self._download_lock:
|
||||
if download_id in self.active_downloads:
|
||||
self.active_downloads[download_id]['size'] = 0
|
||||
if self._engine is not None:
|
||||
self._engine.update_record('hifi', download_id, {'size': 0})
|
||||
|
||||
with intermediate_path.open('wb') as output_file:
|
||||
if init_uri:
|
||||
|
|
@ -777,80 +743,77 @@ class HiFiClient:
|
|||
def _update_download_progress(self, download_id: str, downloaded: int,
|
||||
segments_completed: int, total_segments: int,
|
||||
speed_start: float):
|
||||
with self._download_lock:
|
||||
if download_id not in self.active_downloads:
|
||||
return
|
||||
info = self.active_downloads[download_id]
|
||||
info['transferred'] = downloaded
|
||||
if self._engine is None:
|
||||
return
|
||||
record = self._engine.get_record('hifi', download_id)
|
||||
if record is None:
|
||||
return
|
||||
|
||||
now = time.time()
|
||||
elapsed_total = now - speed_start
|
||||
speed = int(downloaded / elapsed_total) if elapsed_total > 0 else 0
|
||||
info['speed'] = speed
|
||||
now = time.time()
|
||||
elapsed_total = now - speed_start
|
||||
speed = int(downloaded / elapsed_total) if elapsed_total > 0 else 0
|
||||
|
||||
if total_segments > 0:
|
||||
progress = (segments_completed / total_segments) * 100
|
||||
info['progress'] = round(min(progress, 99.9), 1)
|
||||
progress = record.get('progress', 0.0)
|
||||
if total_segments > 0:
|
||||
progress = round(min((segments_completed / total_segments) * 100, 99.9), 1)
|
||||
|
||||
time_remaining = None
|
||||
if speed > 0:
|
||||
remaining_bytes = downloaded * (total_segments / max(segments_completed, 1)) - downloaded
|
||||
if remaining_bytes > 0:
|
||||
time_remaining = int(remaining_bytes / speed)
|
||||
info['time_remaining'] = time_remaining
|
||||
time_remaining = None
|
||||
if speed > 0:
|
||||
remaining_bytes = downloaded * (total_segments / max(segments_completed, 1)) - downloaded
|
||||
if remaining_bytes > 0:
|
||||
time_remaining = int(remaining_bytes / speed)
|
||||
|
||||
self._engine.update_record('hifi', download_id, {
|
||||
'transferred': downloaded,
|
||||
'speed': speed,
|
||||
'progress': progress,
|
||||
'time_remaining': time_remaining,
|
||||
})
|
||||
|
||||
def _record_to_status(self, record):
|
||||
return DownloadStatus(
|
||||
id=record['id'],
|
||||
filename=record['filename'],
|
||||
username=record['username'],
|
||||
state=record['state'],
|
||||
progress=record['progress'],
|
||||
size=record.get('size', 0),
|
||||
transferred=record.get('transferred', 0),
|
||||
speed=record.get('speed', 0),
|
||||
time_remaining=record.get('time_remaining'),
|
||||
file_path=record.get('file_path'),
|
||||
)
|
||||
|
||||
async def get_all_downloads(self) -> List[DownloadStatus]:
|
||||
statuses = []
|
||||
with self._download_lock:
|
||||
for _dl_id, info in self.active_downloads.items():
|
||||
statuses.append(DownloadStatus(
|
||||
id=info['id'],
|
||||
filename=info['filename'],
|
||||
username=info['username'],
|
||||
state=info['state'],
|
||||
progress=info['progress'],
|
||||
size=info['size'],
|
||||
transferred=info['transferred'],
|
||||
speed=info['speed'],
|
||||
time_remaining=info.get('time_remaining'),
|
||||
file_path=info.get('file_path'),
|
||||
))
|
||||
return statuses
|
||||
if self._engine is None:
|
||||
return []
|
||||
return [
|
||||
self._record_to_status(record)
|
||||
for record in self._engine.iter_records_for_source('hifi')
|
||||
]
|
||||
|
||||
async def get_download_status(self, download_id: str) -> Optional[DownloadStatus]:
|
||||
with self._download_lock:
|
||||
info = self.active_downloads.get(download_id)
|
||||
if not info:
|
||||
return None
|
||||
return DownloadStatus(
|
||||
id=info['id'],
|
||||
filename=info['filename'],
|
||||
username=info['username'],
|
||||
state=info['state'],
|
||||
progress=info['progress'],
|
||||
size=info['size'],
|
||||
transferred=info['transferred'],
|
||||
speed=info['speed'],
|
||||
time_remaining=info.get('time_remaining'),
|
||||
file_path=info.get('file_path'),
|
||||
)
|
||||
if self._engine is None:
|
||||
return None
|
||||
record = self._engine.get_record('hifi', download_id)
|
||||
return self._record_to_status(record) if record is not None else None
|
||||
|
||||
async def cancel_download(self, download_id: str, username: str = None,
|
||||
remove: bool = False) -> bool:
|
||||
with self._download_lock:
|
||||
if download_id not in self.active_downloads:
|
||||
return False
|
||||
self.active_downloads[download_id]['state'] = 'Cancelled'
|
||||
if remove:
|
||||
del self.active_downloads[download_id]
|
||||
if self._engine is None:
|
||||
return False
|
||||
if self._engine.get_record('hifi', download_id) is None:
|
||||
return False
|
||||
self._engine.update_record('hifi', download_id, {'state': 'Cancelled'})
|
||||
if remove:
|
||||
self._engine.remove_record('hifi', download_id)
|
||||
return True
|
||||
|
||||
async def clear_all_completed_downloads(self) -> bool:
|
||||
with self._download_lock:
|
||||
to_remove = [
|
||||
did for did, info in self.active_downloads.items()
|
||||
if info.get('state', '') in ('Completed, Succeeded', 'Cancelled', 'Errored', 'Aborted')
|
||||
]
|
||||
for did in to_remove:
|
||||
del self.active_downloads[did]
|
||||
if self._engine is None:
|
||||
return True
|
||||
terminal = {'Completed, Succeeded', 'Cancelled', 'Errored', 'Aborted'}
|
||||
for record in list(self._engine.iter_records_for_source('hifi')):
|
||||
if record.get('state') in terminal:
|
||||
self._engine.remove_record('hifi', record['id'])
|
||||
return True
|
||||
|
|
|
|||
|
|
@ -208,7 +208,7 @@ def redownload_start(track_id):
|
|||
# Build a TrackResult-like candidate and submit to download
|
||||
def _run_redownload():
|
||||
try:
|
||||
from core.soulseek_client import TrackResult
|
||||
from core.download_plugins.types import TrackResult
|
||||
from core.itunes_client import Track as MetaTrack
|
||||
tr = TrackResult(
|
||||
username=candidate['username'],
|
||||
|
|
|
|||
|
|
@ -28,12 +28,15 @@ from utils.logging_config import get_logger
|
|||
from config.settings import config_manager
|
||||
|
||||
# Import Soulseek data structures for drop-in replacement compatibility
|
||||
from core.soulseek_client import TrackResult, AlbumResult, DownloadStatus
|
||||
from core.download_plugins.types import TrackResult, AlbumResult, DownloadStatus
|
||||
|
||||
logger = get_logger("lidarr_client")
|
||||
|
||||
|
||||
class LidarrDownloadClient:
|
||||
from core.download_plugins.base import DownloadSourcePlugin
|
||||
|
||||
|
||||
class LidarrDownloadClient(DownloadSourcePlugin):
|
||||
"""Lidarr download client — uses Lidarr as a download source for Usenet/torrent content.
|
||||
|
||||
Implements the same interface as SoulseekClient, QobuzClient, TidalDownloadClient
|
||||
|
|
|
|||
|
|
@ -8,7 +8,7 @@ from config.settings import config_manager
|
|||
|
||||
from core.spotify_client import Track as SpotifyTrack
|
||||
from core.plex_client import PlexTrackInfo
|
||||
from core.soulseek_client import TrackResult, AlbumResult
|
||||
from core.download_plugins.types import TrackResult, AlbumResult
|
||||
|
||||
|
||||
logger = get_logger("matching_engine")
|
||||
|
|
|
|||
|
|
@ -28,7 +28,7 @@ from utils.logging_config import get_logger
|
|||
from config.settings import config_manager
|
||||
|
||||
# Import Soulseek data structures for drop-in replacement compatibility
|
||||
from core.soulseek_client import TrackResult, AlbumResult, DownloadStatus
|
||||
from core.download_plugins.types import TrackResult, AlbumResult, DownloadStatus
|
||||
|
||||
logger = get_logger("qobuz_client")
|
||||
|
||||
|
|
@ -104,7 +104,10 @@ QOBUZ_QUALITY_MAP = {
|
|||
}
|
||||
|
||||
|
||||
class QobuzClient:
|
||||
from core.download_plugins.base import DownloadSourcePlugin
|
||||
|
||||
|
||||
class QobuzClient(DownloadSourcePlugin):
|
||||
"""
|
||||
Qobuz download client using Qobuz REST API.
|
||||
Provides search, matching, and download capabilities as a drop-in alternative to Soulseek/YouTube/Tidal.
|
||||
|
|
@ -136,13 +139,19 @@ class QobuzClient:
|
|||
self.user_info: Optional[Dict] = None
|
||||
self._auth_error: Optional[str] = None
|
||||
|
||||
# Download queue management
|
||||
self.active_downloads: Dict[str, Dict[str, Any]] = {}
|
||||
self._download_lock = threading.Lock()
|
||||
# Engine reference is populated by set_engine() at registration
|
||||
# time. None until orchestrator wires the registry.
|
||||
self._engine = None
|
||||
|
||||
# Try to restore saved session
|
||||
self._restore_session()
|
||||
|
||||
def set_engine(self, engine):
|
||||
"""Engine callback — gives the client access to the central
|
||||
thread worker + state store. Engine calls this during
|
||||
``register_plugin`` if the plugin defines it."""
|
||||
self._engine = engine
|
||||
|
||||
def set_shutdown_check(self, check_callable):
|
||||
"""Set a callback function to check for system shutdown"""
|
||||
self.shutdown_check = check_callable
|
||||
|
|
@ -520,7 +529,7 @@ class QobuzClient:
|
|||
"""Pull session state from config without making a network probe.
|
||||
|
||||
SoulSync runs two ``QobuzClient`` instances side by side — one wired
|
||||
through ``soulseek_client.qobuz`` for the auth-flow endpoints, and a
|
||||
through ``soulseek_client.client('qobuz')`` for the auth-flow endpoints, and a
|
||||
second owned by the enrichment worker for thread safety. When the user
|
||||
logs in via ``/api/qobuz/auth/login`` or ``/api/qobuz/auth/token`` only
|
||||
the auth-flow instance's in-memory state is updated; the worker's
|
||||
|
|
@ -884,97 +893,38 @@ class QobuzClient:
|
|||
return None
|
||||
|
||||
async def download(self, username: str, filename: str, file_size: int = 0) -> Optional[str]:
|
||||
"""
|
||||
Download a Qobuz track (async, Soulseek-compatible interface).
|
||||
"""Download a Qobuz track. Delegates to engine.worker which
|
||||
spawns the background thread + manages state."""
|
||||
if '||' not in filename:
|
||||
logger.error(f"Invalid filename format: {filename}")
|
||||
return None
|
||||
if self._engine is None:
|
||||
# Raise rather than return None so the orchestrator's
|
||||
# download_with_fallback surfaces a real warning + tries
|
||||
# the next source. Returning None silently dropped the
|
||||
# download with no user feedback (per JohnBaumb).
|
||||
raise RuntimeError("Qobuz client has no engine reference — cannot dispatch download")
|
||||
|
||||
Returns download_id immediately and runs download in background thread.
|
||||
|
||||
Args:
|
||||
username: Ignored for Qobuz (always "qobuz")
|
||||
filename: Encoded as "track_id||display_name"
|
||||
file_size: Ignored
|
||||
"""
|
||||
track_id_str, display_name = filename.split('||', 1)
|
||||
try:
|
||||
if '||' not in filename:
|
||||
logger.error(f"Invalid filename format: {filename}")
|
||||
return None
|
||||
|
||||
track_id_str, display_name = filename.split('||', 1)
|
||||
try:
|
||||
track_id = int(track_id_str)
|
||||
except ValueError:
|
||||
logger.error(f"Invalid Qobuz track ID: {track_id_str}")
|
||||
return None
|
||||
|
||||
logger.info(f"Starting Qobuz download: {display_name}")
|
||||
|
||||
download_id = str(uuid.uuid4())
|
||||
|
||||
with self._download_lock:
|
||||
self.active_downloads[download_id] = {
|
||||
'id': download_id,
|
||||
'filename': filename,
|
||||
'username': 'qobuz',
|
||||
'state': 'Initializing',
|
||||
'progress': 0.0,
|
||||
'size': 0,
|
||||
'transferred': 0,
|
||||
'speed': 0,
|
||||
'time_remaining': None,
|
||||
'track_id': track_id,
|
||||
'display_name': display_name,
|
||||
'file_path': None,
|
||||
}
|
||||
|
||||
# Start download in background thread
|
||||
download_thread = threading.Thread(
|
||||
target=self._download_thread_worker,
|
||||
args=(download_id, track_id, display_name, filename),
|
||||
daemon=True,
|
||||
)
|
||||
download_thread.start()
|
||||
|
||||
logger.info(f"Qobuz download {download_id} started in background")
|
||||
return download_id
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to start Qobuz download: {e}")
|
||||
import traceback
|
||||
traceback.print_exc()
|
||||
track_id = int(track_id_str)
|
||||
except ValueError:
|
||||
logger.error(f"Invalid Qobuz track ID: {track_id_str}")
|
||||
return None
|
||||
|
||||
def _download_thread_worker(self, download_id: str, track_id: int, display_name: str, original_filename: str):
|
||||
"""Background thread worker for downloading Qobuz tracks."""
|
||||
try:
|
||||
with self._download_lock:
|
||||
if download_id in self.active_downloads:
|
||||
self.active_downloads[download_id]['state'] = 'InProgress, Downloading'
|
||||
logger.info(f"Starting Qobuz download: {display_name}")
|
||||
|
||||
file_path = self._download_sync(download_id, track_id, display_name)
|
||||
|
||||
if file_path:
|
||||
with self._download_lock:
|
||||
if download_id in self.active_downloads:
|
||||
self.active_downloads[download_id]['state'] = 'Completed, Succeeded'
|
||||
self.active_downloads[download_id]['progress'] = 100.0
|
||||
self.active_downloads[download_id]['file_path'] = file_path
|
||||
|
||||
logger.info(f"Qobuz download {download_id} completed: {file_path}")
|
||||
else:
|
||||
with self._download_lock:
|
||||
if download_id in self.active_downloads:
|
||||
self.active_downloads[download_id]['state'] = 'Errored'
|
||||
|
||||
logger.error(f"Qobuz download {download_id} failed")
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Qobuz download thread failed for {download_id}: {e}")
|
||||
import traceback
|
||||
traceback.print_exc()
|
||||
|
||||
with self._download_lock:
|
||||
if download_id in self.active_downloads:
|
||||
self.active_downloads[download_id]['state'] = 'Errored'
|
||||
return self._engine.worker.dispatch(
|
||||
source_name='qobuz',
|
||||
target_id=track_id,
|
||||
display_name=display_name,
|
||||
original_filename=filename,
|
||||
impl_callable=self._download_sync,
|
||||
extra_record_fields={
|
||||
'track_id': track_id,
|
||||
'display_name': display_name,
|
||||
},
|
||||
)
|
||||
|
||||
def _download_sync(self, download_id: str, track_id: int, display_name: str) -> Optional[str]:
|
||||
"""
|
||||
|
|
@ -1050,9 +1000,8 @@ class QobuzClient:
|
|||
chunk_size = 64 * 1024 # 64KB chunks
|
||||
start_time = time.time()
|
||||
|
||||
with self._download_lock:
|
||||
if download_id in self.active_downloads:
|
||||
self.active_downloads[download_id]['size'] = total_size
|
||||
if self._engine is not None:
|
||||
self._engine.update_record('qobuz', download_id, {'size': total_size})
|
||||
|
||||
with open(out_path, 'wb') as f:
|
||||
for chunk in response.iter_content(chunk_size=chunk_size):
|
||||
|
|
@ -1061,9 +1010,10 @@ class QobuzClient:
|
|||
|
||||
# Check for shutdown or cancellation
|
||||
cancelled = False
|
||||
with self._download_lock:
|
||||
if download_id in self.active_downloads:
|
||||
cancelled = self.active_downloads[download_id].get('state') == 'Cancelled'
|
||||
if self._engine is not None:
|
||||
rec = self._engine.get_record('qobuz', download_id)
|
||||
if rec is not None:
|
||||
cancelled = rec.get('state') == 'Cancelled'
|
||||
if cancelled or (self.shutdown_check and self.shutdown_check()):
|
||||
reason = "cancelled" if cancelled else "server shutting down"
|
||||
logger.info(f"Aborting Qobuz download mid-stream: {reason}")
|
||||
|
|
@ -1084,18 +1034,20 @@ class QobuzClient:
|
|||
progress = 0
|
||||
time_remaining = None
|
||||
|
||||
with self._download_lock:
|
||||
if download_id in self.active_downloads:
|
||||
self.active_downloads[download_id]['transferred'] = downloaded
|
||||
self.active_downloads[download_id]['progress'] = round(progress, 1)
|
||||
self.active_downloads[download_id]['speed'] = int(speed)
|
||||
self.active_downloads[download_id]['time_remaining'] = time_remaining
|
||||
if self._engine is not None:
|
||||
self._engine.update_record('qobuz', download_id, {
|
||||
'transferred': downloaded,
|
||||
'progress': round(progress, 1),
|
||||
'speed': int(speed),
|
||||
'time_remaining': time_remaining,
|
||||
})
|
||||
|
||||
# If download was aborted (shutdown/cancel), clean up partial file
|
||||
abort_check = False
|
||||
with self._download_lock:
|
||||
if download_id in self.active_downloads:
|
||||
abort_check = self.active_downloads[download_id].get('state') == 'Cancelled'
|
||||
if self._engine is not None:
|
||||
rec = self._engine.get_record('qobuz', download_id)
|
||||
if rec is not None:
|
||||
abort_check = rec.get('state') == 'Cancelled'
|
||||
if abort_check or (self.shutdown_check and self.shutdown_check()):
|
||||
out_path.unlink(missing_ok=True)
|
||||
return None
|
||||
|
|
@ -1139,79 +1091,55 @@ class QobuzClient:
|
|||
|
||||
# ===================== Status / Cancel / Clear =====================
|
||||
|
||||
def _record_to_status(self, record):
|
||||
return DownloadStatus(
|
||||
id=record['id'],
|
||||
filename=record['filename'],
|
||||
username=record['username'],
|
||||
state=record['state'],
|
||||
progress=record['progress'],
|
||||
size=record.get('size', 0),
|
||||
transferred=record.get('transferred', 0),
|
||||
speed=record.get('speed', 0),
|
||||
time_remaining=record.get('time_remaining'),
|
||||
file_path=record.get('file_path'),
|
||||
)
|
||||
|
||||
async def get_all_downloads(self) -> List[DownloadStatus]:
|
||||
"""Get all active downloads (matches Soulseek interface)."""
|
||||
download_statuses = []
|
||||
|
||||
with self._download_lock:
|
||||
for _download_id, info in self.active_downloads.items():
|
||||
status = DownloadStatus(
|
||||
id=info['id'],
|
||||
filename=info['filename'],
|
||||
username=info['username'],
|
||||
state=info['state'],
|
||||
progress=info['progress'],
|
||||
size=info['size'],
|
||||
transferred=info['transferred'],
|
||||
speed=info['speed'],
|
||||
time_remaining=info.get('time_remaining'),
|
||||
file_path=info.get('file_path'),
|
||||
)
|
||||
download_statuses.append(status)
|
||||
|
||||
return download_statuses
|
||||
if self._engine is None:
|
||||
return []
|
||||
return [
|
||||
self._record_to_status(record)
|
||||
for record in self._engine.iter_records_for_source('qobuz')
|
||||
]
|
||||
|
||||
async def get_download_status(self, download_id: str) -> Optional[DownloadStatus]:
|
||||
"""Get status of a specific download (matches Soulseek interface)."""
|
||||
with self._download_lock:
|
||||
if download_id not in self.active_downloads:
|
||||
return None
|
||||
|
||||
info = self.active_downloads[download_id]
|
||||
return DownloadStatus(
|
||||
id=info['id'],
|
||||
filename=info['filename'],
|
||||
username=info['username'],
|
||||
state=info['state'],
|
||||
progress=info['progress'],
|
||||
size=info['size'],
|
||||
transferred=info['transferred'],
|
||||
speed=info['speed'],
|
||||
time_remaining=info.get('time_remaining'),
|
||||
file_path=info.get('file_path'),
|
||||
)
|
||||
if self._engine is None:
|
||||
return None
|
||||
record = self._engine.get_record('qobuz', download_id)
|
||||
return self._record_to_status(record) if record is not None else None
|
||||
|
||||
async def cancel_download(self, download_id: str, username: str = None, remove: bool = False) -> bool:
|
||||
"""Cancel an active download (matches Soulseek interface)."""
|
||||
try:
|
||||
with self._download_lock:
|
||||
if download_id not in self.active_downloads:
|
||||
logger.warning(f"Download {download_id} not found")
|
||||
return False
|
||||
|
||||
self.active_downloads[download_id]['state'] = 'Cancelled'
|
||||
logger.info(f"Marked Qobuz download {download_id} as cancelled")
|
||||
|
||||
if remove:
|
||||
del self.active_downloads[download_id]
|
||||
logger.info(f"Removed Qobuz download {download_id} from queue")
|
||||
|
||||
return True
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to cancel download {download_id}: {e}")
|
||||
if self._engine is None:
|
||||
return False
|
||||
if self._engine.get_record('qobuz', download_id) is None:
|
||||
logger.warning(f"Qobuz download {download_id} not found")
|
||||
return False
|
||||
self._engine.update_record('qobuz', download_id, {'state': 'Cancelled'})
|
||||
logger.info(f"Marked Qobuz download {download_id} as cancelled")
|
||||
if remove:
|
||||
self._engine.remove_record('qobuz', download_id)
|
||||
logger.info(f"Removed Qobuz download {download_id} from queue")
|
||||
return True
|
||||
|
||||
async def clear_all_completed_downloads(self) -> bool:
|
||||
"""Clear all terminal downloads from the list (matches Soulseek interface)."""
|
||||
if self._engine is None:
|
||||
return True
|
||||
try:
|
||||
with self._download_lock:
|
||||
ids_to_remove = [
|
||||
did for did, info in self.active_downloads.items()
|
||||
if info.get('state', '') in ('Completed, Succeeded', 'Cancelled', 'Errored', 'Aborted')
|
||||
]
|
||||
for did in ids_to_remove:
|
||||
del self.active_downloads[did]
|
||||
|
||||
terminal = {'Completed, Succeeded', 'Cancelled', 'Errored', 'Aborted'}
|
||||
for record in list(self._engine.iter_records_for_source('qobuz')):
|
||||
if record.get('state') in terminal:
|
||||
self._engine.remove_record('qobuz', record['id'])
|
||||
return True
|
||||
except Exception as e:
|
||||
logger.error(f"Error clearing downloads: {e}")
|
||||
|
|
|
|||
|
|
@ -14,7 +14,7 @@ logger = logging.getLogger(__name__)
|
|||
|
||||
def run_basic_soulseek_search(
|
||||
query: str,
|
||||
soulseek_client,
|
||||
download_orchestrator,
|
||||
run_async: Callable,
|
||||
) -> list[dict]:
|
||||
"""Search Soulseek for `query`, normalize albums + tracks to one sorted list.
|
||||
|
|
@ -22,7 +22,7 @@ def run_basic_soulseek_search(
|
|||
Returns dicts with `result_type` set to "album" or "track" and sorted by
|
||||
`quality_score` descending. Empty list on any failure (caller logs).
|
||||
"""
|
||||
tracks, albums = run_async(soulseek_client.search(query))
|
||||
tracks, albums = run_async(download_orchestrator.search(query))
|
||||
|
||||
processed_albums = []
|
||||
for album in albums:
|
||||
|
|
|
|||
|
|
@ -49,7 +49,7 @@ class SearchDeps:
|
|||
spotify_client: Any
|
||||
hydrabase_client: Any
|
||||
hydrabase_worker: Any
|
||||
soulseek_client: Any
|
||||
download_orchestrator: Any
|
||||
fix_artist_image_url: Callable[[Optional[str]], Optional[str]]
|
||||
is_hydrabase_active: Callable[[], bool]
|
||||
get_metadata_fallback_source: Callable[[], str]
|
||||
|
|
@ -289,10 +289,11 @@ def run_enhanced_search(query: str, requested_source: str, deps: SearchDeps) ->
|
|||
# ---------------------------------------------------------------------------
|
||||
|
||||
def resolve_youtube_videos_client(deps: SearchDeps):
|
||||
"""Return the soulseek_client.youtube subclient or None when unavailable."""
|
||||
if not deps.soulseek_client:
|
||||
"""Return the YouTube download client (used for music-video search)
|
||||
via the orchestrator's generic accessor, or None when unavailable."""
|
||||
if not deps.download_orchestrator or not hasattr(deps.download_orchestrator, 'client'):
|
||||
return None
|
||||
return getattr(deps.soulseek_client, 'youtube', None)
|
||||
return deps.download_orchestrator.client('youtube')
|
||||
|
||||
|
||||
def stream_youtube_videos(query: str, youtube_client, run_async: Callable) -> Iterator[str]:
|
||||
|
|
|
|||
|
|
@ -96,7 +96,7 @@ def stream_search_track(
|
|||
album_name: Optional[str],
|
||||
duration_ms: int,
|
||||
config_manager,
|
||||
soulseek_client,
|
||||
download_orchestrator,
|
||||
matching_engine,
|
||||
run_async: Callable,
|
||||
) -> Optional[dict]:
|
||||
|
|
@ -118,15 +118,9 @@ def stream_search_track(
|
|||
|
||||
queries = _build_stream_queries(track_name, artist_name, effective_mode)
|
||||
|
||||
stream_clients = {
|
||||
'youtube': soulseek_client.youtube,
|
||||
'tidal': soulseek_client.tidal,
|
||||
'qobuz': soulseek_client.qobuz,
|
||||
'hifi': soulseek_client.hifi,
|
||||
'deezer_dl': soulseek_client.deezer_dl,
|
||||
'lidarr': soulseek_client.lidarr,
|
||||
}
|
||||
stream_client = stream_clients.get(effective_mode)
|
||||
# Map mode name to canonical registry name (legacy 'deezer_dl'
|
||||
# alias resolves to 'deezer' via the orchestrator's registry).
|
||||
stream_client = download_orchestrator.client(effective_mode)
|
||||
use_direct_client = stream_client is not None
|
||||
|
||||
max_peer_queue = config_manager.get('soulseek.max_peer_queue', 0) or 0
|
||||
|
|
@ -137,7 +131,7 @@ def stream_search_track(
|
|||
if use_direct_client:
|
||||
tracks_result, _ = run_async(stream_client.search(query, timeout=15))
|
||||
else:
|
||||
tracks_result, _ = run_async(soulseek_client.search(query, timeout=15))
|
||||
tracks_result, _ = run_async(download_orchestrator.search(query, timeout=15))
|
||||
|
||||
if not tracks_result:
|
||||
logger.info(f"No results for query '{query}', trying next...")
|
||||
|
|
|
|||
|
|
@ -9,186 +9,21 @@ from pathlib import Path
|
|||
from utils.logging_config import get_logger
|
||||
from config.settings import config_manager
|
||||
from core.imports.filename import parse_filename_metadata
|
||||
# Shared download-result dataclasses + plugin contract live in the
|
||||
# neutral plugin package — every source uses the same types, so they
|
||||
# belong there rather than this soulseek-specific module.
|
||||
from core.download_plugins.types import (
|
||||
AlbumResult,
|
||||
DownloadStatus,
|
||||
SearchResult,
|
||||
TrackResult,
|
||||
)
|
||||
from core.download_plugins.base import DownloadSourcePlugin
|
||||
|
||||
logger = get_logger("soulseek_client")
|
||||
|
||||
@dataclass
|
||||
class SearchResult:
|
||||
"""Base class for search results"""
|
||||
username: str
|
||||
filename: str
|
||||
size: int
|
||||
bitrate: Optional[int]
|
||||
duration: Optional[int] # Duration in milliseconds (converted from slskd's seconds)
|
||||
quality: str
|
||||
free_upload_slots: int
|
||||
upload_speed: int
|
||||
queue_length: int
|
||||
result_type: str = "track" # "track" or "album"
|
||||
|
||||
@property
|
||||
def quality_score(self) -> float:
|
||||
quality_weights = {
|
||||
'flac': 1.0,
|
||||
'mp3': 0.8,
|
||||
'ogg': 0.7,
|
||||
'aac': 0.6,
|
||||
'wma': 0.5
|
||||
}
|
||||
|
||||
base_score = quality_weights.get(self.quality.lower(), 0.3)
|
||||
|
||||
if self.bitrate:
|
||||
if self.bitrate >= 320:
|
||||
base_score += 0.2
|
||||
elif self.bitrate >= 256:
|
||||
base_score += 0.1
|
||||
elif self.bitrate < 128:
|
||||
base_score -= 0.2
|
||||
|
||||
# Free upload slots
|
||||
if self.free_upload_slots == 0:
|
||||
base_score -= 0.15
|
||||
elif self.free_upload_slots > 0:
|
||||
base_score += 0.05
|
||||
|
||||
# Upload speed in bytes/sec (tiered)
|
||||
if self.upload_speed >= 5_000_000: # ~5 MB/s / 40 Mbps
|
||||
base_score += 0.15
|
||||
elif self.upload_speed >= 1_000_000: # ~1 MB/s / 8 Mbps
|
||||
base_score += 0.10
|
||||
elif self.upload_speed >= 500_000: # ~500 KB/s / 4 Mbps
|
||||
base_score += 0.05
|
||||
elif self.upload_speed < 100_000: # ~100 KB/s / 800 kbps
|
||||
base_score -= 0.05
|
||||
|
||||
# Queue length (graduated penalty)
|
||||
if self.queue_length > 50:
|
||||
base_score -= 0.25
|
||||
elif self.queue_length > 20:
|
||||
base_score -= 0.15
|
||||
elif self.queue_length > 10:
|
||||
base_score -= 0.10
|
||||
|
||||
return min(base_score, 1.0)
|
||||
|
||||
@dataclass
|
||||
class TrackResult(SearchResult):
|
||||
"""Individual track search result"""
|
||||
artist: Optional[str] = None
|
||||
title: Optional[str] = None
|
||||
album: Optional[str] = None
|
||||
track_number: Optional[int] = None
|
||||
_source_metadata: Optional[Dict[str, Any]] = None
|
||||
|
||||
def __post_init__(self):
|
||||
self.result_type = "track"
|
||||
# Try to extract metadata from filename if not provided
|
||||
if not self.title or not self.artist:
|
||||
self._parse_filename_metadata()
|
||||
|
||||
def _parse_filename_metadata(self):
|
||||
"""Extract artist, title, album from filename patterns"""
|
||||
parsed = parse_filename_metadata(self.filename)
|
||||
if not self.artist and parsed.get("artist"):
|
||||
self.artist = parsed["artist"]
|
||||
if not self.title and parsed.get("title"):
|
||||
self.title = parsed["title"]
|
||||
if not self.album and parsed.get("album"):
|
||||
self.album = parsed["album"]
|
||||
if self.track_number is None:
|
||||
track_number = parsed.get("track_number")
|
||||
if track_number is not None:
|
||||
self.track_number = track_number
|
||||
|
||||
@dataclass
|
||||
class AlbumResult:
|
||||
"""Album/folder search result containing multiple tracks"""
|
||||
username: str
|
||||
album_path: str # Directory path
|
||||
album_title: str
|
||||
artist: Optional[str]
|
||||
track_count: int
|
||||
total_size: int
|
||||
tracks: List[TrackResult]
|
||||
dominant_quality: str # Most common quality in album
|
||||
year: Optional[str] = None
|
||||
free_upload_slots: int = 0
|
||||
upload_speed: int = 0
|
||||
queue_length: int = 0
|
||||
result_type: str = "album"
|
||||
|
||||
@property
|
||||
def quality_score(self) -> float:
|
||||
"""Calculate album quality score based on dominant quality and track count"""
|
||||
quality_weights = {
|
||||
'flac': 1.0,
|
||||
'mp3': 0.8,
|
||||
'ogg': 0.7,
|
||||
'aac': 0.6,
|
||||
'wma': 0.5
|
||||
}
|
||||
|
||||
base_score = quality_weights.get(self.dominant_quality.lower(), 0.3)
|
||||
|
||||
# Bonus for complete albums (typically 8-15 tracks)
|
||||
if 8 <= self.track_count <= 20:
|
||||
base_score += 0.1
|
||||
elif self.track_count > 20:
|
||||
base_score += 0.05
|
||||
|
||||
# Free upload slots
|
||||
if self.free_upload_slots == 0:
|
||||
base_score -= 0.15
|
||||
elif self.free_upload_slots > 0:
|
||||
base_score += 0.05
|
||||
|
||||
# Upload speed in bytes/sec (tiered)
|
||||
if self.upload_speed >= 5_000_000: # ~5 MB/s / 40 Mbps
|
||||
base_score += 0.15
|
||||
elif self.upload_speed >= 1_000_000: # ~1 MB/s / 8 Mbps
|
||||
base_score += 0.10
|
||||
elif self.upload_speed >= 500_000: # ~500 KB/s / 4 Mbps
|
||||
base_score += 0.05
|
||||
elif self.upload_speed < 100_000: # ~100 KB/s / 800 kbps
|
||||
base_score -= 0.05
|
||||
|
||||
# Queue length (graduated penalty)
|
||||
if self.queue_length > 50:
|
||||
base_score -= 0.25
|
||||
elif self.queue_length > 20:
|
||||
base_score -= 0.15
|
||||
elif self.queue_length > 10:
|
||||
base_score -= 0.10
|
||||
|
||||
return min(base_score, 1.0)
|
||||
|
||||
@property
|
||||
def size_mb(self) -> int:
|
||||
"""Album size in MB"""
|
||||
return self.total_size // (1024 * 1024)
|
||||
|
||||
@property
|
||||
def average_track_size_mb(self) -> float:
|
||||
"""Average track size in MB"""
|
||||
if self.track_count > 0:
|
||||
return self.size_mb / self.track_count
|
||||
return 0
|
||||
|
||||
@dataclass
|
||||
class DownloadStatus:
|
||||
id: str
|
||||
filename: str
|
||||
username: str
|
||||
state: str
|
||||
progress: float
|
||||
size: int
|
||||
transferred: int
|
||||
speed: int
|
||||
time_remaining: Optional[int] = None
|
||||
file_path: Optional[str] = None
|
||||
|
||||
class SoulseekClient:
|
||||
class SoulseekClient(DownloadSourcePlugin):
|
||||
def __init__(self):
|
||||
self.base_url: Optional[str] = None
|
||||
self.api_key: Optional[str] = None
|
||||
|
|
|
|||
|
|
@ -27,7 +27,6 @@ import re
|
|||
import asyncio
|
||||
import uuid
|
||||
import time
|
||||
import threading
|
||||
from typing import List, Optional, Dict, Any, Tuple, Callable
|
||||
from pathlib import Path
|
||||
|
||||
|
|
@ -41,7 +40,7 @@ from config.settings import config_manager
|
|||
|
||||
# Standard data structures shared across all download clients so downstream
|
||||
# matching/post-processing stays source-agnostic.
|
||||
from core.soulseek_client import TrackResult, AlbumResult, DownloadStatus
|
||||
from core.download_plugins.types import TrackResult, AlbumResult, DownloadStatus
|
||||
|
||||
logger = get_logger("soundcloud_client")
|
||||
|
||||
|
|
@ -83,7 +82,10 @@ def _sanitize_filename(name: str) -> str:
|
|||
return cleaned or 'soundcloud_track'
|
||||
|
||||
|
||||
class SoundcloudClient:
|
||||
from core.download_plugins.base import DownloadSourcePlugin
|
||||
|
||||
|
||||
class SoundcloudClient(DownloadSourcePlugin):
|
||||
"""SoundCloud download client built on yt-dlp's SoundCloud extractor.
|
||||
|
||||
Mirrors the public surface of TidalDownloadClient / QobuzClient so the
|
||||
|
|
@ -106,13 +108,16 @@ class SoundcloudClient:
|
|||
# in-flight downloads when the worker is shutting down.
|
||||
self.shutdown_check: Optional[Callable[[], bool]] = None
|
||||
|
||||
self.active_downloads: Dict[str, Dict[str, Any]] = {}
|
||||
self._download_lock = threading.Lock()
|
||||
self._engine = None
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Lifecycle / availability
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def set_engine(self, engine):
|
||||
"""Engine callback — wires the central thread worker + state store."""
|
||||
self._engine = engine
|
||||
|
||||
def set_shutdown_check(self, check_callable: Optional[Callable[[], bool]]) -> None:
|
||||
self.shutdown_check = check_callable
|
||||
|
||||
|
|
@ -316,98 +321,46 @@ class SoundcloudClient:
|
|||
# ------------------------------------------------------------------
|
||||
|
||||
async def download(self, username: str, filename: str, file_size: int = 0) -> Optional[str]:
|
||||
"""Kick off a SoundCloud download in a background thread.
|
||||
|
||||
Returns the internal download_id used by status/cancel calls,
|
||||
matching the contract of every other download client.
|
||||
"""
|
||||
try:
|
||||
parts = filename.split('||', 2)
|
||||
if len(parts) < 2:
|
||||
logger.error(f"Invalid SoundCloud filename format: {filename}")
|
||||
return None
|
||||
|
||||
sc_track_id = parts[0]
|
||||
permalink_url = parts[1]
|
||||
display_name = parts[2] if len(parts) > 2 else sc_track_id
|
||||
|
||||
if not sc_track_id or not permalink_url:
|
||||
logger.error(f"Missing SoundCloud track id or url in: {filename}")
|
||||
return None
|
||||
|
||||
logger.info(f"Starting SoundCloud download: {display_name}")
|
||||
|
||||
download_id = str(uuid.uuid4())
|
||||
|
||||
with self._download_lock:
|
||||
self.active_downloads[download_id] = {
|
||||
'id': download_id,
|
||||
'filename': filename,
|
||||
'username': 'soundcloud',
|
||||
'state': 'Initializing',
|
||||
'progress': 0.0,
|
||||
'size': 0,
|
||||
'transferred': 0,
|
||||
'speed': 0,
|
||||
'time_remaining': None,
|
||||
'track_id': sc_track_id,
|
||||
'permalink_url': permalink_url,
|
||||
'display_name': display_name,
|
||||
'file_path': None,
|
||||
}
|
||||
|
||||
download_thread = threading.Thread(
|
||||
target=self._download_thread_worker,
|
||||
args=(download_id, permalink_url, display_name, filename),
|
||||
daemon=True,
|
||||
)
|
||||
download_thread.start()
|
||||
|
||||
logger.info(f"SoundCloud download {download_id} started in background")
|
||||
return download_id
|
||||
|
||||
except Exception as exc:
|
||||
logger.error(f"Failed to start SoundCloud download: {exc}")
|
||||
import traceback
|
||||
traceback.print_exc()
|
||||
"""Kick off a SoundCloud download via engine.worker."""
|
||||
parts = filename.split('||', 2)
|
||||
if len(parts) < 2:
|
||||
logger.error(f"Invalid SoundCloud filename format: {filename}")
|
||||
return None
|
||||
|
||||
def _download_thread_worker(self, download_id: str, permalink_url: str,
|
||||
display_name: str, original_filename: str) -> None:
|
||||
"""Background-thread wrapper around `_download_sync`.
|
||||
sc_track_id = parts[0]
|
||||
permalink_url = parts[1]
|
||||
display_name = parts[2] if len(parts) > 2 else sc_track_id
|
||||
|
||||
Owns the state transitions on `self.active_downloads[...]` so the
|
||||
sync impl can just return a path / None and not worry about state.
|
||||
"""
|
||||
try:
|
||||
with self._download_lock:
|
||||
if download_id in self.active_downloads:
|
||||
self.active_downloads[download_id]['state'] = 'InProgress, Downloading'
|
||||
if not sc_track_id or not permalink_url:
|
||||
logger.error(f"Missing SoundCloud track id or url in: {filename}")
|
||||
return None
|
||||
if self._engine is None:
|
||||
# Raise rather than return None so the orchestrator's
|
||||
# download_with_fallback surfaces a real warning + tries
|
||||
# the next source. Returning None silently dropped the
|
||||
# download with no user feedback (per JohnBaumb).
|
||||
raise RuntimeError("SoundCloud client has no engine reference — cannot dispatch download")
|
||||
|
||||
file_path = self._download_sync(download_id, permalink_url, display_name)
|
||||
logger.info(f"Starting SoundCloud download: {display_name}")
|
||||
|
||||
if file_path:
|
||||
with self._download_lock:
|
||||
if download_id in self.active_downloads:
|
||||
self.active_downloads[download_id]['state'] = 'Completed, Succeeded'
|
||||
self.active_downloads[download_id]['progress'] = 100.0
|
||||
self.active_downloads[download_id]['file_path'] = file_path
|
||||
logger.info(f"SoundCloud download {download_id} completed: {file_path}")
|
||||
else:
|
||||
with self._download_lock:
|
||||
if download_id in self.active_downloads:
|
||||
# Don't clobber an explicit Cancelled state with Errored.
|
||||
if self.active_downloads[download_id]['state'] != 'Cancelled':
|
||||
self.active_downloads[download_id]['state'] = 'Errored'
|
||||
logger.error(f"SoundCloud download {download_id} failed")
|
||||
# Worker passes (download_id, target_id, display_name) to impl;
|
||||
# SoundCloud's _download_sync wants permalink_url (not track_id),
|
||||
# so adapt by closing over permalink_url here.
|
||||
def _impl(download_id, _target_id, _display_name):
|
||||
return self._download_sync(download_id, permalink_url, display_name)
|
||||
|
||||
except Exception as exc:
|
||||
logger.error(f"SoundCloud download thread failed for {download_id}: {exc}")
|
||||
import traceback
|
||||
traceback.print_exc()
|
||||
with self._download_lock:
|
||||
if download_id in self.active_downloads:
|
||||
self.active_downloads[download_id]['state'] = 'Errored'
|
||||
return self._engine.worker.dispatch(
|
||||
source_name='soundcloud',
|
||||
target_id=sc_track_id,
|
||||
display_name=display_name,
|
||||
original_filename=filename,
|
||||
impl_callable=_impl,
|
||||
extra_record_fields={
|
||||
'track_id': sc_track_id,
|
||||
'permalink_url': permalink_url,
|
||||
'display_name': display_name,
|
||||
},
|
||||
)
|
||||
|
||||
def _download_sync(self, download_id: str, permalink_url: str,
|
||||
display_name: str) -> Optional[str]:
|
||||
|
|
@ -524,162 +477,128 @@ class SoundcloudClient:
|
|||
def _update_download_progress_fragmented(self, download_id: str, downloaded: int,
|
||||
fragment_index: int, fragment_count: int,
|
||||
speed_start: float) -> None:
|
||||
"""HLS-aware progress update.
|
||||
"""HLS-aware progress update — fragment_index / fragment_count
|
||||
gives an accurate signal even when each fragment's
|
||||
``total_bytes`` only describes the current fragment."""
|
||||
if self._engine is None:
|
||||
return
|
||||
record = self._engine.get_record('soundcloud', download_id)
|
||||
if record is None:
|
||||
return
|
||||
|
||||
SoundCloud's HLS streams arrive as N small fragments. yt-dlp can't
|
||||
compute a true byte-based percentage because each fragment's
|
||||
``total_bytes`` only describes the current fragment, not the
|
||||
whole download. fragment_index / fragment_count gives an accurate
|
||||
progress signal — N fragments downloaded out of M known fragments
|
||||
is the real ratio.
|
||||
"""
|
||||
with self._download_lock:
|
||||
if download_id not in self.active_downloads:
|
||||
return
|
||||
info = self.active_downloads[download_id]
|
||||
info['transferred'] = downloaded
|
||||
now = time.time()
|
||||
elapsed = now - speed_start
|
||||
speed = int(downloaded / elapsed) if elapsed > 0 else 0
|
||||
|
||||
now = time.time()
|
||||
elapsed = now - speed_start
|
||||
info['speed'] = int(downloaded / elapsed) if elapsed > 0 else 0
|
||||
progress = round(min((fragment_index / fragment_count) * 100, 99.9), 1) if fragment_count > 0 else 0.0
|
||||
|
||||
# `fragment_index` is 1-based when yt-dlp finishes a fragment;
|
||||
# use it directly. Cap below 100% so the worker thread owns the
|
||||
# final flip.
|
||||
progress = (fragment_index / fragment_count) * 100
|
||||
info['progress'] = round(min(progress, 99.9), 1)
|
||||
# Estimate total size from per-fragment average.
|
||||
if fragment_index > 0 and downloaded > 0:
|
||||
est_total = int(downloaded * (fragment_count / fragment_index))
|
||||
else:
|
||||
est_total = downloaded
|
||||
|
||||
# Estimate total size so the UI's bytes-remaining reads match
|
||||
# roughly what users expect — extrapolate from per-fragment
|
||||
# average. Defensive against fragment_index being 0 on the
|
||||
# very first callback.
|
||||
if fragment_index > 0 and downloaded > 0:
|
||||
est_total = int(downloaded * (fragment_count / fragment_index))
|
||||
info['size'] = est_total
|
||||
else:
|
||||
info['size'] = downloaded
|
||||
time_remaining: Optional[int] = None
|
||||
remaining_fragments = max(0, fragment_count - fragment_index)
|
||||
if speed > 0 and remaining_fragments > 0 and fragment_index > 0:
|
||||
seconds_per_fragment = elapsed / fragment_index if fragment_index > 0 else 0
|
||||
time_remaining = int(remaining_fragments * seconds_per_fragment)
|
||||
|
||||
time_remaining: Optional[int] = None
|
||||
remaining_fragments = max(0, fragment_count - fragment_index)
|
||||
if info['speed'] > 0 and remaining_fragments > 0 and fragment_index > 0:
|
||||
# Extrapolate seconds from per-fragment average download time.
|
||||
seconds_per_fragment = elapsed / fragment_index if fragment_index > 0 else 0
|
||||
time_remaining = int(remaining_fragments * seconds_per_fragment)
|
||||
info['time_remaining'] = time_remaining
|
||||
self._engine.update_record('soundcloud', download_id, {
|
||||
'transferred': downloaded,
|
||||
'speed': speed,
|
||||
'progress': progress,
|
||||
'size': est_total,
|
||||
'time_remaining': time_remaining,
|
||||
})
|
||||
|
||||
def _update_download_progress(self, download_id: str, downloaded: int,
|
||||
total: int, speed_start: float) -> None:
|
||||
"""Push a progress update into the active_downloads ledger.
|
||||
"""Byte-based progress update for non-HLS streams."""
|
||||
if self._engine is None:
|
||||
return
|
||||
record = self._engine.get_record('soundcloud', download_id)
|
||||
if record is None:
|
||||
return
|
||||
|
||||
Mirrors the structure other download clients populate so the
|
||||
existing /api/downloads endpoint can serialize it without caring
|
||||
about the source.
|
||||
"""
|
||||
with self._download_lock:
|
||||
if download_id not in self.active_downloads:
|
||||
return
|
||||
info = self.active_downloads[download_id]
|
||||
info['transferred'] = downloaded
|
||||
info['size'] = total
|
||||
now = time.time()
|
||||
elapsed = now - speed_start
|
||||
speed = int(downloaded / elapsed) if elapsed > 0 else 0
|
||||
|
||||
now = time.time()
|
||||
elapsed = now - speed_start
|
||||
info['speed'] = int(downloaded / elapsed) if elapsed > 0 else 0
|
||||
progress = record.get('progress', 0.0)
|
||||
if total > 0:
|
||||
progress = round(min((downloaded / total) * 100, 99.9), 1)
|
||||
|
||||
if total > 0:
|
||||
progress = (downloaded / total) * 100
|
||||
# Cap pre-completion progress at 99.9% so the worker thread
|
||||
# owns the final flip to 100% / Completed.
|
||||
info['progress'] = round(min(progress, 99.9), 1)
|
||||
time_remaining: Optional[int] = None
|
||||
if speed > 0 and total > 0:
|
||||
remaining = total - downloaded
|
||||
if remaining > 0:
|
||||
time_remaining = int(remaining / speed)
|
||||
|
||||
time_remaining: Optional[int] = None
|
||||
if info['speed'] > 0 and total > 0:
|
||||
remaining = total - downloaded
|
||||
if remaining > 0:
|
||||
time_remaining = int(remaining / info['speed'])
|
||||
info['time_remaining'] = time_remaining
|
||||
self._engine.update_record('soundcloud', download_id, {
|
||||
'transferred': downloaded,
|
||||
'size': total,
|
||||
'speed': speed,
|
||||
'progress': progress,
|
||||
'time_remaining': time_remaining,
|
||||
})
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Status / cancellation
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def _record_to_status(self, record: dict) -> DownloadStatus:
|
||||
return DownloadStatus(
|
||||
id=record['id'],
|
||||
filename=record['filename'],
|
||||
username=record['username'],
|
||||
state=record['state'],
|
||||
progress=record['progress'],
|
||||
size=record.get('size', 0),
|
||||
transferred=record.get('transferred', 0),
|
||||
speed=record.get('speed', 0),
|
||||
time_remaining=record.get('time_remaining'),
|
||||
file_path=record.get('file_path'),
|
||||
)
|
||||
|
||||
async def get_all_downloads(self) -> List[DownloadStatus]:
|
||||
"""Snapshot every tracked download as DownloadStatus objects."""
|
||||
out: List[DownloadStatus] = []
|
||||
with self._download_lock:
|
||||
for _download_id, info in self.active_downloads.items():
|
||||
out.append(DownloadStatus(
|
||||
id=info['id'],
|
||||
filename=info['filename'],
|
||||
username=info['username'],
|
||||
state=info['state'],
|
||||
progress=info['progress'],
|
||||
size=info['size'],
|
||||
transferred=info['transferred'],
|
||||
speed=info['speed'],
|
||||
time_remaining=info.get('time_remaining'),
|
||||
file_path=info.get('file_path'),
|
||||
))
|
||||
return out
|
||||
if self._engine is None:
|
||||
return []
|
||||
return [
|
||||
self._record_to_status(record)
|
||||
for record in self._engine.iter_records_for_source('soundcloud')
|
||||
]
|
||||
|
||||
async def get_download_status(self, download_id: str) -> Optional[DownloadStatus]:
|
||||
with self._download_lock:
|
||||
info = self.active_downloads.get(download_id)
|
||||
if info is None:
|
||||
return None
|
||||
return DownloadStatus(
|
||||
id=info['id'],
|
||||
filename=info['filename'],
|
||||
username=info['username'],
|
||||
state=info['state'],
|
||||
progress=info['progress'],
|
||||
size=info['size'],
|
||||
transferred=info['transferred'],
|
||||
speed=info['speed'],
|
||||
time_remaining=info.get('time_remaining'),
|
||||
file_path=info.get('file_path'),
|
||||
)
|
||||
if self._engine is None:
|
||||
return None
|
||||
record = self._engine.get_record('soundcloud', download_id)
|
||||
return self._record_to_status(record) if record is not None else None
|
||||
|
||||
async def cancel_download(self, download_id: str, username: Optional[str] = None,
|
||||
remove: bool = False) -> bool:
|
||||
"""Mark a download as cancelled.
|
||||
|
||||
Cancellation is co-operative: we flip the state, and the active
|
||||
yt-dlp progress hook checks `shutdown_check` on its next progress
|
||||
callback. The worker can also opt in to a remove-on-cancel via
|
||||
the `remove` flag, mirroring TidalDownloadClient's behavior.
|
||||
"""
|
||||
try:
|
||||
with self._download_lock:
|
||||
info = self.active_downloads.get(download_id)
|
||||
if info is None:
|
||||
logger.warning(f"SoundCloud download {download_id} not found")
|
||||
return False
|
||||
|
||||
info['state'] = 'Cancelled'
|
||||
logger.info(f"Marked SoundCloud download {download_id} as cancelled")
|
||||
|
||||
if remove:
|
||||
del self.active_downloads[download_id]
|
||||
logger.info(f"Removed SoundCloud download {download_id} from queue")
|
||||
return True
|
||||
except Exception as exc:
|
||||
logger.error(f"Failed to cancel SoundCloud download {download_id}: {exc}")
|
||||
"""Mark a download as cancelled. Co-operative — yt-dlp's
|
||||
progress hook checks shutdown_check on next callback."""
|
||||
if self._engine is None:
|
||||
return False
|
||||
if self._engine.get_record('soundcloud', download_id) is None:
|
||||
logger.warning(f"SoundCloud download {download_id} not found")
|
||||
return False
|
||||
self._engine.update_record('soundcloud', download_id, {'state': 'Cancelled'})
|
||||
logger.info(f"Marked SoundCloud download {download_id} as cancelled")
|
||||
if remove:
|
||||
self._engine.remove_record('soundcloud', download_id)
|
||||
logger.info(f"Removed SoundCloud download {download_id} from queue")
|
||||
return True
|
||||
|
||||
async def clear_all_completed_downloads(self) -> bool:
|
||||
"""Drop terminal-state entries from the active_downloads ledger."""
|
||||
try:
|
||||
with self._download_lock:
|
||||
terminal_states = {'Completed, Succeeded', 'Cancelled', 'Errored', 'Aborted'}
|
||||
ids_to_remove = [
|
||||
did for did, info in self.active_downloads.items()
|
||||
if info.get('state', '') in terminal_states
|
||||
]
|
||||
for did in ids_to_remove:
|
||||
del self.active_downloads[did]
|
||||
logger.info(f"Cleared {len(ids_to_remove)} completed SoundCloud downloads")
|
||||
if self._engine is None:
|
||||
return True
|
||||
except Exception as exc:
|
||||
logger.error(f"Failed to clear SoundCloud downloads: {exc}")
|
||||
return False
|
||||
terminal = {'Completed, Succeeded', 'Cancelled', 'Errored', 'Aborted'}
|
||||
cleared = 0
|
||||
for record in list(self._engine.iter_records_for_source('soundcloud')):
|
||||
if record.get('state') in terminal:
|
||||
self._engine.remove_record('soundcloud', record['id'])
|
||||
cleared += 1
|
||||
logger.info(f"Cleared {cleared} completed SoundCloud downloads")
|
||||
return True
|
||||
|
|
|
|||
|
|
@ -7,9 +7,9 @@ it in the local Stream/ folder for the browser audio player.
|
|||
1. Reset stream state to 'loading' with the new track info.
|
||||
2. Clear any prior file from the Stream/ folder (only one stream lives
|
||||
there at a time).
|
||||
3. Spin up a fresh asyncio event loop and `soulseek_client.download()`
|
||||
3. Spin up a fresh asyncio event loop and `download_orchestrator.download()`
|
||||
the track.
|
||||
4. Poll `soulseek_client.get_all_downloads()` every 1.5 s to track
|
||||
4. Poll `download_orchestrator.get_all_downloads()` every 1.5 s to track
|
||||
progress, with separate handling for queued vs actively downloading
|
||||
states. Queue timeout = 15 s; overall timeout = 60 s.
|
||||
5. On completion (state ~ 'succeeded' or progress >= 100% AND bytes
|
||||
|
|
@ -45,7 +45,7 @@ logger = logging.getLogger(__name__)
|
|||
class PrepareStreamDeps:
|
||||
"""Bundle of cross-cutting deps the stream-prep worker needs."""
|
||||
config_manager: Any
|
||||
soulseek_client: Any
|
||||
download_orchestrator: Any
|
||||
stream_lock: Any # threading.Lock
|
||||
project_root: str # absolute path to web_server.py's directory
|
||||
docker_resolve_path: Callable[[str], str]
|
||||
|
|
@ -112,7 +112,7 @@ def prepare_stream_task(track_data, deps: PrepareStreamDeps):
|
|||
asyncio.set_event_loop(loop)
|
||||
|
||||
try:
|
||||
download_result = loop.run_until_complete(deps.soulseek_client.download(
|
||||
download_result = loop.run_until_complete(deps.download_orchestrator.download(
|
||||
track_data.get('username'),
|
||||
track_data.get('filename'),
|
||||
track_data.get('size', 0)
|
||||
|
|
@ -144,7 +144,7 @@ def prepare_stream_task(track_data, deps: PrepareStreamDeps):
|
|||
|
||||
try:
|
||||
# Use orchestrator's get_all_downloads() which works for both sources
|
||||
all_downloads = loop.run_until_complete(deps.soulseek_client.get_all_downloads())
|
||||
all_downloads = loop.run_until_complete(deps.download_orchestrator.get_all_downloads())
|
||||
download_status = deps.find_streaming_download_in_all_downloads(all_downloads, track_data)
|
||||
|
||||
if download_status:
|
||||
|
|
@ -252,7 +252,7 @@ def prepare_stream_task(track_data, deps: PrepareStreamDeps):
|
|||
download_id = download_status.get('id', '')
|
||||
if download_id and track_data.get('username'):
|
||||
success = loop.run_until_complete(
|
||||
deps.soulseek_client.signal_download_completion(
|
||||
deps.download_orchestrator.signal_download_completion(
|
||||
download_id, track_data.get('username'), remove=True)
|
||||
)
|
||||
if success:
|
||||
|
|
|
|||
|
|
@ -14,7 +14,6 @@ import re
|
|||
import asyncio
|
||||
import uuid
|
||||
import time
|
||||
import threading
|
||||
import shutil
|
||||
import subprocess
|
||||
from typing import List, Optional, Dict, Any, Tuple
|
||||
|
|
@ -33,7 +32,7 @@ from utils.logging_config import get_logger
|
|||
from config.settings import config_manager
|
||||
|
||||
# Import Soulseek data structures for drop-in replacement compatibility
|
||||
from core.soulseek_client import TrackResult, AlbumResult, DownloadStatus
|
||||
from core.download_plugins.types import TrackResult, AlbumResult, DownloadStatus
|
||||
|
||||
logger = get_logger("tidal_download_client")
|
||||
|
||||
|
|
@ -93,7 +92,10 @@ HLS_QUALITY_MAP = {
|
|||
HLS_MAP_TAG_RE = re.compile(r'#EXT-X-MAP:.*URI="([^"]+)"')
|
||||
|
||||
|
||||
class TidalDownloadClient:
|
||||
from core.download_plugins.base import DownloadSourcePlugin
|
||||
|
||||
|
||||
class TidalDownloadClient(DownloadSourcePlugin):
|
||||
"""
|
||||
Tidal download client using tidalapi.
|
||||
Provides search, matching, and download capabilities as a drop-in alternative to YouTube/Soulseek.
|
||||
|
|
@ -116,12 +118,21 @@ class TidalDownloadClient:
|
|||
self.session: Optional['tidalapi.Session'] = None
|
||||
self._init_session()
|
||||
|
||||
self.active_downloads: Dict[str, Dict[str, Any]] = {}
|
||||
self._download_lock = threading.Lock()
|
||||
|
||||
self._device_auth_future = None
|
||||
self._device_auth_link = None
|
||||
|
||||
# Engine reference is populated by set_engine() at registration
|
||||
# time. Until then dispatch returns None — orchestrator wires
|
||||
# this immediately so the only None case is tests that bypass
|
||||
# the orchestrator.
|
||||
self._engine = None
|
||||
|
||||
def set_engine(self, engine):
|
||||
"""Engine callback — gives the client access to the central
|
||||
thread worker + state store. Engine calls this during
|
||||
``register_plugin`` if the plugin defines it."""
|
||||
self._engine = engine
|
||||
|
||||
def set_shutdown_check(self, check_callable):
|
||||
self.shutdown_check = check_callable
|
||||
|
||||
|
|
@ -604,85 +615,36 @@ class TidalDownloadClient:
|
|||
) from exc
|
||||
|
||||
async def download(self, username: str, filename: str, file_size: int = 0) -> Optional[str]:
|
||||
if '||' not in filename:
|
||||
logger.error(f"Invalid filename format: {filename}")
|
||||
return None
|
||||
if self._engine is None:
|
||||
# Raise rather than return None so the orchestrator's
|
||||
# download_with_fallback surfaces a real warning + tries
|
||||
# the next source. Returning None silently dropped the
|
||||
# download with no user feedback (per JohnBaumb).
|
||||
raise RuntimeError("Tidal client has no engine reference — cannot dispatch download")
|
||||
|
||||
track_id_str, display_name = filename.split('||', 1)
|
||||
try:
|
||||
if '||' not in filename:
|
||||
logger.error(f"Invalid filename format: {filename}")
|
||||
return None
|
||||
|
||||
track_id_str, display_name = filename.split('||', 1)
|
||||
try:
|
||||
track_id = int(track_id_str)
|
||||
except ValueError:
|
||||
logger.error(f"Invalid Tidal track ID: {track_id_str}")
|
||||
return None
|
||||
|
||||
logger.info(f"Starting Tidal download: {display_name}")
|
||||
|
||||
download_id = str(uuid.uuid4())
|
||||
|
||||
with self._download_lock:
|
||||
self.active_downloads[download_id] = {
|
||||
'id': download_id,
|
||||
'filename': filename,
|
||||
'username': 'tidal',
|
||||
'state': 'Initializing',
|
||||
'progress': 0.0,
|
||||
'size': 0,
|
||||
'transferred': 0,
|
||||
'speed': 0,
|
||||
'time_remaining': None,
|
||||
'track_id': track_id,
|
||||
'display_name': display_name,
|
||||
'file_path': None,
|
||||
}
|
||||
|
||||
download_thread = threading.Thread(
|
||||
target=self._download_thread_worker,
|
||||
args=(download_id, track_id, display_name, filename),
|
||||
daemon=True,
|
||||
)
|
||||
download_thread.start()
|
||||
|
||||
logger.info(f"Tidal download {download_id} started in background")
|
||||
return download_id
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to start Tidal download: {e}")
|
||||
import traceback
|
||||
traceback.print_exc()
|
||||
track_id = int(track_id_str)
|
||||
except ValueError:
|
||||
logger.error(f"Invalid Tidal track ID: {track_id_str}")
|
||||
return None
|
||||
|
||||
def _download_thread_worker(self, download_id: str, track_id: int, display_name: str, original_filename: str):
|
||||
try:
|
||||
with self._download_lock:
|
||||
if download_id in self.active_downloads:
|
||||
self.active_downloads[download_id]['state'] = 'InProgress, Downloading'
|
||||
logger.info(f"Starting Tidal download: {display_name}")
|
||||
|
||||
file_path = self._download_sync(download_id, track_id, display_name)
|
||||
|
||||
if file_path:
|
||||
with self._download_lock:
|
||||
if download_id in self.active_downloads:
|
||||
self.active_downloads[download_id]['state'] = 'Completed, Succeeded'
|
||||
self.active_downloads[download_id]['progress'] = 100.0
|
||||
self.active_downloads[download_id]['file_path'] = file_path
|
||||
|
||||
logger.info(f"Tidal download {download_id} completed: {file_path}")
|
||||
else:
|
||||
with self._download_lock:
|
||||
if download_id in self.active_downloads:
|
||||
self.active_downloads[download_id]['state'] = 'Errored'
|
||||
|
||||
logger.error(f"Tidal download {download_id} failed")
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Tidal download thread failed for {download_id}: {e}")
|
||||
import traceback
|
||||
traceback.print_exc()
|
||||
|
||||
with self._download_lock:
|
||||
if download_id in self.active_downloads:
|
||||
self.active_downloads[download_id]['state'] = 'Errored'
|
||||
return self._engine.worker.dispatch(
|
||||
source_name='tidal',
|
||||
target_id=track_id,
|
||||
display_name=display_name,
|
||||
original_filename=filename,
|
||||
impl_callable=self._download_sync,
|
||||
extra_record_fields={
|
||||
'track_id': track_id,
|
||||
'display_name': display_name,
|
||||
},
|
||||
)
|
||||
|
||||
def _download_sync(self, download_id: str, track_id: int, display_name: str) -> Optional[str]:
|
||||
if not self.session or not self.session.check_login():
|
||||
|
|
@ -727,9 +689,8 @@ class TidalDownloadClient:
|
|||
speed_start = time.time()
|
||||
segments_completed = 0
|
||||
|
||||
with self._download_lock:
|
||||
if download_id in self.active_downloads:
|
||||
self.active_downloads[download_id]['size'] = 0
|
||||
if self._engine is not None:
|
||||
self._engine.update_record('tidal', download_id, {'size': 0})
|
||||
|
||||
with intermediate_path.open('wb') as output_file:
|
||||
if init_uri:
|
||||
|
|
@ -828,97 +789,82 @@ class TidalDownloadClient:
|
|||
def _update_download_progress(self, download_id: str, downloaded: int,
|
||||
segments_completed: int, total_segments: int,
|
||||
speed_start: float):
|
||||
with self._download_lock:
|
||||
if download_id not in self.active_downloads:
|
||||
return
|
||||
info = self.active_downloads[download_id]
|
||||
info['transferred'] = downloaded
|
||||
if self._engine is None:
|
||||
return
|
||||
record = self._engine.get_record('tidal', download_id)
|
||||
if record is None:
|
||||
return
|
||||
|
||||
now = time.time()
|
||||
elapsed_total = now - speed_start
|
||||
speed = int(downloaded / elapsed_total) if elapsed_total > 0 else 0
|
||||
info['speed'] = speed
|
||||
now = time.time()
|
||||
elapsed_total = now - speed_start
|
||||
speed = int(downloaded / elapsed_total) if elapsed_total > 0 else 0
|
||||
|
||||
if total_segments > 0:
|
||||
progress = (segments_completed / total_segments) * 100
|
||||
info['progress'] = round(min(progress, 99.9), 1)
|
||||
progress = record.get('progress', 0.0)
|
||||
if total_segments > 0:
|
||||
progress = round(min((segments_completed / total_segments) * 100, 99.9), 1)
|
||||
|
||||
time_remaining = None
|
||||
if speed > 0:
|
||||
remaining_bytes = downloaded * (total_segments / max(segments_completed, 1)) - downloaded
|
||||
if remaining_bytes > 0:
|
||||
time_remaining = int(remaining_bytes / speed)
|
||||
info['time_remaining'] = time_remaining
|
||||
time_remaining = None
|
||||
if speed > 0:
|
||||
remaining_bytes = downloaded * (total_segments / max(segments_completed, 1)) - downloaded
|
||||
if remaining_bytes > 0:
|
||||
time_remaining = int(remaining_bytes / speed)
|
||||
|
||||
self._engine.update_record('tidal', download_id, {
|
||||
'transferred': downloaded,
|
||||
'speed': speed,
|
||||
'progress': progress,
|
||||
'time_remaining': time_remaining,
|
||||
})
|
||||
|
||||
def _record_to_status(self, record):
|
||||
return DownloadStatus(
|
||||
id=record['id'],
|
||||
filename=record['filename'],
|
||||
username=record['username'],
|
||||
state=record['state'],
|
||||
progress=record['progress'],
|
||||
size=record.get('size', 0),
|
||||
transferred=record.get('transferred', 0),
|
||||
speed=record.get('speed', 0),
|
||||
time_remaining=record.get('time_remaining'),
|
||||
file_path=record.get('file_path'),
|
||||
)
|
||||
|
||||
async def get_all_downloads(self) -> List[DownloadStatus]:
|
||||
download_statuses = []
|
||||
|
||||
with self._download_lock:
|
||||
for _download_id, info in self.active_downloads.items():
|
||||
status = DownloadStatus(
|
||||
id=info['id'],
|
||||
filename=info['filename'],
|
||||
username=info['username'],
|
||||
state=info['state'],
|
||||
progress=info['progress'],
|
||||
size=info['size'],
|
||||
transferred=info['transferred'],
|
||||
speed=info['speed'],
|
||||
time_remaining=info.get('time_remaining'),
|
||||
file_path=info.get('file_path'),
|
||||
)
|
||||
download_statuses.append(status)
|
||||
|
||||
return download_statuses
|
||||
if self._engine is None:
|
||||
return []
|
||||
return [
|
||||
self._record_to_status(record)
|
||||
for record in self._engine.iter_records_for_source('tidal')
|
||||
]
|
||||
|
||||
async def get_download_status(self, download_id: str) -> Optional[DownloadStatus]:
|
||||
with self._download_lock:
|
||||
if download_id not in self.active_downloads:
|
||||
return None
|
||||
|
||||
info = self.active_downloads[download_id]
|
||||
return DownloadStatus(
|
||||
id=info['id'],
|
||||
filename=info['filename'],
|
||||
username=info['username'],
|
||||
state=info['state'],
|
||||
progress=info['progress'],
|
||||
size=info['size'],
|
||||
transferred=info['transferred'],
|
||||
speed=info['speed'],
|
||||
time_remaining=info.get('time_remaining'),
|
||||
file_path=info.get('file_path'),
|
||||
)
|
||||
if self._engine is None:
|
||||
return None
|
||||
record = self._engine.get_record('tidal', download_id)
|
||||
return self._record_to_status(record) if record is not None else None
|
||||
|
||||
async def cancel_download(self, download_id: str, username: str = None, remove: bool = False) -> bool:
|
||||
try:
|
||||
with self._download_lock:
|
||||
if download_id not in self.active_downloads:
|
||||
logger.warning(f"Download {download_id} not found")
|
||||
return False
|
||||
|
||||
self.active_downloads[download_id]['state'] = 'Cancelled'
|
||||
logger.info(f"Marked Tidal download {download_id} as cancelled")
|
||||
|
||||
if remove:
|
||||
del self.active_downloads[download_id]
|
||||
logger.info(f"Removed Tidal download {download_id} from queue")
|
||||
|
||||
return True
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to cancel download {download_id}: {e}")
|
||||
if self._engine is None:
|
||||
return False
|
||||
if self._engine.get_record('tidal', download_id) is None:
|
||||
logger.warning(f"Tidal download {download_id} not found")
|
||||
return False
|
||||
self._engine.update_record('tidal', download_id, {'state': 'Cancelled'})
|
||||
logger.info(f"Marked Tidal download {download_id} as cancelled")
|
||||
if remove:
|
||||
self._engine.remove_record('tidal', download_id)
|
||||
logger.info(f"Removed Tidal download {download_id} from queue")
|
||||
return True
|
||||
|
||||
async def clear_all_completed_downloads(self) -> bool:
|
||||
if self._engine is None:
|
||||
return True
|
||||
try:
|
||||
with self._download_lock:
|
||||
ids_to_remove = [
|
||||
did for did, info in self.active_downloads.items()
|
||||
if info.get('state', '') in ('Completed, Succeeded', 'Cancelled', 'Errored', 'Aborted')
|
||||
]
|
||||
for did in ids_to_remove:
|
||||
del self.active_downloads[did]
|
||||
|
||||
terminal = {'Completed, Succeeded', 'Cancelled', 'Errored', 'Aborted'}
|
||||
for record in list(self._engine.iter_records_for_source('tidal')):
|
||||
if record.get('state') in terminal:
|
||||
self._engine.remove_record('tidal', record['id'])
|
||||
return True
|
||||
except Exception as e:
|
||||
logger.error(f"Error clearing downloads: {e}")
|
||||
|
|
|
|||
|
|
@ -17,7 +17,6 @@ import time
|
|||
import platform
|
||||
import asyncio
|
||||
import uuid
|
||||
import threading
|
||||
from typing import List, Optional, Dict, Any, Tuple
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
|
|
@ -34,7 +33,7 @@ from core.matching_engine import MusicMatchingEngine
|
|||
from core.spotify_client import Track as SpotifyTrack
|
||||
|
||||
# Import Soulseek data structures for drop-in replacement compatibility
|
||||
from core.soulseek_client import SearchResult, TrackResult, AlbumResult, DownloadStatus
|
||||
from core.download_plugins.types import SearchResult, TrackResult, AlbumResult, DownloadStatus
|
||||
|
||||
logger = get_logger("youtube_client")
|
||||
|
||||
|
|
@ -92,7 +91,10 @@ class YouTubeSearchResult:
|
|||
self.parsed_artist = self.channel
|
||||
|
||||
|
||||
class YouTubeClient:
|
||||
from core.download_plugins.base import DownloadSourcePlugin
|
||||
|
||||
|
||||
class YouTubeClient(DownloadSourcePlugin):
|
||||
"""
|
||||
YouTube download client using yt-dlp.
|
||||
Provides search, matching, and download capabilities with full Spotify metadata integration.
|
||||
|
|
@ -112,10 +114,39 @@ class YouTubeClient:
|
|||
# Callback for shutdown check (avoids circular imports)
|
||||
self.shutdown_check = None
|
||||
|
||||
# Rate limiting — serialize YouTube downloads with delay
|
||||
self._download_semaphore = threading.Semaphore(1)
|
||||
# Rate-limit policy — applied to engine.worker once the engine
|
||||
# is wired in via set_engine(). Kept as an attribute for
|
||||
# backward-compat external readers + so settings reload can
|
||||
# update it without touching the engine.
|
||||
self._download_delay = config_manager.get('youtube.download_delay', 3)
|
||||
self._last_download_time = 0
|
||||
|
||||
# Engine reference is populated by set_engine() at registration
|
||||
# time. Until then the client can't dispatch downloads — but
|
||||
# in production the orchestrator wires the engine immediately
|
||||
# after constructing the registry, so this is only None in
|
||||
# tests that bypass the orchestrator.
|
||||
self._engine = None
|
||||
|
||||
def rate_limit_policy(self):
|
||||
"""YouTube reads its download delay from user-tunable config
|
||||
(``youtube.download_delay``, default 3s). Engine reads this
|
||||
at ``register_plugin`` time, then ``set_engine`` runs and
|
||||
re-applies if the config changed since instance construction."""
|
||||
from core.download_engine import RateLimitPolicy
|
||||
return RateLimitPolicy(
|
||||
download_concurrency=1,
|
||||
download_delay_seconds=float(self._download_delay),
|
||||
)
|
||||
|
||||
def set_engine(self, engine):
|
||||
"""Engine callback — gives the client access to the central
|
||||
thread worker + state store. Engine calls this during
|
||||
``register_plugin`` if the plugin defines it. Worker delay
|
||||
was already set from rate_limit_policy() — re-apply here so
|
||||
runtime ``reload_settings`` updates take effect via the
|
||||
same pathway."""
|
||||
self._engine = engine
|
||||
engine.worker.set_delay('youtube', float(self._download_delay))
|
||||
|
||||
def set_shutdown_check(self, check_callable):
|
||||
"""Set a callback function to check for system shutdown"""
|
||||
|
|
@ -130,11 +161,6 @@ class YouTubeClient:
|
|||
logger.error("ffmpeg is required but not found")
|
||||
logger.error("The client will attempt to auto-download ffmpeg on first use")
|
||||
|
||||
# Download queue management (mirrors Soulseek's download tracking)
|
||||
# Maps download_id -> download_info dict
|
||||
self.active_downloads: Dict[str, Dict[str, Any]] = {}
|
||||
self._download_lock = threading.Lock() # Use threading.Lock for thread safety
|
||||
|
||||
# Configure yt-dlp options with bot detection bypass
|
||||
self.download_opts = {
|
||||
'format': 'bestaudio/best',
|
||||
|
|
@ -272,15 +298,14 @@ class YouTubeClient:
|
|||
self.progress_callback = callback
|
||||
|
||||
def _progress_hook(self, d):
|
||||
"""
|
||||
yt-dlp progress hook - called during download to report progress.
|
||||
Updates the active_downloads dictionary for the current download.
|
||||
Mirrors Soulseek's transfer status updates.
|
||||
"""
|
||||
"""yt-dlp progress hook — called during download to report
|
||||
progress. Writes to the engine record (Phase C2 lifted state
|
||||
out of the per-client dict; this hook follows suit)."""
|
||||
try:
|
||||
# Only update if we have a current download ID
|
||||
if not self.current_download_id:
|
||||
return
|
||||
if self._engine is None:
|
||||
return
|
||||
|
||||
status = d.get('status', 'unknown')
|
||||
|
||||
|
|
@ -289,24 +314,18 @@ class YouTubeClient:
|
|||
total = d.get('total_bytes') or d.get('total_bytes_estimate', 0)
|
||||
speed = d.get('speed', 0) or 0
|
||||
eta = d.get('eta', 0) or 0
|
||||
percent = (downloaded / total) * 100 if total > 0 else 0
|
||||
|
||||
if total > 0:
|
||||
percent = (downloaded / total) * 100
|
||||
else:
|
||||
percent = 0
|
||||
self._engine.update_record('youtube', self.current_download_id, {
|
||||
'state': 'InProgress, Downloading',
|
||||
'progress': round(percent, 1),
|
||||
'transferred': downloaded,
|
||||
'size': total,
|
||||
'speed': int(speed),
|
||||
'time_remaining': int(eta) if eta > 0 else None,
|
||||
})
|
||||
|
||||
# Update active downloads dictionary (thread-safe update with lock)
|
||||
with self._download_lock:
|
||||
if self.current_download_id in self.active_downloads:
|
||||
download_info = self.active_downloads[self.current_download_id]
|
||||
download_info['state'] = 'InProgress, Downloading' # Match Soulseek state format
|
||||
download_info['progress'] = round(percent, 1)
|
||||
download_info['transferred'] = downloaded
|
||||
download_info['size'] = total
|
||||
download_info['speed'] = int(speed)
|
||||
download_info['time_remaining'] = int(eta) if eta > 0 else None
|
||||
|
||||
# Also update current_download_progress for legacy compatibility
|
||||
# Legacy progress dict for any external listeners.
|
||||
self.current_download_progress = {
|
||||
'status': 'downloading',
|
||||
'percent': round(percent, 1),
|
||||
|
|
@ -316,30 +335,27 @@ class YouTubeClient:
|
|||
'eta': int(eta),
|
||||
'filename': d.get('filename', '')
|
||||
}
|
||||
|
||||
# Call progress callback if set (for UI updates)
|
||||
if self.progress_callback:
|
||||
self.progress_callback(self.current_download_progress)
|
||||
|
||||
elif status == 'finished':
|
||||
# Download finished, ffmpeg is converting to MP3
|
||||
# Keep state as 'InProgress, Downloading' - the download thread will set final state
|
||||
with self._download_lock:
|
||||
if self.current_download_id in self.active_downloads:
|
||||
self.active_downloads[self.current_download_id]['progress'] = 95.0 # Almost done (converting)
|
||||
|
||||
# Download finished — ffmpeg now converts to MP3. The
|
||||
# engine.worker thread flips to 'Completed, Succeeded'
|
||||
# once _download_sync returns; this just bumps progress
|
||||
# to 95% so the UI doesn't sit at 99.9% during the
|
||||
# ffmpeg post-process.
|
||||
self._engine.update_record('youtube', self.current_download_id, {
|
||||
'progress': 95.0,
|
||||
})
|
||||
self.current_download_progress['status'] = 'postprocessing'
|
||||
self.current_download_progress['percent'] = 95.0
|
||||
|
||||
if self.progress_callback:
|
||||
self.progress_callback(self.current_download_progress)
|
||||
|
||||
elif status == 'error':
|
||||
# Mark as error (thread-safe)
|
||||
with self._download_lock:
|
||||
if self.current_download_id in self.active_downloads:
|
||||
self.active_downloads[self.current_download_id]['state'] = 'Errored'
|
||||
|
||||
self._engine.update_record('youtube', self.current_download_id, {
|
||||
'state': 'Errored',
|
||||
})
|
||||
self.current_download_progress['status'] = 'error'
|
||||
if self.progress_callback:
|
||||
self.progress_callback(self.current_download_progress)
|
||||
|
|
@ -859,137 +875,57 @@ class YouTubeClient:
|
|||
return matches
|
||||
|
||||
async def download(self, username: str, filename: str, file_size: int = 0) -> Optional[str]:
|
||||
"""
|
||||
Download YouTube video as audio (async, Soulseek-compatible interface).
|
||||
"""Download YouTube video as audio.
|
||||
|
||||
Returns download_id immediately and runs download in background thread.
|
||||
Monitor via get_download_status() or get_all_downloads().
|
||||
Returns download_id immediately; the actual download runs in
|
||||
a background thread spawned by ``engine.worker``. Monitor
|
||||
via ``orchestrator.get_download_status(download_id)``.
|
||||
|
||||
Args:
|
||||
username: Ignored for YouTube (always "youtube")
|
||||
filename: Encoded as "video_id||title" from search results
|
||||
file_size: Ignored for YouTube (kept for interface compatibility)
|
||||
|
||||
Returns:
|
||||
download_id: Unique ID for tracking this download
|
||||
"""
|
||||
try:
|
||||
# Parse filename to extract video_id
|
||||
if '||' not in filename:
|
||||
logger.error(f"Invalid filename format: {filename}")
|
||||
return None
|
||||
|
||||
video_id, title = filename.split('||', 1)
|
||||
youtube_url = f"https://www.youtube.com/watch?v={video_id}"
|
||||
|
||||
logger.info(f"Starting YouTube download: {title}")
|
||||
logger.info(f" URL: {youtube_url}")
|
||||
|
||||
# Create unique download ID
|
||||
download_id = str(uuid.uuid4())
|
||||
|
||||
# Initialize download info in active downloads
|
||||
with self._download_lock:
|
||||
self.active_downloads[download_id] = {
|
||||
'id': download_id,
|
||||
'filename': filename, # Keep original encoded format for context matching!
|
||||
'username': 'youtube',
|
||||
'state': 'Initializing', # Soulseek-style states
|
||||
'progress': 0.0,
|
||||
'size': file_size or 0,
|
||||
'transferred': 0,
|
||||
'speed': 0,
|
||||
'time_remaining': None,
|
||||
'video_id': video_id,
|
||||
'url': youtube_url,
|
||||
'title': title,
|
||||
'file_path': None, # Will be set when download completes
|
||||
}
|
||||
|
||||
# Start download in background thread (returns immediately)
|
||||
download_thread = threading.Thread(
|
||||
target=self._download_thread_worker,
|
||||
args=(download_id, youtube_url, title, filename),
|
||||
daemon=True
|
||||
)
|
||||
download_thread.start()
|
||||
|
||||
logger.info(f"YouTube download {download_id} started in background")
|
||||
return download_id
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to start YouTube download: {e}")
|
||||
import traceback
|
||||
traceback.print_exc()
|
||||
if '||' not in filename:
|
||||
logger.error(f"Invalid filename format: {filename}")
|
||||
return None
|
||||
if self._engine is None:
|
||||
# Raise rather than return None so the orchestrator's
|
||||
# download_with_fallback surfaces a real warning + tries
|
||||
# the next source. Returning None silently dropped the
|
||||
# download with no user feedback (per JohnBaumb).
|
||||
raise RuntimeError("YouTube client has no engine reference — cannot dispatch download")
|
||||
|
||||
def _download_thread_worker(self, download_id: str, youtube_url: str, title: str, original_filename: str):
|
||||
"""
|
||||
Background thread worker for downloading YouTube videos.
|
||||
Updates active_downloads dict with progress.
|
||||
Serialized via semaphore with configurable delay between downloads.
|
||||
"""
|
||||
try:
|
||||
with self._download_semaphore:
|
||||
# Enforce delay since last download completed
|
||||
elapsed = time.time() - self._last_download_time
|
||||
if self._last_download_time > 0 and elapsed < self._download_delay:
|
||||
wait_time = self._download_delay - elapsed
|
||||
logger.info(f"Rate limiting: waiting {wait_time:.1f}s before next YouTube download")
|
||||
time.sleep(wait_time)
|
||||
video_id, title = filename.split('||', 1)
|
||||
youtube_url = f"https://www.youtube.com/watch?v={video_id}"
|
||||
logger.info("Starting YouTube download: %s (%s)", title, youtube_url)
|
||||
|
||||
# Update state to downloading
|
||||
with self._download_lock:
|
||||
if download_id in self.active_downloads:
|
||||
self.active_downloads[download_id]['state'] = 'InProgress, Downloading' # Match Soulseek state
|
||||
|
||||
# Set current download ID for progress hook
|
||||
self.current_download_id = download_id
|
||||
|
||||
# Perform actual download
|
||||
file_path = self._download_sync(youtube_url, title)
|
||||
|
||||
# Clear current download ID
|
||||
def _impl(download_id, _target_id, display_name):
|
||||
# The progress hook reads ``current_download_id`` to know
|
||||
# which download to update. Set it before the call, clear
|
||||
# after, even on exception.
|
||||
self.current_download_id = download_id
|
||||
try:
|
||||
return self._download_sync(youtube_url, title)
|
||||
finally:
|
||||
self.current_download_id = None
|
||||
|
||||
# Record completion time for rate limiting
|
||||
self._last_download_time = time.time()
|
||||
|
||||
if file_path:
|
||||
# Mark as completed/succeeded (match Soulseek state)
|
||||
with self._download_lock:
|
||||
if download_id in self.active_downloads:
|
||||
# IMPORTANT: Keep original filename for context lookup!
|
||||
# The filename must match what was used to create the context entry
|
||||
# We store the actual file path separately
|
||||
self.active_downloads[download_id]['state'] = 'Completed, Succeeded' # Match Soulseek
|
||||
self.active_downloads[download_id]['progress'] = 100.0
|
||||
self.active_downloads[download_id]['file_path'] = file_path
|
||||
# DO NOT update filename - keep original_filename for context matching
|
||||
|
||||
logger.info(f"YouTube download {download_id} completed: {file_path}")
|
||||
else:
|
||||
# Mark as errored
|
||||
with self._download_lock:
|
||||
if download_id in self.active_downloads:
|
||||
self.active_downloads[download_id]['state'] = 'Errored'
|
||||
|
||||
logger.error(f"YouTube download {download_id} failed")
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"YouTube download thread failed for {download_id}: {e}")
|
||||
import traceback
|
||||
traceback.print_exc()
|
||||
|
||||
# Mark as errored
|
||||
with self._download_lock:
|
||||
if download_id in self.active_downloads:
|
||||
self.active_downloads[download_id]['state'] = 'Errored'
|
||||
|
||||
# Clear current download ID
|
||||
if self.current_download_id == download_id:
|
||||
self.current_download_id = None
|
||||
return self._engine.worker.dispatch(
|
||||
source_name='youtube',
|
||||
target_id=video_id,
|
||||
display_name=title,
|
||||
original_filename=filename,
|
||||
impl_callable=_impl,
|
||||
extra_record_fields={
|
||||
'video_id': video_id,
|
||||
'url': youtube_url,
|
||||
'title': title,
|
||||
},
|
||||
)
|
||||
|
||||
# Legacy worker stub kept temporarily for legacy comment context —
|
||||
# see _download_sync below for the actual yt-dlp invocation that
|
||||
# the engine's BackgroundDownloadWorker now drives.
|
||||
def _download_sync(self, youtube_url: str, title: str) -> Optional[str]:
|
||||
"""
|
||||
Synchronous download method (runs in thread pool executor).
|
||||
|
|
@ -1138,123 +1074,76 @@ class YouTubeClient:
|
|||
traceback.print_exc()
|
||||
return None
|
||||
|
||||
def _record_to_status(self, record):
|
||||
"""Translate an engine record dict into the DownloadStatus
|
||||
dataclass shape consumers expect."""
|
||||
return DownloadStatus(
|
||||
id=record['id'],
|
||||
filename=record['filename'],
|
||||
username=record['username'],
|
||||
state=record['state'],
|
||||
progress=record['progress'],
|
||||
size=record.get('size', 0),
|
||||
transferred=record.get('transferred', 0),
|
||||
speed=record.get('speed', 0),
|
||||
time_remaining=record.get('time_remaining'),
|
||||
file_path=record.get('file_path'),
|
||||
)
|
||||
|
||||
async def get_all_downloads(self) -> List[DownloadStatus]:
|
||||
"""
|
||||
Get all active downloads (matches Soulseek interface).
|
||||
|
||||
Returns:
|
||||
List of DownloadStatus objects for all active downloads
|
||||
"""
|
||||
download_statuses = []
|
||||
|
||||
with self._download_lock:
|
||||
for _download_id, download_info in self.active_downloads.items():
|
||||
status = DownloadStatus(
|
||||
id=download_info['id'],
|
||||
filename=download_info['filename'],
|
||||
username=download_info['username'],
|
||||
state=download_info['state'],
|
||||
progress=download_info['progress'],
|
||||
size=download_info['size'],
|
||||
transferred=download_info['transferred'],
|
||||
speed=download_info['speed'],
|
||||
time_remaining=download_info.get('time_remaining')
|
||||
)
|
||||
download_statuses.append(status)
|
||||
|
||||
return download_statuses
|
||||
"""Active downloads owned by the YouTube source — read from
|
||||
engine state."""
|
||||
if self._engine is None:
|
||||
return []
|
||||
return [
|
||||
self._record_to_status(record)
|
||||
for record in self._engine.iter_records_for_source('youtube')
|
||||
]
|
||||
|
||||
async def get_download_status(self, download_id: str) -> Optional[DownloadStatus]:
|
||||
"""
|
||||
Get status of a specific download (matches Soulseek interface).
|
||||
|
||||
Args:
|
||||
download_id: Download ID to query
|
||||
|
||||
Returns:
|
||||
DownloadStatus object or None if not found
|
||||
"""
|
||||
with self._download_lock:
|
||||
if download_id not in self.active_downloads:
|
||||
return None
|
||||
|
||||
download_info = self.active_downloads[download_id]
|
||||
|
||||
return DownloadStatus(
|
||||
id=download_info['id'],
|
||||
filename=download_info['filename'],
|
||||
username=download_info['username'],
|
||||
state=download_info['state'],
|
||||
progress=download_info['progress'],
|
||||
size=download_info['size'],
|
||||
transferred=download_info['transferred'],
|
||||
speed=download_info['speed'],
|
||||
time_remaining=download_info.get('time_remaining'),
|
||||
file_path=download_info.get('file_path')
|
||||
)
|
||||
|
||||
"""Single download status — read from engine state. Returns
|
||||
None if this id isn't owned by YouTube (or not found)."""
|
||||
if self._engine is None:
|
||||
return None
|
||||
record = self._engine.get_record('youtube', download_id)
|
||||
if record is None:
|
||||
return None
|
||||
return self._record_to_status(record)
|
||||
|
||||
async def clear_all_completed_downloads(self) -> bool:
|
||||
"""
|
||||
Clear all terminal (completed, cancelled, errored) downloads from the list.
|
||||
Matches Soulseek interface.
|
||||
"""
|
||||
"""Clear terminal-state downloads (Completed / Cancelled /
|
||||
Errored / Aborted) from engine state."""
|
||||
if self._engine is None:
|
||||
return True
|
||||
try:
|
||||
with self._download_lock:
|
||||
# Identify IDs to remove
|
||||
ids_to_remove = []
|
||||
for download_id, info in self.active_downloads.items():
|
||||
state = info.get('state', '')
|
||||
# Check for terminal states
|
||||
# Note: We check exact strings used in _download_thread_worker and cancel_download
|
||||
if state in ['Completed, Succeeded', 'Cancelled', 'Errored', 'Aborted']:
|
||||
ids_to_remove.append(download_id)
|
||||
|
||||
# Remove them
|
||||
for download_id in ids_to_remove:
|
||||
del self.active_downloads[download_id]
|
||||
logger.debug(f"Cleared finished download {download_id}")
|
||||
|
||||
terminal_states = {'Completed, Succeeded', 'Cancelled', 'Errored', 'Aborted'}
|
||||
for record in list(self._engine.iter_records_for_source('youtube')):
|
||||
if record.get('state') in terminal_states:
|
||||
self._engine.remove_record('youtube', record['id'])
|
||||
logger.debug("Cleared finished YouTube download %s", record['id'])
|
||||
return True
|
||||
except Exception as e:
|
||||
logger.error(f"Error clearing downloads: {e}")
|
||||
return False
|
||||
|
||||
async def cancel_download(self, download_id: str, username: str = None, remove: bool = False) -> bool:
|
||||
"""
|
||||
Cancel an active download (matches Soulseek interface).
|
||||
|
||||
NOTE: YouTube downloads cannot be truly cancelled mid-download,
|
||||
but we mark them as cancelled for UI consistency.
|
||||
|
||||
Args:
|
||||
download_id: Download ID to cancel
|
||||
username: Ignored for YouTube (kept for interface compatibility)
|
||||
remove: If True, remove from active downloads after cancelling
|
||||
|
||||
Returns:
|
||||
True if cancelled successfully, False otherwise
|
||||
"""
|
||||
try:
|
||||
with self._download_lock:
|
||||
if download_id not in self.active_downloads:
|
||||
logger.warning(f"Download {download_id} not found")
|
||||
return False
|
||||
|
||||
# Update state to cancelled
|
||||
self.active_downloads[download_id]['state'] = 'Cancelled'
|
||||
logger.info(f"Marked YouTube download {download_id} as cancelled")
|
||||
|
||||
# Remove from active downloads if requested
|
||||
if remove:
|
||||
del self.active_downloads[download_id]
|
||||
logger.info(f"Removed YouTube download {download_id} from queue")
|
||||
|
||||
return True
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to cancel download {download_id}: {e}")
|
||||
"""Mark a YouTube download as cancelled. yt-dlp downloads
|
||||
can't be truly interrupted mid-stream — this only flips
|
||||
the state for UI consistency. ``remove=True`` also drops
|
||||
the engine record."""
|
||||
if self._engine is None:
|
||||
return False
|
||||
record = self._engine.get_record('youtube', download_id)
|
||||
if record is None:
|
||||
logger.warning(f"YouTube download {download_id} not found")
|
||||
return False
|
||||
|
||||
self._engine.update_record('youtube', download_id, {'state': 'Cancelled'})
|
||||
logger.info(f"Marked YouTube download {download_id} as cancelled")
|
||||
if remove:
|
||||
self._engine.remove_record('youtube', download_id)
|
||||
logger.info(f"Removed YouTube download {download_id} from queue")
|
||||
return True
|
||||
|
||||
def _enhance_metadata(self, filepath: str, spotify_track: Optional[SpotifyTrack], yt_result: YouTubeSearchResult, track_number: int = 1, disc_number: int = 1, release_year: str = None, artist_genres: list = None):
|
||||
"""
|
||||
|
|
|
|||
206
docs/download-engine-refactor-plan.md
Normal file
206
docs/download-engine-refactor-plan.md
Normal file
|
|
@ -0,0 +1,206 @@
|
|||
# Download Engine Refactor Plan
|
||||
|
||||
## Goal
|
||||
|
||||
Mirror Cin's "metadata engine" architecture for the download dispatcher. Move shared logic OUT of the per-source clients (currently 1600+ LOC of duplicated thread workers, search retry ladders, rate-limiters, state machines) and INTO a central `DownloadEngine`. Clients become dumb: make raw API requests + manage their own auth state. Everything else is the engine.
|
||||
|
||||
This is the SAME architectural smell Cin flagged on the metadata layer, applied to downloads. If we keep adding sources (usenet planned + likely more), the only honest fix is to stop reinventing the wheel per client.
|
||||
|
||||
## Architecture target
|
||||
|
||||
```
|
||||
┌─────────────┐ ┌──────────────────┐
|
||||
│ feature │ ── search/download ──▶ ┌────────────────────┐ ─▶│ Soulseek (raw) │
|
||||
│ │ ◀── normalized ──────── │ DownloadEngine │ ─▶│ YouTube (raw) │
|
||||
└─────────────┘ │ │ ─▶│ Tidal (raw) │
|
||||
│ ◆ thread workers │ ─▶│ Qobuz (raw) │
|
||||
│ ◆ rate-limit pool │ ─▶│ HiFi (raw) │
|
||||
│ ◆ search retry │ ─▶│ Deezer (raw) │
|
||||
│ ◆ quality filter │ ─▶│ SoundCloud (raw) │
|
||||
│ ◆ state tracking │ ─▶│ Lidarr (album) │
|
||||
│ ◆ fallback chain │ └──────────────────┘
|
||||
│ ◆ cache │ clients only do:
|
||||
└────────────────────┘ - raw API request
|
||||
- auth/token state
|
||||
```
|
||||
|
||||
## What clients keep (legitimately per-source)
|
||||
|
||||
- Auth flow + token refresh (Tidal OAuth, Qobuz session, Deezer ARL, slskd API key, etc.)
|
||||
- Source-specific protocol (slskd events vs HTTP REST vs HLS demux vs Blowfish decrypt vs yt-dlp subprocess)
|
||||
- Source-specific search query shape (free text vs keyword filters vs MusicBrainz ID lookup)
|
||||
- Source-specific "download a thing" atomic operation (`_download_impl(target_id) → file_path`)
|
||||
|
||||
## What moves into the engine
|
||||
|
||||
| Today (per-client, duplicated) | Tomorrow (engine, single source of truth) |
|
||||
|---|---|
|
||||
| `self.active_downloads = {}` per client | `engine.active_downloads = {}` |
|
||||
| `self._download_lock = Lock()` per client | `engine.state_lock = Lock()` |
|
||||
| `self._download_semaphore = Semaphore(...)` per client | `engine.download_pool` (per-source semaphore from registry) |
|
||||
| `self._last_download_time / _download_delay` per client | `engine.rate_limiter.acquire(source)` |
|
||||
| `_download_thread_worker` × 7 (~70 LOC each) | `engine.dispatch_download(plugin, target_id)` |
|
||||
| Search retry ladder × 7 | `engine.search(query)` with shared retry policy |
|
||||
| Quality filter × 7 | `engine.filter_by_quality(results, prefs)` |
|
||||
| Result dedup × 7 | `engine.dedup(results)` |
|
||||
| Hybrid fallback (search only) | `engine.fallback_chain(operation)` (search AND download) |
|
||||
|
||||
## New plugin contract (much smaller)
|
||||
|
||||
```python
|
||||
class DownloadSourcePlugin(Protocol):
|
||||
# Identity
|
||||
name: str
|
||||
|
||||
# Lifecycle
|
||||
def is_configured(self) -> bool: ...
|
||||
async def check_connection(self) -> bool: ...
|
||||
def reload_settings(self) -> None: ...
|
||||
|
||||
# Search — DUMB. Just hit the API.
|
||||
async def search_raw(self, query: str) -> List[RawSearchResult]: ...
|
||||
|
||||
# Download — DUMB. Just download the bytes to a file.
|
||||
# The engine handles thread spawning, state tracking, rate limits.
|
||||
# Plugin returns the final file path on success or raises.
|
||||
async def download_raw(self, target_id: str, dest_dir: Path) -> Path: ...
|
||||
|
||||
# Cancel — best-effort. Engine handles state cleanup.
|
||||
async def cancel_raw(self, target_id: str) -> bool: ...
|
||||
```
|
||||
|
||||
Compare to today's plugin protocol (which my Phase 0 PR introduced) — that one was wrapped around fat clients. This one is dumber. Clients shrink dramatically (estimated 40-60% LOC reduction per file).
|
||||
|
||||
## Special cases
|
||||
|
||||
- **Soulseek** — slskd is event-driven, NOT thread-based. Engine's BackgroundDownloadWorker doesn't apply. Keep Soulseek's path special: `download_raw` returns immediately; engine subscribes to slskd events for state updates instead of running a thread.
|
||||
- **YouTube/SoundCloud** — yt-dlp is a subprocess. The "thread" is really `subprocess.run(['yt-dlp', ...]).wait()`. Engine handles thread; plugin's `download_raw` just runs subprocess and returns file path.
|
||||
- **Lidarr** — album-grabber, not track-grabber. Different contract. Either separate `AlbumOnlyPlugin` interface OR plugin declares `supports_track_search: bool = False`. Decide during migration.
|
||||
|
||||
## Phased commit plan
|
||||
|
||||
Each phase is one or more commits. Each commit independently revertable. Tests stay green between commits — never ship a half-broken state.
|
||||
|
||||
### Phase A — Behavior pinning tests (BEFORE any code moves)
|
||||
|
||||
**Goal:** Baseline tests for what each source's download path currently does. Catches regressions during extraction.
|
||||
|
||||
**Commit A1:** `tests/downloads/test_soulseek_download_path.py` — pin Soulseek's download lifecycle (search → download → completion → file path returned).
|
||||
**Commit A2:** Same for YouTube. Mock yt-dlp subprocess.
|
||||
**Commit A3:** Same for Tidal. Mock tidalapi.Session.
|
||||
**Commit A4:** Same for Qobuz. Mock Qobuz REST API.
|
||||
**Commit A5:** Same for HiFi. Mock hifi-api instance.
|
||||
**Commit A6:** Same for Deezer. Mock Deezer GW API + Blowfish stream.
|
||||
**Commit A7:** Same for SoundCloud. Mock yt-dlp scsearch.
|
||||
**Commit A8:** Same for Lidarr. Mock Lidarr REST API.
|
||||
|
||||
After Phase A: ~50 new tests pinning current behavior. We can refactor with confidence.
|
||||
|
||||
### Phase B — Engine skeleton + state lift
|
||||
|
||||
**Commit B1:** Create `core/download_engine/` package with `DownloadEngine` class. Engine starts EMPTY — just exposes `register_plugin(plugin)`, `active_downloads` dict, `state_lock`. Orchestrator gets a `self.engine` reference but doesn't use it yet.
|
||||
|
||||
**Commit B2:** Move `active_downloads` state out of every client into `engine.active_downloads`. Each client's `download()` now updates engine state via callback instead of `self.active_downloads[id] = ...`. Backward compat: each client's `self.active_downloads` becomes a property that delegates to `engine.active_downloads.filter(source=self.name)`.
|
||||
|
||||
**Commit B3:** Move `get_all_downloads` / `get_download_status` / `cancel_download` dispatch from orchestrator (which iterates plugins) into engine (which queries unified state). Orchestrator's methods become thin pass-throughs.
|
||||
|
||||
### Phase C — Background download worker lift
|
||||
|
||||
**Commit C1:** New `core/download_engine/worker.py` — `BackgroundDownloadWorker` class. Owns semaphore, rate-limit sleep, state-update lock pattern. Provides `dispatch(plugin, target_id, display_name) → download_id`.
|
||||
|
||||
**Commit C2:** Migrate YouTube to use BackgroundDownloadWorker. Strip `_download_thread_worker` from `youtube_client.py`. Add `download_raw(video_id, dest) → Path`. Tests stay green (Phase A pinned them).
|
||||
|
||||
**Commit C3:** Same for Tidal.
|
||||
**Commit C4:** Same for Qobuz.
|
||||
**Commit C5:** Same for HiFi.
|
||||
**Commit C6:** Same for Deezer.
|
||||
**Commit C7:** Same for SoundCloud.
|
||||
|
||||
After Phase C: ~490 LOC of duplicated thread management deleted. Each affected client shrinks.
|
||||
|
||||
### Phase D — SKIPPED
|
||||
|
||||
**Original intent:** Lift search retry / query normalization / quality filter into engine. **Dropped after surveying actual per-source search code.** Search is 90% source-specific (slskd event subscription vs yt-dlp subprocess vs HTTP REST vs HLS quality map), not 60% like the original plan estimated. Lifting would be either lossy (force per-source quirks through a uniform interface) or bloated (adapter code bigger than the original). The shared portion is ~10 LOC per source — not worth a SearchOrchestrator. Per-source search stays per-source.
|
||||
|
||||
### Phase D (original — kept for reference, NOT executed)
|
||||
|
||||
**Commit D1:** New `core/download_engine/search.py` — `SearchOrchestrator`. Owns: query normalization, shortened-query retry ladder, quality filter, dedup. Calls `plugin.search_raw(query)` for the actual API hit.
|
||||
|
||||
**Commit D2:** Migrate Tidal's search. Strip `_generate_shortened_queries`, quality filter, dedup from client. Add `search_raw(query) → List[RawResult]`.
|
||||
**Commit D3:** Same for Qobuz.
|
||||
**Commit D4:** Same for HiFi.
|
||||
**Commit D5:** Same for YouTube.
|
||||
**Commit D6:** Same for Deezer.
|
||||
**Commit D7:** Same for SoundCloud.
|
||||
**Commit D8:** Same for Soulseek (keep slskd event-driven specifics, but the post-search filter/dedup moves out).
|
||||
**Commit D9:** Same for Lidarr.
|
||||
|
||||
### Phase E — Rate-limit pool
|
||||
|
||||
**Commit E1:** New `core/download_engine/rate_limit.py` — per-source rate limiter registry. Spotify limit, Qobuz 1/sec, etc. Each plugin declares its limits in its registry spec.
|
||||
**Commit E2:** Strip per-client rate-limit state. Replace with `await engine.rate_limit.acquire(self.name)` at the top of `search_raw` / `download_raw`.
|
||||
|
||||
### Phase F — Fallback chain into engine
|
||||
|
||||
**Commit F1:** Engine owns fallback: `engine.search_with_fallback(query, source_chain)` and `engine.download_with_fallback(target_id, source_chain)`. Search hybrid behavior preserved; download hybrid newly works (today it silently routes to one source).
|
||||
**Commit F2:** Orchestrator's `search` and `download` methods delegate to engine's fallback methods. Hybrid mode logic moves out of orchestrator.
|
||||
|
||||
### Phase G — Plugin contract narrows
|
||||
|
||||
**Commit G1:** Update `DownloadSourcePlugin` Protocol — narrow to the small surface (`search_raw`, `download_raw`, `cancel_raw`, `is_configured`, `check_connection`, `reload_settings`). Conformance tests updated.
|
||||
**Commit G2:** Remove dead methods from clients that the engine now owns (`_download_thread_worker`, `_filter_results_by_quality`, etc.). Clean up imports.
|
||||
|
||||
### Phase H — Cleanup + WHATS_NEW + version bump
|
||||
|
||||
**Commit H1:** Final cleanup pass — remove backward-compat shims that are no longer needed (legacy `self.active_downloads` properties etc., once nothing reaches in for them).
|
||||
**Commit H2:** WHATS_NEW entry, PR description.
|
||||
|
||||
## Total estimated scope
|
||||
|
||||
- ~25-30 commits
|
||||
- ~2000 LOC removed (duplicated thread workers, search retries, etc.)
|
||||
- ~1200 LOC added (engine + per-source slim adapters)
|
||||
- Net reduction: ~800 LOC
|
||||
- ~50 new tests (Phase A pinning) + ~20 engine-level tests
|
||||
- 1-2 days of focused work
|
||||
|
||||
## Risk profile
|
||||
|
||||
**Low risk:**
|
||||
- Phase A (only adds tests, never changes behavior)
|
||||
- Phase B1 (new file, doesn't touch existing code)
|
||||
- Phase H (cleanup of dead code)
|
||||
|
||||
**Medium risk:**
|
||||
- Phase B2-B3 (state lift — race conditions, lock contention)
|
||||
- Phase C (thread worker extraction — semaphore semantics, exception propagation)
|
||||
- Phase G (contract narrows — anything reaching in for removed methods breaks)
|
||||
|
||||
**High risk:**
|
||||
- Phase D (search retry — easy to subtly change retry ladder shape)
|
||||
- Phase E (rate-limit — wrong order can cause deadlocks or under-limit violations)
|
||||
- Phase F (fallback — easy to accidentally change hybrid mode behavior)
|
||||
|
||||
**Mitigation:** Phase A pinning tests catch behavior drift in every later phase. Each commit must pass full suite. Manual smoke test per source after Phase C and again at end.
|
||||
|
||||
## Coordination with Cin
|
||||
|
||||
- Cin's metadata engine PR will likely set the precedent for HOW abstractions look (Protocol vs ABC, sync vs async, state location). This plan defaults to Protocol + async (matches what we already have) but easy to mirror Cin's exact pattern when his PR lands.
|
||||
- If his pattern differs significantly, we may need to redo some commits. Best mitigation: don't dig too deep on contract shape (Phase G) until his PR is visible. Phases A-C don't depend on contract shape; they're safe to do regardless.
|
||||
- Send Cin a heads-up before starting — he may have feedback on the plan that saves a redesign later.
|
||||
|
||||
## Compatibility commitments
|
||||
|
||||
- Backward-compat `orchestrator.soulseek` / `orchestrator.youtube` / etc. attributes preserved through every phase.
|
||||
- Backward-compat `clear_all_searches`, `_make_request`, `signal_download_completion`, etc. (Soulseek-specific reaches) preserved through every phase.
|
||||
- Frontend status dashboard keys (`deezer_dl` alias) preserved.
|
||||
- Config format unchanged.
|
||||
- DB schema unchanged.
|
||||
- API endpoint surface unchanged.
|
||||
|
||||
## What's NOT in this PR
|
||||
|
||||
- Cin's metadata engine work (separate, his domain)
|
||||
- Media server client refactor (different subsystem, separate PR)
|
||||
- Match engine refactor (different subsystem, separate PR)
|
||||
- Adding new download sources (out of scope; the new contract makes them easier later)
|
||||
|
|
@ -7,7 +7,7 @@ from core.spotify_client import SpotifyClient, Playlist as SpotifyPlaylist, Trac
|
|||
from core.plex_client import PlexClient, PlexTrackInfo
|
||||
from core.jellyfin_client import JellyfinClient
|
||||
from core.navidrome_client import NavidromeClient
|
||||
from core.soulseek_client import SoulseekClient
|
||||
from core.download_orchestrator import DownloadOrchestrator
|
||||
from core.matching_engine import MusicMatchingEngine, MatchResult
|
||||
|
||||
logger = get_logger("sync_service")
|
||||
|
|
@ -44,12 +44,12 @@ class SyncProgress:
|
|||
failed_tracks: int = 0
|
||||
|
||||
class PlaylistSyncService:
|
||||
def __init__(self, spotify_client: SpotifyClient, plex_client: PlexClient, soulseek_client: SoulseekClient, jellyfin_client: JellyfinClient = None, navidrome_client = None):
|
||||
def __init__(self, spotify_client: SpotifyClient, plex_client: PlexClient, download_orchestrator: DownloadOrchestrator, jellyfin_client: JellyfinClient = None, navidrome_client = None):
|
||||
self.spotify_client = spotify_client
|
||||
self.plex_client = plex_client
|
||||
self.jellyfin_client = jellyfin_client
|
||||
self.navidrome_client = navidrome_client
|
||||
self.soulseek_client = soulseek_client
|
||||
self.download_orchestrator = download_orchestrator
|
||||
self.progress_callbacks = {} # Playlist-specific progress callbacks
|
||||
self.syncing_playlists = set() # Track multiple syncing playlists
|
||||
self._cancelled = False
|
||||
|
|
@ -635,7 +635,7 @@ class PlaylistSyncService:
|
|||
query = self.matching_engine.generate_download_query(match_result.spotify_track)
|
||||
logger.info(f"Attempting to download: {query}")
|
||||
|
||||
download_id = await self.soulseek_client.search_and_download_best(query, expected_track=match_result.spotify_track)
|
||||
download_id = await self.download_orchestrator.search_and_download_best(query, expected_track=match_result.spotify_track)
|
||||
|
||||
if download_id:
|
||||
downloaded_count += 1
|
||||
|
|
|
|||
542
tests/downloads/test_background_download_worker.py
Normal file
542
tests/downloads/test_background_download_worker.py
Normal file
|
|
@ -0,0 +1,542 @@
|
|||
"""Tests for `BackgroundDownloadWorker` (Phase C1).
|
||||
|
||||
These tests pin the worker's state-machine semantics, semaphore
|
||||
serialization, rate-limit-delay behavior, and exception handling.
|
||||
Future phases (C2–C7) migrate each per-source client onto this
|
||||
worker — these tests stay green as the regression net.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import threading
|
||||
import time
|
||||
|
||||
from core.download_engine import DownloadEngine
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Dispatch — initial state + thread spawn
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_dispatch_returns_uuid_download_id():
|
||||
engine = DownloadEngine()
|
||||
|
||||
def impl(download_id, target_id, display_name):
|
||||
return '/tmp/file.flac'
|
||||
|
||||
download_id = engine.worker.dispatch(
|
||||
source_name='youtube',
|
||||
target_id='abc123',
|
||||
display_name='Some Song',
|
||||
original_filename='abc123||Some Song',
|
||||
impl_callable=impl,
|
||||
)
|
||||
assert len(download_id) == 36 # UUID4
|
||||
assert download_id.count('-') == 4
|
||||
|
||||
|
||||
def test_dispatch_inserts_initial_record_with_canonical_state():
|
||||
"""Pinning: initial record matches the legacy per-client shape so
|
||||
consumers reading the state dict via API or context-key lookup
|
||||
keep working unchanged after migration."""
|
||||
engine = DownloadEngine()
|
||||
captured = threading.Event()
|
||||
|
||||
def impl(download_id, target_id, display_name):
|
||||
captured.wait(timeout=1.0) # block so we can read 'Initializing' / 'InProgress' state
|
||||
return '/tmp/file.flac'
|
||||
|
||||
download_id = engine.worker.dispatch(
|
||||
source_name='youtube',
|
||||
target_id='abc',
|
||||
display_name='X',
|
||||
original_filename='abc||X',
|
||||
impl_callable=impl,
|
||||
)
|
||||
record = engine.get_record('youtube', download_id)
|
||||
assert record is not None
|
||||
assert record['id'] == download_id
|
||||
assert record['filename'] == 'abc||X'
|
||||
assert record['username'] == 'youtube'
|
||||
assert record['state'] in ('Initializing', 'InProgress, Downloading')
|
||||
assert record['progress'] == 0.0
|
||||
assert record['file_path'] is None
|
||||
captured.set() # release impl
|
||||
|
||||
|
||||
def test_dispatch_merges_extra_record_fields():
|
||||
"""Pinning: source-specific slots (video_id, track_id, etc.)
|
||||
merge into the initial record so frontend + status APIs that
|
||||
read those keys keep working."""
|
||||
engine = DownloadEngine()
|
||||
started = threading.Event()
|
||||
release = threading.Event()
|
||||
|
||||
def impl(download_id, target_id, display_name):
|
||||
started.set()
|
||||
release.wait(timeout=1.0)
|
||||
return '/tmp/x.flac'
|
||||
|
||||
download_id = engine.worker.dispatch(
|
||||
source_name='youtube',
|
||||
target_id='vid123',
|
||||
display_name='Title',
|
||||
original_filename='vid123||Title',
|
||||
impl_callable=impl,
|
||||
extra_record_fields={
|
||||
'video_id': 'vid123',
|
||||
'url': 'https://youtube.com/watch?v=vid123',
|
||||
'title': 'Title',
|
||||
},
|
||||
)
|
||||
started.wait(timeout=1.0)
|
||||
record = engine.get_record('youtube', download_id)
|
||||
assert record['video_id'] == 'vid123'
|
||||
assert record['url'] == 'https://youtube.com/watch?v=vid123'
|
||||
assert record['title'] == 'Title'
|
||||
release.set()
|
||||
|
||||
|
||||
def test_dispatch_username_override_preserves_legacy_slot():
|
||||
"""Pinning: Deezer's record stores `'deezer_dl'` (legacy) in the
|
||||
username slot, not the canonical `'deezer'`. Worker accepts
|
||||
override so frontend status indicators keep their key."""
|
||||
engine = DownloadEngine()
|
||||
release = threading.Event()
|
||||
|
||||
def impl(download_id, target_id, display_name):
|
||||
release.wait(timeout=1.0)
|
||||
return '/tmp/x.flac'
|
||||
|
||||
download_id = engine.worker.dispatch(
|
||||
source_name='deezer',
|
||||
target_id='999',
|
||||
display_name='X',
|
||||
original_filename='999||X',
|
||||
impl_callable=impl,
|
||||
username_override='deezer_dl',
|
||||
)
|
||||
record = engine.get_record('deezer', download_id)
|
||||
assert record['username'] == 'deezer_dl'
|
||||
release.set()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Worker lifecycle — state transitions
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_worker_marks_completed_on_successful_impl():
|
||||
engine = DownloadEngine()
|
||||
|
||||
def impl(download_id, target_id, display_name):
|
||||
return '/tmp/done.flac'
|
||||
|
||||
download_id = engine.worker.dispatch(
|
||||
source_name='youtube',
|
||||
target_id='vid',
|
||||
display_name='X',
|
||||
original_filename='vid||X',
|
||||
impl_callable=impl,
|
||||
)
|
||||
|
||||
# Wait for thread to finish.
|
||||
deadline = time.time() + 2.0
|
||||
while time.time() < deadline:
|
||||
record = engine.get_record('youtube', download_id)
|
||||
if record and record['state'] == 'Completed, Succeeded':
|
||||
break
|
||||
time.sleep(0.01)
|
||||
|
||||
record = engine.get_record('youtube', download_id)
|
||||
assert record['state'] == 'Completed, Succeeded'
|
||||
assert record['progress'] == 100.0
|
||||
assert record['file_path'] == '/tmp/done.flac'
|
||||
|
||||
|
||||
def test_worker_preserves_cancelled_when_impl_returns_none():
|
||||
"""Pinning: if the user cancels mid-download (state flips to
|
||||
Cancelled via engine.update_record from cancel_download), the
|
||||
worker must NOT clobber it back to Errored when impl returns
|
||||
None. The legacy per-client thread workers had this guard
|
||||
(``if state != 'Cancelled': state = 'Errored'``); the shared
|
||||
worker preserves that contract."""
|
||||
engine = DownloadEngine()
|
||||
|
||||
def impl(download_id, target_id, display_name):
|
||||
# Simulate user cancelling mid-impl by writing Cancelled.
|
||||
engine.update_record('youtube', download_id, {'state': 'Cancelled'})
|
||||
return None # impl returns None because download was interrupted
|
||||
|
||||
download_id = engine.worker.dispatch(
|
||||
source_name='youtube',
|
||||
target_id='vid',
|
||||
display_name='X',
|
||||
original_filename='vid||X',
|
||||
impl_callable=impl,
|
||||
)
|
||||
|
||||
deadline = time.time() + 2.0
|
||||
while time.time() < deadline:
|
||||
record = engine.get_record('youtube', download_id)
|
||||
if record and record['state'] in ('Cancelled', 'Errored'):
|
||||
break
|
||||
time.sleep(0.01)
|
||||
|
||||
record = engine.get_record('youtube', download_id)
|
||||
assert record['state'] == 'Cancelled', (
|
||||
f"Worker clobbered user's Cancelled with {record['state']}"
|
||||
)
|
||||
|
||||
|
||||
def test_worker_preserves_cancelled_when_impl_returns_success():
|
||||
"""Cin's bug 3 follow-up: the success path also has a read-then-write
|
||||
race. If the user cancels between the impl returning a valid file
|
||||
path and the worker writing 'Completed, Succeeded', the cancel is
|
||||
overwritten. The success-path write must use the same atomic
|
||||
Cancelled-preserve guard as _mark_terminal."""
|
||||
engine = DownloadEngine()
|
||||
|
||||
def impl(download_id, target_id, display_name):
|
||||
# User cancels mid-impl, then impl finishes successfully.
|
||||
engine.update_record('youtube', download_id, {'state': 'Cancelled'})
|
||||
return '/tmp/file.flac'
|
||||
|
||||
download_id = engine.worker.dispatch(
|
||||
source_name='youtube',
|
||||
target_id='vid',
|
||||
display_name='X',
|
||||
original_filename='vid||X',
|
||||
impl_callable=impl,
|
||||
)
|
||||
|
||||
deadline = time.time() + 2.0
|
||||
while time.time() < deadline:
|
||||
record = engine.get_record('youtube', download_id)
|
||||
if record and record['state'] in ('Cancelled', 'Completed, Succeeded'):
|
||||
break
|
||||
time.sleep(0.01)
|
||||
|
||||
record = engine.get_record('youtube', download_id)
|
||||
assert record['state'] == 'Cancelled', (
|
||||
f"Worker clobbered user's Cancelled with {record['state']}"
|
||||
)
|
||||
|
||||
|
||||
def test_worker_preserves_cancelled_when_impl_raises():
|
||||
"""Same Cancelled-preserve guard, but for the impl-raises path."""
|
||||
engine = DownloadEngine()
|
||||
|
||||
def impl(download_id, target_id, display_name):
|
||||
engine.update_record('youtube', download_id, {'state': 'Cancelled'})
|
||||
raise RuntimeError("simulated mid-cancel exception")
|
||||
|
||||
download_id = engine.worker.dispatch(
|
||||
source_name='youtube',
|
||||
target_id='vid',
|
||||
display_name='X',
|
||||
original_filename='vid||X',
|
||||
impl_callable=impl,
|
||||
)
|
||||
|
||||
deadline = time.time() + 2.0
|
||||
while time.time() < deadline:
|
||||
record = engine.get_record('youtube', download_id)
|
||||
if record and record['state'] in ('Cancelled', 'Errored'):
|
||||
break
|
||||
time.sleep(0.01)
|
||||
|
||||
assert engine.get_record('youtube', download_id)['state'] == 'Cancelled'
|
||||
|
||||
|
||||
def test_worker_marks_errored_when_impl_returns_none():
|
||||
engine = DownloadEngine()
|
||||
|
||||
def impl(download_id, target_id, display_name):
|
||||
return None # signaling failure
|
||||
|
||||
download_id = engine.worker.dispatch(
|
||||
source_name='youtube',
|
||||
target_id='vid',
|
||||
display_name='X',
|
||||
original_filename='vid||X',
|
||||
impl_callable=impl,
|
||||
)
|
||||
|
||||
deadline = time.time() + 2.0
|
||||
while time.time() < deadline:
|
||||
record = engine.get_record('youtube', download_id)
|
||||
if record and record['state'] == 'Errored':
|
||||
break
|
||||
time.sleep(0.01)
|
||||
|
||||
record = engine.get_record('youtube', download_id)
|
||||
assert record['state'] == 'Errored'
|
||||
# file_path stays None (default).
|
||||
assert record['file_path'] is None
|
||||
|
||||
|
||||
def test_worker_marks_errored_and_captures_message_when_impl_raises():
|
||||
engine = DownloadEngine()
|
||||
|
||||
def impl(download_id, target_id, display_name):
|
||||
raise RuntimeError("api blew up")
|
||||
|
||||
download_id = engine.worker.dispatch(
|
||||
source_name='youtube',
|
||||
target_id='vid',
|
||||
display_name='X',
|
||||
original_filename='vid||X',
|
||||
impl_callable=impl,
|
||||
)
|
||||
|
||||
deadline = time.time() + 2.0
|
||||
while time.time() < deadline:
|
||||
record = engine.get_record('youtube', download_id)
|
||||
if record and record['state'] == 'Errored':
|
||||
break
|
||||
time.sleep(0.01)
|
||||
|
||||
record = engine.get_record('youtube', download_id)
|
||||
assert record['state'] == 'Errored'
|
||||
assert 'api blew up' in record.get('error', '')
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Per-source semaphore serialization
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_semaphore_serializes_downloads_for_same_source():
|
||||
"""Pinning: with concurrency=1 (default), two dispatches against
|
||||
the same source run sequentially. The legacy per-client
|
||||
semaphore did the same — consumers depend on this for
|
||||
rate-limit safety against APIs like YouTube."""
|
||||
engine = DownloadEngine()
|
||||
in_progress = threading.Event()
|
||||
can_finish = threading.Event()
|
||||
overlap_count = 0
|
||||
overlap_lock = threading.Lock()
|
||||
active_count = [0]
|
||||
|
||||
def impl(download_id, target_id, display_name):
|
||||
nonlocal overlap_count
|
||||
with overlap_lock:
|
||||
active_count[0] += 1
|
||||
if active_count[0] > 1:
|
||||
overlap_count += 1
|
||||
in_progress.set()
|
||||
can_finish.wait(timeout=2.0)
|
||||
with overlap_lock:
|
||||
active_count[0] -= 1
|
||||
return '/tmp/x.flac'
|
||||
|
||||
# Default concurrency=1 — two dispatches must serialize.
|
||||
dl1 = engine.worker.dispatch(
|
||||
source_name='youtube', target_id='a', display_name='A',
|
||||
original_filename='a||A', impl_callable=impl,
|
||||
)
|
||||
in_progress.wait(timeout=1.0)
|
||||
in_progress.clear()
|
||||
dl2 = engine.worker.dispatch(
|
||||
source_name='youtube', target_id='b', display_name='B',
|
||||
original_filename='b||B', impl_callable=impl,
|
||||
)
|
||||
# Give second dispatch a chance to attempt running in parallel
|
||||
# (it should be blocked on the semaphore).
|
||||
time.sleep(0.1)
|
||||
assert overlap_count == 0, "second dispatch should be blocked behind semaphore"
|
||||
|
||||
# Release first; second proceeds.
|
||||
can_finish.set()
|
||||
|
||||
# Wait for both to finish.
|
||||
deadline = time.time() + 3.0
|
||||
while time.time() < deadline:
|
||||
r1 = engine.get_record('youtube', dl1)
|
||||
r2 = engine.get_record('youtube', dl2)
|
||||
if r1 and r2 and r1['state'] == 'Completed, Succeeded' and r2['state'] == 'Completed, Succeeded':
|
||||
break
|
||||
time.sleep(0.01)
|
||||
|
||||
assert overlap_count == 0
|
||||
|
||||
|
||||
def test_semaphore_concurrency_can_be_increased():
|
||||
"""When `set_concurrency(source, N)` is called, N downloads can
|
||||
run in parallel for that source. Used by sources that support
|
||||
parallel transfers (none today, but contract supports it)."""
|
||||
engine = DownloadEngine()
|
||||
engine.worker.set_concurrency('parallel-source', 3)
|
||||
|
||||
in_flight = []
|
||||
in_flight_lock = threading.Lock()
|
||||
can_finish = threading.Event()
|
||||
max_observed = [0]
|
||||
|
||||
def impl(download_id, target_id, display_name):
|
||||
with in_flight_lock:
|
||||
in_flight.append(download_id)
|
||||
max_observed[0] = max(max_observed[0], len(in_flight))
|
||||
can_finish.wait(timeout=2.0)
|
||||
with in_flight_lock:
|
||||
in_flight.remove(download_id)
|
||||
return '/tmp/x.flac'
|
||||
|
||||
for i in range(3):
|
||||
engine.worker.dispatch(
|
||||
source_name='parallel-source',
|
||||
target_id=str(i),
|
||||
display_name=f'd{i}',
|
||||
original_filename=f'{i}||d{i}',
|
||||
impl_callable=impl,
|
||||
)
|
||||
# Give threads time to ramp up.
|
||||
time.sleep(0.2)
|
||||
can_finish.set()
|
||||
|
||||
# Wait for them to finish.
|
||||
time.sleep(0.5)
|
||||
assert max_observed[0] == 3
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Per-source rate-limit delay
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_impl_can_observe_cancel_mid_flight_via_state_check():
|
||||
"""Per JohnBaumb: existing tests cover Cancelled-preserve AFTER impl
|
||||
returns — but plugins also poll engine state mid-download (via
|
||||
``_is_cancelled`` helpers) to abort partial transfers. Pin that
|
||||
contract: a cancel landing while impl is mid-flight must be
|
||||
visible to a subsequent ``engine.get_record()`` from the impl
|
||||
thread.
|
||||
"""
|
||||
engine = DownloadEngine()
|
||||
impl_started = threading.Event()
|
||||
impl_can_finish = threading.Event()
|
||||
observed_state_during_impl = []
|
||||
|
||||
def impl(download_id, target_id, display_name):
|
||||
impl_started.set()
|
||||
# Wait for the test thread to write Cancelled, then check
|
||||
# what we can observe from inside the impl callback.
|
||||
impl_can_finish.wait(timeout=2.0)
|
||||
record = engine.get_record('youtube', download_id)
|
||||
observed_state_during_impl.append(record.get('state') if record else None)
|
||||
return None # impl noticed cancel + bailed out
|
||||
|
||||
download_id = engine.worker.dispatch(
|
||||
source_name='youtube',
|
||||
target_id='vid',
|
||||
display_name='X',
|
||||
original_filename='vid||X',
|
||||
impl_callable=impl,
|
||||
)
|
||||
|
||||
# Wait for impl to start, then inject a cancel.
|
||||
impl_started.wait(timeout=1.0)
|
||||
engine.update_record('youtube', download_id, {'state': 'Cancelled'})
|
||||
impl_can_finish.set()
|
||||
|
||||
# Wait for impl to finish (its append populates observed_state).
|
||||
# Engine state is already Cancelled at this point, so polling on
|
||||
# state would race: it'd break before impl ran the get_record line.
|
||||
deadline = time.time() + 2.0
|
||||
while not observed_state_during_impl and time.time() < deadline:
|
||||
time.sleep(0.01)
|
||||
|
||||
# impl observed the Cancelled state mid-flight via get_record,
|
||||
# AND the worker preserved Cancelled after impl returned None.
|
||||
assert observed_state_during_impl == ['Cancelled']
|
||||
assert engine.get_record('youtube', download_id)['state'] == 'Cancelled'
|
||||
|
||||
|
||||
def test_per_source_delays_dont_block_other_sources():
|
||||
"""Per JohnBaumb: per-source semaphores + delays must not let one
|
||||
slow source stall another. YouTube's 3s rate-limit delay should
|
||||
not delay a Tidal download starting in parallel.
|
||||
|
||||
Configure YouTube with a 0.5s delay, dispatch one YouTube download
|
||||
(which holds the source's serial slot + arms the next-call delay),
|
||||
then immediately dispatch a Tidal download. Tidal must complete
|
||||
well before YouTube's delay window would have elapsed.
|
||||
"""
|
||||
engine = DownloadEngine()
|
||||
engine.worker.set_delay('youtube', 0.5)
|
||||
|
||||
yt_completed = threading.Event()
|
||||
td_completed = threading.Event()
|
||||
|
||||
def yt_impl(download_id, target_id, display_name):
|
||||
time.sleep(0.05)
|
||||
yt_completed.set()
|
||||
return '/tmp/yt.mp3'
|
||||
|
||||
def td_impl(download_id, target_id, display_name):
|
||||
td_completed.set()
|
||||
return '/tmp/td.flac'
|
||||
|
||||
# First YouTube call. After it finishes, the worker arms the
|
||||
# 0.5s delay BEFORE the next youtube dispatch can run.
|
||||
engine.worker.dispatch(
|
||||
source_name='youtube', target_id='a', display_name='A',
|
||||
original_filename='a||A', impl_callable=yt_impl,
|
||||
)
|
||||
yt_completed.wait(timeout=1.0)
|
||||
|
||||
# Second YouTube would now block 0.5s on the rate-limit delay.
|
||||
# Dispatch one to occupy that wait, then dispatch Tidal — Tidal
|
||||
# must NOT wait on YouTube's delay.
|
||||
engine.worker.dispatch(
|
||||
source_name='youtube', target_id='b', display_name='B',
|
||||
original_filename='b||B', impl_callable=lambda *a: '/tmp/y.mp3',
|
||||
)
|
||||
|
||||
td_start = time.time()
|
||||
engine.worker.dispatch(
|
||||
source_name='tidal', target_id='t1', display_name='T',
|
||||
original_filename='t1||T', impl_callable=td_impl,
|
||||
)
|
||||
td_completed.wait(timeout=0.4)
|
||||
td_elapsed = time.time() - td_start
|
||||
|
||||
# Tidal must have finished in well under 0.5s (the YouTube delay).
|
||||
assert td_completed.is_set(), "Tidal blocked on YouTube's per-source delay"
|
||||
assert td_elapsed < 0.4, f"Tidal took {td_elapsed:.2f}s — should be near-instant"
|
||||
|
||||
|
||||
def test_delay_enforces_minimum_gap_between_downloads():
|
||||
"""Pinning: YouTube uses 3s delay today (legacy
|
||||
`_download_delay`). Worker-driven delay must enforce the same
|
||||
gap so YouTube doesn't 429."""
|
||||
engine = DownloadEngine()
|
||||
engine.worker.set_delay('youtube', 0.2) # 200ms — short for test speed
|
||||
|
||||
completion_times = []
|
||||
|
||||
def impl(download_id, target_id, display_name):
|
||||
completion_times.append(time.time())
|
||||
return '/tmp/x.flac'
|
||||
|
||||
# Two back-to-back dispatches.
|
||||
engine.worker.dispatch(
|
||||
source_name='youtube', target_id='a', display_name='A',
|
||||
original_filename='a||A', impl_callable=impl,
|
||||
)
|
||||
engine.worker.dispatch(
|
||||
source_name='youtube', target_id='b', display_name='B',
|
||||
original_filename='b||B', impl_callable=impl,
|
||||
)
|
||||
|
||||
# Wait for both to finish (semaphore serializes + delay).
|
||||
deadline = time.time() + 3.0
|
||||
while time.time() < deadline and len(completion_times) < 2:
|
||||
time.sleep(0.01)
|
||||
|
||||
assert len(completion_times) == 2
|
||||
gap = completion_times[1] - completion_times[0]
|
||||
# Gap is at LEAST the configured delay.
|
||||
assert gap >= 0.18, f"expected gap >= 0.2s, got {gap:.3f}"
|
||||
154
tests/downloads/test_deezer_pinning.py
Normal file
154
tests/downloads/test_deezer_pinning.py
Normal file
|
|
@ -0,0 +1,154 @@
|
|||
"""Phase A pinning tests for DeezerDownloadClient — UPDATED for Phase C6.
|
||||
|
||||
Deezer has the same engine-driven dispatch as the other streaming
|
||||
sources, with three Deezer-specific quirks preserved:
|
||||
- track_id stays as STRING (Deezer GW API uses string IDs).
|
||||
- Engine record's `username` slot is the legacy `'deezer_dl'`
|
||||
via worker username_override.
|
||||
- Worker thread is named `deezer-dl-<track_id>` for diagnostics.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import threading
|
||||
from pathlib import Path
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
|
||||
from core.download_engine import DownloadEngine
|
||||
from core.deezer_download_client import DeezerDownloadClient
|
||||
|
||||
|
||||
def _run_async(coro):
|
||||
loop = asyncio.new_event_loop()
|
||||
try:
|
||||
return loop.run_until_complete(coro)
|
||||
finally:
|
||||
loop.close()
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def deezer_client_with_engine():
|
||||
client = DeezerDownloadClient.__new__(DeezerDownloadClient)
|
||||
client.download_path = Path('./test_deezer_downloads')
|
||||
client.shutdown_check = None
|
||||
client._authenticated = True
|
||||
client._engine = None
|
||||
engine = DownloadEngine()
|
||||
client.set_engine(engine)
|
||||
return client, engine
|
||||
|
||||
|
||||
def test_download_returns_none_when_not_authenticated(deezer_client_with_engine):
|
||||
client, _ = deezer_client_with_engine
|
||||
client._authenticated = False
|
||||
result = _run_async(client.download('deezer_dl', '12345||x', 0))
|
||||
assert result is None
|
||||
|
||||
|
||||
def test_download_raises_when_engine_not_wired():
|
||||
"""Defensive: client without engine reference must raise so the
|
||||
orchestrator's download_with_fallback surfaces the error and
|
||||
moves on to the next source. Returning None silently would drop
|
||||
the download with no user feedback (per JohnBaumb)."""
|
||||
import pytest
|
||||
client = DeezerDownloadClient.__new__(DeezerDownloadClient)
|
||||
client._engine = None
|
||||
# Bypass auth gate so we exercise the engine check.
|
||||
client._authenticated = True
|
||||
with pytest.raises(RuntimeError, match="engine reference"):
|
||||
_run_async(client.download('deezer', 'v||t', 0))
|
||||
|
||||
|
||||
def test_download_track_id_stays_as_string(deezer_client_with_engine):
|
||||
"""Pinning: Deezer GW API uses string IDs — engine record must
|
||||
keep track_id as str."""
|
||||
client, engine = deezer_client_with_engine
|
||||
started = threading.Event()
|
||||
release = threading.Event()
|
||||
|
||||
def slow_impl(*args, **kwargs):
|
||||
started.set()
|
||||
release.wait(timeout=1.0)
|
||||
return '/tmp/done.flac'
|
||||
|
||||
with patch.object(client, '_download_sync', side_effect=slow_impl):
|
||||
download_id = _run_async(client.download('deezer_dl', '999||X', 0))
|
||||
started.wait(timeout=1.0)
|
||||
record = engine.get_record('deezer', download_id)
|
||||
assert record['track_id'] == '999'
|
||||
assert isinstance(record['track_id'], str)
|
||||
release.set()
|
||||
|
||||
|
||||
def test_download_username_slot_is_legacy_deezer_dl(deezer_client_with_engine):
|
||||
"""Pinning: frontend status indicators key off `'deezer_dl'`,
|
||||
not the canonical `'deezer'`."""
|
||||
client, engine = deezer_client_with_engine
|
||||
started = threading.Event()
|
||||
release = threading.Event()
|
||||
|
||||
def slow_impl(*args, **kwargs):
|
||||
started.set()
|
||||
release.wait(timeout=1.0)
|
||||
return '/tmp/done.flac'
|
||||
|
||||
with patch.object(client, '_download_sync', side_effect=slow_impl):
|
||||
download_id = _run_async(client.download('deezer_dl', '999||x', 0))
|
||||
started.wait(timeout=1.0)
|
||||
assert engine.get_record('deezer', download_id)['username'] == 'deezer_dl'
|
||||
release.set()
|
||||
|
||||
|
||||
def test_download_handles_missing_display_name_with_fallback(deezer_client_with_engine):
|
||||
"""Pinning: filename without `||` synthesizes display name `Track <id>`."""
|
||||
client, engine = deezer_client_with_engine
|
||||
started = threading.Event()
|
||||
release = threading.Event()
|
||||
|
||||
def slow_impl(*args, **kwargs):
|
||||
started.set()
|
||||
release.wait(timeout=1.0)
|
||||
return '/tmp/x.flac'
|
||||
|
||||
with patch.object(client, '_download_sync', side_effect=slow_impl):
|
||||
download_id = _run_async(client.download('deezer_dl', '12345', 0))
|
||||
started.wait(timeout=1.0)
|
||||
assert engine.get_record('deezer', download_id)['display_name'] == 'Track 12345'
|
||||
release.set()
|
||||
|
||||
|
||||
def test_download_engine_record_carries_error_slot(deezer_client_with_engine):
|
||||
"""Pinning: Deezer-specific `error` slot for ARL re-auth failure
|
||||
messages must be present on init."""
|
||||
client, engine = deezer_client_with_engine
|
||||
started = threading.Event()
|
||||
release = threading.Event()
|
||||
|
||||
def slow_impl(*args, **kwargs):
|
||||
started.set()
|
||||
release.wait(timeout=1.0)
|
||||
return '/tmp/x.flac'
|
||||
|
||||
with patch.object(client, '_download_sync', side_effect=slow_impl):
|
||||
download_id = _run_async(client.download('deezer_dl', '999||X', 1024))
|
||||
started.wait(timeout=1.0)
|
||||
record = engine.get_record('deezer', download_id)
|
||||
assert 'error' in record
|
||||
assert record['error'] is None
|
||||
assert record['size'] == 1024
|
||||
release.set()
|
||||
|
||||
|
||||
def test_get_all_downloads_reads_engine_records(deezer_client_with_engine):
|
||||
client, engine = deezer_client_with_engine
|
||||
engine.add_record('deezer', 'dl-1', {
|
||||
'id': 'dl-1', 'filename': '111||A', 'username': 'deezer_dl',
|
||||
'state': 'InProgress, Downloading', 'progress': 50.0,
|
||||
})
|
||||
result = _run_async(client.get_all_downloads())
|
||||
assert len(result) == 1
|
||||
assert result[0].id == 'dl-1'
|
||||
assert result[0].username == 'deezer_dl'
|
||||
724
tests/downloads/test_download_engine.py
Normal file
724
tests/downloads/test_download_engine.py
Normal file
|
|
@ -0,0 +1,724 @@
|
|||
"""Tests for the DownloadEngine skeleton (Phase B).
|
||||
|
||||
Pinning the engine's state-storage contract: add/update/remove,
|
||||
per-source iteration, find-by-id, plugin registration, lock-held
|
||||
mutations vs lock-released reads. Future phases (C/D/E/F) bolt
|
||||
behavior on top of this surface — these tests stay green and act
|
||||
as the regression net while behavior moves in.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import threading
|
||||
|
||||
import pytest
|
||||
|
||||
from core.download_engine import DownloadEngine
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Plugin registration
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_register_plugin_stores_under_source_name():
|
||||
engine = DownloadEngine()
|
||||
plugin = object()
|
||||
engine.register_plugin('soulseek', plugin)
|
||||
assert engine.get_plugin('soulseek') is plugin
|
||||
assert 'soulseek' in engine.registered_sources()
|
||||
|
||||
|
||||
def test_get_plugin_returns_none_for_unknown_source():
|
||||
engine = DownloadEngine()
|
||||
assert engine.get_plugin('made_up') is None
|
||||
|
||||
|
||||
def test_register_plugin_swallows_set_engine_failure(caplog):
|
||||
"""Per JohnBaumb: if a plugin's ``set_engine`` callback raises,
|
||||
registration shouldn't take down the engine — the failure gets
|
||||
logged + the plugin stays registered. The plugin's own download()
|
||||
method is responsible for surfacing the missing-engine state to
|
||||
the user (see ``test_download_raises_when_engine_not_wired`` per
|
||||
source). Pinning this so a future refactor can't accidentally
|
||||
propagate the set_engine exception and crash boot.
|
||||
"""
|
||||
class _BrokenSetEngine:
|
||||
def set_engine(self, engine):
|
||||
raise RuntimeError("plugin's set_engine blew up")
|
||||
|
||||
engine = DownloadEngine()
|
||||
plugin = _BrokenSetEngine()
|
||||
# Should not raise.
|
||||
engine.register_plugin('flaky', plugin)
|
||||
|
||||
# Plugin still registered + lookupable.
|
||||
assert engine.get_plugin('flaky') is plugin
|
||||
assert 'flaky' in engine.registered_sources()
|
||||
|
||||
|
||||
def test_register_plugin_overwrites_on_duplicate(caplog):
|
||||
"""Re-registering under the same name overwrites and warns. Not a
|
||||
common path but useful so test fixtures that build a fresh engine
|
||||
can swap a mock plugin in without setup gymnastics."""
|
||||
engine = DownloadEngine()
|
||||
first = object()
|
||||
second = object()
|
||||
engine.register_plugin('soulseek', first)
|
||||
engine.register_plugin('soulseek', second)
|
||||
assert engine.get_plugin('soulseek') is second
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Active-download state — add / get / update / remove
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_add_record_inserts_under_composite_key():
|
||||
engine = DownloadEngine()
|
||||
engine.add_record('youtube', 'dl-1', {'state': 'Initializing', 'progress': 0.0})
|
||||
|
||||
rec = engine.get_record('youtube', 'dl-1')
|
||||
assert rec is not None
|
||||
assert rec['state'] == 'Initializing'
|
||||
assert rec['progress'] == 0.0
|
||||
|
||||
|
||||
def test_get_record_returns_shallow_copy():
|
||||
"""Mutating the returned dict must NOT affect engine state.
|
||||
Engine reads should be safe to hold / iterate without locks."""
|
||||
engine = DownloadEngine()
|
||||
engine.add_record('youtube', 'dl-1', {'state': 'Initializing'})
|
||||
|
||||
rec = engine.get_record('youtube', 'dl-1')
|
||||
rec['state'] = 'TamperedByCaller'
|
||||
|
||||
# Engine state still has the original.
|
||||
fresh = engine.get_record('youtube', 'dl-1')
|
||||
assert fresh['state'] == 'Initializing'
|
||||
|
||||
|
||||
def test_update_record_applies_partial_patch():
|
||||
engine = DownloadEngine()
|
||||
engine.add_record('tidal', 'dl-2', {'state': 'Initializing', 'progress': 0.0,
|
||||
'file_path': None})
|
||||
|
||||
engine.update_record('tidal', 'dl-2', {'state': 'Completed, Succeeded',
|
||||
'progress': 100.0,
|
||||
'file_path': '/tmp/song.flac'})
|
||||
|
||||
rec = engine.get_record('tidal', 'dl-2')
|
||||
assert rec['state'] == 'Completed, Succeeded'
|
||||
assert rec['progress'] == 100.0
|
||||
assert rec['file_path'] == '/tmp/song.flac'
|
||||
|
||||
|
||||
def test_update_record_is_noop_when_record_removed():
|
||||
"""If a record was removed (e.g. user cancelled mid-download),
|
||||
the worker thread's late update is silently dropped — never
|
||||
raises. Mirrors the per-client `if download_id in active_downloads`
|
||||
guard pattern that's all over the existing clients."""
|
||||
engine = DownloadEngine()
|
||||
engine.add_record('tidal', 'dl-2', {'state': 'Initializing'})
|
||||
engine.remove_record('tidal', 'dl-2')
|
||||
|
||||
# Should not raise.
|
||||
engine.update_record('tidal', 'dl-2', {'state': 'Completed, Succeeded'})
|
||||
|
||||
assert engine.get_record('tidal', 'dl-2') is None
|
||||
|
||||
|
||||
def test_remove_record_returns_removed_record():
|
||||
engine = DownloadEngine()
|
||||
engine.add_record('qobuz', 'dl-3', {'state': 'InProgress'})
|
||||
|
||||
removed = engine.remove_record('qobuz', 'dl-3')
|
||||
assert removed is not None
|
||||
assert removed['state'] == 'InProgress'
|
||||
assert engine.get_record('qobuz', 'dl-3') is None
|
||||
|
||||
|
||||
def test_remove_record_returns_none_when_missing():
|
||||
engine = DownloadEngine()
|
||||
assert engine.remove_record('qobuz', 'never-existed') is None
|
||||
|
||||
|
||||
def test_per_source_locks_dont_block_each_other():
|
||||
"""Per JohnBaumb: each source must have its own lock so a long-held
|
||||
write on one source doesn't block writes on another. Pre-refactor
|
||||
each client owned its own download lock; the engine has to match
|
||||
that semantic.
|
||||
|
||||
Hold source-A's lock from one thread, then mutate source-B from
|
||||
another thread. Source-B's mutation must complete promptly even
|
||||
while source-A is locked.
|
||||
"""
|
||||
engine = DownloadEngine()
|
||||
engine.add_record('youtube', 'yt-1', {'state': 'InProgress'})
|
||||
engine.add_record('tidal', 'td-1', {'state': 'InProgress'})
|
||||
|
||||
held = threading.Event()
|
||||
release = threading.Event()
|
||||
other_done = threading.Event()
|
||||
|
||||
def hold_youtube_lock():
|
||||
with engine._source_lock('youtube'):
|
||||
held.set()
|
||||
release.wait(timeout=2.0)
|
||||
|
||||
def update_tidal_while_youtube_held():
|
||||
held.wait(timeout=1.0)
|
||||
engine.update_record('tidal', 'td-1', {'state': 'Completed, Succeeded'})
|
||||
other_done.set()
|
||||
|
||||
holder = threading.Thread(target=hold_youtube_lock)
|
||||
other = threading.Thread(target=update_tidal_while_youtube_held)
|
||||
holder.start()
|
||||
other.start()
|
||||
|
||||
# Tidal write must complete even though YouTube's lock is held.
|
||||
assert other_done.wait(timeout=1.0), (
|
||||
"Tidal mutation blocked by YouTube's lock — sources are not "
|
||||
"independently shardable"
|
||||
)
|
||||
|
||||
release.set()
|
||||
holder.join()
|
||||
other.join()
|
||||
assert engine.get_record('tidal', 'td-1')['state'] == 'Completed, Succeeded'
|
||||
|
||||
|
||||
def test_remove_record_drops_empty_source_bucket():
|
||||
"""Per JohnBaumb: nested layout makes per-source iteration
|
||||
O(source_records). Removing the last record for a source must
|
||||
also drop the empty source bucket so future iter_records_for_source
|
||||
calls don't see a stale source key."""
|
||||
engine = DownloadEngine()
|
||||
engine.add_record('youtube', 'yt-1', {'title': 'A'})
|
||||
engine.remove_record('youtube', 'yt-1')
|
||||
|
||||
# Source bucket gone — internal state matches the contract.
|
||||
assert 'youtube' not in engine._records
|
||||
# Public surface still answers correctly.
|
||||
assert list(engine.iter_records_for_source('youtube')) == []
|
||||
assert engine.get_record('youtube', 'yt-1') is None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Iteration
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_iter_records_for_source_filters_correctly():
|
||||
engine = DownloadEngine()
|
||||
engine.add_record('youtube', 'yt-1', {'title': 'A'})
|
||||
engine.add_record('youtube', 'yt-2', {'title': 'B'})
|
||||
engine.add_record('tidal', 'td-1', {'title': 'C'})
|
||||
|
||||
yt_records = list(engine.iter_records_for_source('youtube'))
|
||||
assert len(yt_records) == 2
|
||||
assert {r['title'] for r in yt_records} == {'A', 'B'}
|
||||
|
||||
td_records = list(engine.iter_records_for_source('tidal'))
|
||||
assert len(td_records) == 1
|
||||
assert td_records[0]['title'] == 'C'
|
||||
|
||||
|
||||
def test_iter_yields_shallow_copies():
|
||||
"""Iteration returns COPIES — caller can hold the list and mutate
|
||||
each record without affecting engine state. Important: future
|
||||
Phase B3's `get_all_downloads` will iterate then build
|
||||
DownloadStatus objects from the snapshots."""
|
||||
engine = DownloadEngine()
|
||||
engine.add_record('youtube', 'yt-1', {'title': 'A'})
|
||||
|
||||
snapshot = list(engine.iter_records_for_source('youtube'))
|
||||
snapshot[0]['title'] = 'TAMPERED'
|
||||
|
||||
fresh = engine.get_record('youtube', 'yt-1')
|
||||
assert fresh['title'] == 'A'
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Thread safety — basic concurrent-mutation smoke
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_concurrent_adds_dont_lose_records():
|
||||
"""Hammer the engine with concurrent add_record from multiple
|
||||
threads. With proper locking, every record lands in state.
|
||||
Future Phase C BackgroundDownloadWorker spawns N threads doing
|
||||
exactly this kind of mutation."""
|
||||
engine = DownloadEngine()
|
||||
|
||||
def add_records(source, base):
|
||||
for i in range(50):
|
||||
engine.add_record(source, f'{base}-{i}', {'i': i})
|
||||
|
||||
threads = [
|
||||
threading.Thread(target=add_records, args=(f'src-{n}', f'dl-{n}'))
|
||||
for n in range(4)
|
||||
]
|
||||
for t in threads:
|
||||
t.start()
|
||||
for t in threads:
|
||||
t.join()
|
||||
|
||||
total = sum(
|
||||
1
|
||||
for n in range(4)
|
||||
for _ in engine.iter_records_for_source(f'src-{n}')
|
||||
)
|
||||
assert total == 4 * 50 # 200 records, none lost
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Cross-source query dispatch (Phase B2)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _run_async(coro):
|
||||
import asyncio
|
||||
loop = asyncio.new_event_loop()
|
||||
try:
|
||||
return loop.run_until_complete(coro)
|
||||
finally:
|
||||
loop.close()
|
||||
|
||||
|
||||
class _FakePlugin:
|
||||
"""Minimal plugin double for engine query tests. Exposes the
|
||||
methods engine.get_all_downloads / get_download_status /
|
||||
cancel_download / clear_all_completed_downloads call."""
|
||||
|
||||
def __init__(self, name, configured=True, downloads=None,
|
||||
cancel_result=True, clear_result=True):
|
||||
self.name = name
|
||||
self._configured = configured
|
||||
self._downloads = downloads or []
|
||||
self._cancel_result = cancel_result
|
||||
self._clear_result = clear_result
|
||||
self.cancel_calls = []
|
||||
self.clear_calls = 0
|
||||
|
||||
def is_configured(self):
|
||||
return self._configured
|
||||
|
||||
async def get_all_downloads(self):
|
||||
return list(self._downloads)
|
||||
|
||||
async def get_download_status(self, download_id):
|
||||
for d in self._downloads:
|
||||
if getattr(d, 'id', None) == download_id:
|
||||
return d
|
||||
return None
|
||||
|
||||
async def cancel_download(self, download_id, source_hint, remove):
|
||||
self.cancel_calls.append((download_id, source_hint, remove))
|
||||
return self._cancel_result
|
||||
|
||||
async def clear_all_completed_downloads(self):
|
||||
self.clear_calls += 1
|
||||
return self._clear_result
|
||||
|
||||
|
||||
class _FakeStatus:
|
||||
def __init__(self, id, source):
|
||||
self.id = id
|
||||
self.source = source
|
||||
|
||||
|
||||
def test_engine_get_all_downloads_aggregates_across_plugins():
|
||||
"""Engine concatenates every plugin's get_all_downloads output —
|
||||
same behavior as the legacy orchestrator."""
|
||||
engine = DownloadEngine()
|
||||
yt_plugin = _FakePlugin('youtube', downloads=[_FakeStatus('yt-1', 'youtube')])
|
||||
td_plugin = _FakePlugin('tidal', downloads=[_FakeStatus('td-1', 'tidal'),
|
||||
_FakeStatus('td-2', 'tidal')])
|
||||
engine.register_plugin('youtube', yt_plugin)
|
||||
engine.register_plugin('tidal', td_plugin)
|
||||
|
||||
result = _run_async(engine.get_all_downloads())
|
||||
assert len(result) == 3
|
||||
assert {r.id for r in result} == {'yt-1', 'td-1', 'td-2'}
|
||||
|
||||
|
||||
def test_engine_get_all_downloads_excludes_dont_invoke_plugin():
|
||||
"""Per JohnBaumb: the monitor calls engine.get_all_downloads(
|
||||
exclude=('soulseek',)) AFTER already pulling slskd transfers via
|
||||
the transfers/downloads endpoint. The whole point of the exclude
|
||||
is to NOT touch soulseek's plugin a second time — so the plugin's
|
||||
get_all_downloads must literally not be called when its name is
|
||||
in the exclude list. Pin that semantic explicitly (a test that
|
||||
just checks IDs would pass even if soulseek was called and
|
||||
returned [])."""
|
||||
engine = DownloadEngine()
|
||||
sl_plugin = _FakePlugin('soulseek', downloads=[_FakeStatus('sl-1', 'soulseek')])
|
||||
yt_plugin = _FakePlugin('youtube', downloads=[_FakeStatus('yt-1', 'youtube')])
|
||||
engine.register_plugin('soulseek', sl_plugin)
|
||||
engine.register_plugin('youtube', yt_plugin)
|
||||
|
||||
# Sentinel — flips True if get_all_downloads is called on soulseek.
|
||||
soulseek_called = []
|
||||
original_get_all = sl_plugin.get_all_downloads
|
||||
async def _tracking_get_all():
|
||||
soulseek_called.append(True)
|
||||
return await original_get_all()
|
||||
sl_plugin.get_all_downloads = _tracking_get_all
|
||||
|
||||
_run_async(engine.get_all_downloads(exclude=('soulseek',)))
|
||||
assert soulseek_called == [], (
|
||||
"soulseek's get_all_downloads was invoked despite being in exclude — "
|
||||
"monitor would still double-fetch slskd"
|
||||
)
|
||||
|
||||
|
||||
def test_engine_get_all_downloads_skips_excluded_sources():
|
||||
"""Per JohnBaumb: monitor pulls slskd transfers via the
|
||||
transfers/downloads endpoint earlier in its loop, so engine
|
||||
aggregation must skip soulseek to avoid double-fetching."""
|
||||
engine = DownloadEngine()
|
||||
sl_plugin = _FakePlugin('soulseek', downloads=[_FakeStatus('sl-1', 'soulseek')])
|
||||
yt_plugin = _FakePlugin('youtube', downloads=[_FakeStatus('yt-1', 'youtube')])
|
||||
td_plugin = _FakePlugin('tidal', downloads=[_FakeStatus('td-1', 'tidal')])
|
||||
engine.register_plugin('soulseek', sl_plugin)
|
||||
engine.register_plugin('youtube', yt_plugin)
|
||||
engine.register_plugin('tidal', td_plugin)
|
||||
|
||||
result = _run_async(engine.get_all_downloads(exclude=('soulseek',)))
|
||||
ids = {r.id for r in result}
|
||||
assert ids == {'yt-1', 'td-1'}
|
||||
assert 'sl-1' not in ids
|
||||
|
||||
|
||||
def test_engine_get_all_downloads_swallows_per_plugin_exceptions():
|
||||
"""One plugin throwing must NOT take down the whole list — same
|
||||
defensive behavior as the legacy orchestrator (matched by
|
||||
`try ... except: pass` on every iteration)."""
|
||||
engine = DownloadEngine()
|
||||
|
||||
class _BrokenPlugin:
|
||||
async def get_all_downloads(self):
|
||||
raise RuntimeError("boom")
|
||||
|
||||
yt_plugin = _FakePlugin('youtube', downloads=[_FakeStatus('yt-1', 'youtube')])
|
||||
engine.register_plugin('broken', _BrokenPlugin())
|
||||
engine.register_plugin('youtube', yt_plugin)
|
||||
|
||||
result = _run_async(engine.get_all_downloads())
|
||||
assert [r.id for r in result] == ['yt-1']
|
||||
|
||||
|
||||
def test_engine_get_download_status_returns_first_match():
|
||||
engine = DownloadEngine()
|
||||
yt_plugin = _FakePlugin('youtube', downloads=[_FakeStatus('shared', 'youtube')])
|
||||
td_plugin = _FakePlugin('tidal', downloads=[])
|
||||
engine.register_plugin('youtube', yt_plugin)
|
||||
engine.register_plugin('tidal', td_plugin)
|
||||
|
||||
result = _run_async(engine.get_download_status('shared'))
|
||||
assert result is not None
|
||||
assert result.id == 'shared'
|
||||
|
||||
|
||||
def test_engine_cancel_routes_streaming_source_directly():
|
||||
"""When source_hint is a known streaming-source name (not
|
||||
'soulseek'), engine routes the cancel to that specific plugin
|
||||
only — doesn't ask every other plugin first."""
|
||||
engine = DownloadEngine()
|
||||
yt_plugin = _FakePlugin('youtube')
|
||||
td_plugin = _FakePlugin('tidal')
|
||||
engine.register_plugin('youtube', yt_plugin)
|
||||
engine.register_plugin('tidal', td_plugin)
|
||||
|
||||
_run_async(engine.cancel_download('dl-1', 'tidal', remove=False))
|
||||
assert yt_plugin.cancel_calls == []
|
||||
assert td_plugin.cancel_calls == [('dl-1', 'tidal', False)]
|
||||
|
||||
|
||||
def test_engine_cancel_routes_unknown_source_hint_to_soulseek():
|
||||
"""A username that's NOT in the plugin registry is a real
|
||||
Soulseek peer name — route to the soulseek plugin."""
|
||||
engine = DownloadEngine()
|
||||
sl_plugin = _FakePlugin('soulseek')
|
||||
yt_plugin = _FakePlugin('youtube')
|
||||
engine.register_plugin('soulseek', sl_plugin)
|
||||
engine.register_plugin('youtube', yt_plugin)
|
||||
|
||||
_run_async(engine.cancel_download('dl-1', 'random_peer_username', remove=False))
|
||||
assert sl_plugin.cancel_calls == [('dl-1', 'random_peer_username', False)]
|
||||
assert yt_plugin.cancel_calls == []
|
||||
|
||||
|
||||
def test_engine_cancel_falls_back_to_iterating_all_plugins_without_hint():
|
||||
"""No source hint → ask every plugin until one accepts the
|
||||
cancel (returns True). Mirrors legacy orchestrator behavior."""
|
||||
engine = DownloadEngine()
|
||||
yt_plugin = _FakePlugin('youtube', cancel_result=False)
|
||||
td_plugin = _FakePlugin('tidal', cancel_result=True)
|
||||
engine.register_plugin('youtube', yt_plugin)
|
||||
engine.register_plugin('tidal', td_plugin)
|
||||
|
||||
result = _run_async(engine.cancel_download('dl-1', None, remove=False))
|
||||
assert result is True
|
||||
# Both plugins were asked; tidal accepted.
|
||||
assert len(yt_plugin.cancel_calls) == 1
|
||||
assert len(td_plugin.cancel_calls) == 1
|
||||
|
||||
|
||||
def test_engine_clear_all_skips_unconfigured_plugins():
|
||||
"""Unconfigured plugins are silently skipped (no API call, no
|
||||
error) — matches legacy orchestrator's defensive handling."""
|
||||
engine = DownloadEngine()
|
||||
configured = _FakePlugin('youtube', configured=True, clear_result=True)
|
||||
unconfigured = _FakePlugin('tidal', configured=False)
|
||||
engine.register_plugin('youtube', configured)
|
||||
engine.register_plugin('tidal', unconfigured)
|
||||
|
||||
result = _run_async(engine.clear_all_completed_downloads())
|
||||
assert result is True
|
||||
assert configured.clear_calls == 1
|
||||
assert unconfigured.clear_calls == 0
|
||||
|
||||
|
||||
def test_engine_clear_all_returns_false_when_any_configured_plugin_fails():
|
||||
engine = DownloadEngine()
|
||||
failing = _FakePlugin('youtube', configured=True, clear_result=False)
|
||||
engine.register_plugin('youtube', failing)
|
||||
|
||||
result = _run_async(engine.clear_all_completed_downloads())
|
||||
assert result is False
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Hybrid fallback (Phase F)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class _FakeSearchPlugin:
|
||||
def __init__(self, name, configured=True, search_result=None, raises=None):
|
||||
self.name = name
|
||||
self._configured = configured
|
||||
self._search_result = search_result if search_result is not None else ([], [])
|
||||
self._raises = raises
|
||||
self.search_calls = 0
|
||||
|
||||
def is_configured(self):
|
||||
return self._configured
|
||||
|
||||
async def search(self, query, timeout=None, progress_callback=None):
|
||||
self.search_calls += 1
|
||||
if self._raises:
|
||||
raise self._raises
|
||||
return self._search_result
|
||||
|
||||
|
||||
class _FakeDownloadPlugin:
|
||||
def __init__(self, name, configured=True, download_result=None, raises=None):
|
||||
self.name = name
|
||||
self._configured = configured
|
||||
self._download_result = download_result
|
||||
self._raises = raises
|
||||
self.download_calls = []
|
||||
|
||||
def is_configured(self):
|
||||
return self._configured
|
||||
|
||||
async def download(self, username, filename, file_size):
|
||||
self.download_calls.append((username, filename, file_size))
|
||||
if self._raises:
|
||||
raise self._raises
|
||||
return self._download_result
|
||||
|
||||
|
||||
def test_search_with_fallback_returns_first_non_empty_result():
|
||||
engine = DownloadEngine()
|
||||
yt = _FakeSearchPlugin('youtube', search_result=([], []))
|
||||
td = _FakeSearchPlugin('tidal', search_result=(['track1'], []))
|
||||
qz = _FakeSearchPlugin('qobuz', search_result=(['track2'], []))
|
||||
engine.register_plugin('youtube', yt)
|
||||
engine.register_plugin('tidal', td)
|
||||
engine.register_plugin('qobuz', qz)
|
||||
|
||||
tracks, _ = _run_async(engine.search_with_fallback('q', ['youtube', 'tidal', 'qobuz']))
|
||||
assert tracks == ['track1']
|
||||
# Tidal short-circuits — qobuz never queried.
|
||||
assert yt.search_calls == 1
|
||||
assert td.search_calls == 1
|
||||
assert qz.search_calls == 0
|
||||
|
||||
|
||||
def test_search_with_fallback_skips_unconfigured_plugins():
|
||||
engine = DownloadEngine()
|
||||
yt = _FakeSearchPlugin('youtube', configured=False)
|
||||
td = _FakeSearchPlugin('tidal', configured=True, search_result=(['hit'], []))
|
||||
engine.register_plugin('youtube', yt)
|
||||
engine.register_plugin('tidal', td)
|
||||
|
||||
tracks, _ = _run_async(engine.search_with_fallback('q', ['youtube', 'tidal']))
|
||||
assert tracks == ['hit']
|
||||
assert yt.search_calls == 0 # skipped
|
||||
|
||||
|
||||
def test_search_with_fallback_continues_after_per_source_exception():
|
||||
engine = DownloadEngine()
|
||||
yt = _FakeSearchPlugin('youtube', raises=RuntimeError("yt down"))
|
||||
td = _FakeSearchPlugin('tidal', search_result=(['fallback-hit'], []))
|
||||
engine.register_plugin('youtube', yt)
|
||||
engine.register_plugin('tidal', td)
|
||||
|
||||
tracks, _ = _run_async(engine.search_with_fallback('q', ['youtube', 'tidal']))
|
||||
assert tracks == ['fallback-hit']
|
||||
|
||||
|
||||
def test_search_with_fallback_returns_empty_when_chain_exhausted():
|
||||
engine = DownloadEngine()
|
||||
yt = _FakeSearchPlugin('youtube', search_result=([], []))
|
||||
td = _FakeSearchPlugin('tidal', search_result=([], []))
|
||||
engine.register_plugin('youtube', yt)
|
||||
engine.register_plugin('tidal', td)
|
||||
|
||||
tracks, _ = _run_async(engine.search_with_fallback('q', ['youtube', 'tidal']))
|
||||
assert tracks == []
|
||||
assert yt.search_calls == 1
|
||||
assert td.search_calls == 1
|
||||
|
||||
|
||||
def test_download_with_fallback_returns_first_accepted_download_id():
|
||||
"""Phase F bug fix: legacy hybrid download routed to one source
|
||||
via username hint with no retry. Engine now falls through chain."""
|
||||
engine = DownloadEngine()
|
||||
yt = _FakeDownloadPlugin('youtube', download_result=None) # refuses
|
||||
td = _FakeDownloadPlugin('tidal', download_result='td-id')
|
||||
engine.register_plugin('youtube', yt)
|
||||
engine.register_plugin('tidal', td)
|
||||
|
||||
result = _run_async(engine.download_with_fallback(
|
||||
'youtube', 'v||t', 0, ['youtube', 'tidal'],
|
||||
))
|
||||
assert result == 'td-id'
|
||||
assert len(yt.download_calls) == 1 # tried first
|
||||
assert len(td.download_calls) == 1 # took over
|
||||
|
||||
|
||||
def test_download_with_fallback_promotes_username_hint_to_head():
|
||||
"""A username hint that matches a source-chain entry tries that
|
||||
source FIRST regardless of declared chain order."""
|
||||
engine = DownloadEngine()
|
||||
yt = _FakeDownloadPlugin('youtube', download_result='yt-id')
|
||||
td = _FakeDownloadPlugin('tidal', download_result='td-id')
|
||||
engine.register_plugin('youtube', yt)
|
||||
engine.register_plugin('tidal', td)
|
||||
|
||||
# Chain says tidal-first, but username hint promotes youtube.
|
||||
result = _run_async(engine.download_with_fallback(
|
||||
'youtube', 'v||t', 0, ['tidal', 'youtube'],
|
||||
))
|
||||
assert result == 'yt-id'
|
||||
assert len(yt.download_calls) == 1
|
||||
assert len(td.download_calls) == 0 # never reached
|
||||
|
||||
|
||||
def test_download_with_fallback_returns_none_when_all_refuse():
|
||||
engine = DownloadEngine()
|
||||
yt = _FakeDownloadPlugin('youtube', download_result=None)
|
||||
td = _FakeDownloadPlugin('tidal', download_result=None)
|
||||
engine.register_plugin('youtube', yt)
|
||||
engine.register_plugin('tidal', td)
|
||||
|
||||
result = _run_async(engine.download_with_fallback(
|
||||
'youtube', 'v||t', 0, ['youtube', 'tidal'],
|
||||
))
|
||||
assert result is None
|
||||
assert len(yt.download_calls) == 1
|
||||
assert len(td.download_calls) == 1
|
||||
|
||||
|
||||
def test_download_with_fallback_continues_past_exception():
|
||||
engine = DownloadEngine()
|
||||
yt = _FakeDownloadPlugin('youtube', raises=RuntimeError("yt died"))
|
||||
td = _FakeDownloadPlugin('tidal', download_result='td-id')
|
||||
engine.register_plugin('youtube', yt)
|
||||
engine.register_plugin('tidal', td)
|
||||
|
||||
result = _run_async(engine.download_with_fallback(
|
||||
'youtube', 'v||t', 0, ['youtube', 'tidal'],
|
||||
))
|
||||
assert result == 'td-id'
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Cin bug 1: alias resolution on cancel_download
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_register_plugin_records_aliases():
|
||||
"""Aliases passed to register_plugin resolve to the canonical plugin
|
||||
via get_plugin. Cin caught engine.cancel_download routing 'deezer_dl'
|
||||
to soulseek because the alias never made it to the engine."""
|
||||
engine = DownloadEngine()
|
||||
deezer = _FakePlugin('deezer')
|
||||
engine.register_plugin('deezer', deezer, aliases=('deezer_dl',))
|
||||
|
||||
assert engine.get_plugin('deezer') is deezer
|
||||
assert engine.get_plugin('deezer_dl') is deezer
|
||||
|
||||
|
||||
def test_cancel_download_resolves_alias_to_canonical_plugin():
|
||||
"""The legacy 'deezer_dl' source_hint must route to the deezer
|
||||
plugin, not fall through to soulseek. This was Cin's bug 1 —
|
||||
cancel of a Deezer download silently no-op'd."""
|
||||
engine = DownloadEngine()
|
||||
soulseek = _FakePlugin('soulseek')
|
||||
deezer = _FakePlugin('deezer')
|
||||
engine.register_plugin('soulseek', soulseek)
|
||||
engine.register_plugin('deezer', deezer, aliases=('deezer_dl',))
|
||||
|
||||
_run_async(engine.cancel_download('dl-1', 'deezer_dl', remove=False))
|
||||
assert deezer.cancel_calls == [('dl-1', 'deezer_dl', False)]
|
||||
assert soulseek.cancel_calls == []
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Cin bug 3: atomic update_record_unless_state
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_update_record_unless_state_applies_when_state_not_blocked():
|
||||
engine = DownloadEngine()
|
||||
engine.add_record('youtube', 'dl-1', {'state': 'InProgress, Downloading'})
|
||||
|
||||
applied = engine.update_record_unless_state(
|
||||
'youtube', 'dl-1',
|
||||
{'state': 'Completed, Succeeded', 'progress': 100.0},
|
||||
skip_if_state_in=('Cancelled',),
|
||||
)
|
||||
assert applied is True
|
||||
assert engine.get_record('youtube', 'dl-1')['state'] == 'Completed, Succeeded'
|
||||
assert engine.get_record('youtube', 'dl-1')['progress'] == 100.0
|
||||
|
||||
|
||||
def test_update_record_unless_state_skips_when_state_blocked():
|
||||
"""A worker-thread terminal write must NOT clobber a Cancelled
|
||||
state set by the user. Returns False so caller knows the patch
|
||||
was skipped."""
|
||||
engine = DownloadEngine()
|
||||
engine.add_record('youtube', 'dl-1', {'state': 'Cancelled'})
|
||||
|
||||
applied = engine.update_record_unless_state(
|
||||
'youtube', 'dl-1',
|
||||
{'state': 'Completed, Succeeded'},
|
||||
skip_if_state_in=('Cancelled',),
|
||||
)
|
||||
assert applied is False
|
||||
assert engine.get_record('youtube', 'dl-1')['state'] == 'Cancelled'
|
||||
|
||||
|
||||
def test_update_record_unless_state_returns_false_for_missing_record():
|
||||
engine = DownloadEngine()
|
||||
applied = engine.update_record_unless_state(
|
||||
'youtube', 'never-existed',
|
||||
{'state': 'Completed, Succeeded'},
|
||||
skip_if_state_in=('Cancelled',),
|
||||
)
|
||||
assert applied is False
|
||||
|
|
@ -1,4 +1,6 @@
|
|||
from core.download_engine import DownloadEngine
|
||||
from core.download_orchestrator import DownloadOrchestrator
|
||||
from core.download_plugins.registry import DownloadPluginRegistry, PluginSpec
|
||||
|
||||
|
||||
class _FakeClient:
|
||||
|
|
@ -16,15 +18,45 @@ class _FakeClient:
|
|||
|
||||
|
||||
def _build_orchestrator(**clients):
|
||||
"""Build an orchestrator with mock clients via the registry.
|
||||
|
||||
The orchestrator iterates `self.registry.all_plugins()` to drive
|
||||
every per-source operation, so the test must set up a real
|
||||
registry with mock plugins (not just stuff attributes on the
|
||||
orchestrator). Source slots not provided in `clients` are
|
||||
skipped — registry only holds the ones the test cares about.
|
||||
"""
|
||||
registry = DownloadPluginRegistry()
|
||||
name_to_display = {
|
||||
'soulseek': 'Soulseek', 'youtube': 'YouTube', 'tidal': 'Tidal',
|
||||
'qobuz': 'Qobuz', 'hifi': 'HiFi', 'deezer_dl': 'Deezer',
|
||||
'lidarr': 'Lidarr', 'soundcloud': 'SoundCloud',
|
||||
}
|
||||
# 'deezer_dl' is the legacy attr name; canonical registry name is 'deezer'.
|
||||
aliases_for = {'deezer_dl': ('deezer_dl',)}
|
||||
canonical_for = {'deezer_dl': 'deezer'}
|
||||
|
||||
for slot, client in clients.items():
|
||||
if client is None:
|
||||
continue
|
||||
canonical_name = canonical_for.get(slot, slot)
|
||||
registry.register(PluginSpec(
|
||||
name=canonical_name,
|
||||
factory=lambda c=client: c,
|
||||
display_name=name_to_display.get(slot, slot),
|
||||
aliases=aliases_for.get(slot, ()),
|
||||
))
|
||||
registry.initialize()
|
||||
|
||||
orch = DownloadOrchestrator.__new__(DownloadOrchestrator)
|
||||
orch.soulseek = clients.get("soulseek")
|
||||
orch.youtube = clients.get("youtube")
|
||||
orch.tidal = clients.get("tidal")
|
||||
orch.qobuz = clients.get("qobuz")
|
||||
orch.hifi = clients.get("hifi")
|
||||
orch.deezer_dl = clients.get("deezer_dl")
|
||||
orch.lidarr = clients.get("lidarr")
|
||||
orch.soundcloud = clients.get("soundcloud")
|
||||
orch.registry = registry
|
||||
orch._init_failures = registry.init_failures
|
||||
# Engine — orchestrator delegates per-source query/cancel
|
||||
# methods to it, so the test fixture must build one and
|
||||
# register every mock plugin under its canonical name.
|
||||
orch.engine = DownloadEngine()
|
||||
for source_name, plugin in registry.all_plugins():
|
||||
orch.engine.register_plugin(source_name, plugin)
|
||||
return orch
|
||||
|
||||
|
||||
|
|
@ -47,8 +79,8 @@ def test_clear_all_completed_downloads_ignores_unconfigured_clients():
|
|||
result = _run_async(orch.clear_all_completed_downloads())
|
||||
|
||||
assert result is True
|
||||
assert orch.soulseek.clear_calls == 1
|
||||
assert orch.youtube.clear_calls == 0
|
||||
assert orch.client('soulseek').clear_calls == 1
|
||||
assert orch.client('youtube').clear_calls == 0
|
||||
|
||||
|
||||
def test_clear_all_completed_downloads_propagates_configured_failures():
|
||||
|
|
@ -59,4 +91,177 @@ def test_clear_all_completed_downloads_propagates_configured_failures():
|
|||
result = _run_async(orch.clear_all_completed_downloads())
|
||||
|
||||
assert result is False
|
||||
assert orch.soulseek.clear_calls == 1
|
||||
assert orch.client('soulseek').clear_calls == 1
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Cin-2 generic accessors
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_client_returns_registered_client_by_name():
|
||||
"""Cin's review feedback: orch.client('hifi') is the canonical
|
||||
way to reach a per-source client, replacing orch.hifi attribute
|
||||
access."""
|
||||
soulseek = _FakeClient()
|
||||
youtube = _FakeClient()
|
||||
orch = _build_orchestrator(soulseek=soulseek, youtube=youtube)
|
||||
|
||||
assert orch.client('soulseek') is soulseek
|
||||
assert orch.client('youtube') is youtube
|
||||
assert orch.client('made_up') is None
|
||||
|
||||
|
||||
def test_configured_clients_excludes_unconfigured_sources():
|
||||
"""Replaces the legacy iteration pattern: 6+ if/hasattr/is_configured
|
||||
checks per source. Single call returns dict of configured clients."""
|
||||
configured = _FakeClient(configured=True)
|
||||
unconfigured = _FakeClient(configured=False)
|
||||
orch = _build_orchestrator(
|
||||
soulseek=configured,
|
||||
youtube=unconfigured,
|
||||
)
|
||||
result = orch.configured_clients()
|
||||
assert 'soulseek' in result
|
||||
assert 'youtube' not in result
|
||||
assert result['soulseek'] is configured
|
||||
|
||||
|
||||
def test_configured_clients_skips_clients_whose_is_configured_raises():
|
||||
"""Per JohnBaumb: configured_clients() has a try/except so a single
|
||||
broken is_configured() call doesn't crash the whole iteration —
|
||||
pin it so a future refactor can't quietly drop the guard. The
|
||||
broken plugin is skipped; the rest still come back."""
|
||||
|
||||
class _BrokenIsConfigured(_FakeClient):
|
||||
def is_configured(self):
|
||||
raise RuntimeError("is_configured blew up")
|
||||
|
||||
broken = _BrokenIsConfigured()
|
||||
healthy = _FakeClient(configured=True)
|
||||
orch = _build_orchestrator(soulseek=healthy, youtube=broken)
|
||||
|
||||
result = orch.configured_clients()
|
||||
# Healthy plugin still surfaces; broken one is silently skipped.
|
||||
assert 'soulseek' in result
|
||||
assert result['soulseek'] is healthy
|
||||
assert 'youtube' not in result
|
||||
|
||||
|
||||
def test_reload_instances_dispatches_to_named_source():
|
||||
"""Generic dispatch — caller passes source name instead of
|
||||
reaching for orch.hifi.reload_instances() directly."""
|
||||
|
||||
class _ReloadableClient(_FakeClient):
|
||||
def __init__(self):
|
||||
super().__init__(configured=True)
|
||||
self.reload_called = False
|
||||
|
||||
def reload_instances(self):
|
||||
self.reload_called = True
|
||||
|
||||
hifi = _ReloadableClient()
|
||||
soulseek = _FakeClient() # No reload_instances method
|
||||
orch = _build_orchestrator(soulseek=soulseek, hifi=hifi)
|
||||
|
||||
assert orch.reload_instances('hifi') is True
|
||||
assert hifi.reload_called is True
|
||||
|
||||
|
||||
def test_reload_instances_skips_clients_without_method():
|
||||
"""Sources that don't expose reload_instances are skipped, not
|
||||
treated as failures."""
|
||||
soulseek = _FakeClient() # No reload_instances method
|
||||
orch = _build_orchestrator(soulseek=soulseek)
|
||||
# Calling on a source without the method = silent no-op
|
||||
assert orch.reload_instances('soulseek') is True
|
||||
|
||||
|
||||
def test_reload_instances_with_no_args_reloads_every_source():
|
||||
"""When called with no source argument, hits every registered
|
||||
source that exposes reload_instances."""
|
||||
|
||||
class _ReloadableClient(_FakeClient):
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self.reload_called = False
|
||||
|
||||
def reload_instances(self):
|
||||
self.reload_called = True
|
||||
|
||||
a = _ReloadableClient()
|
||||
b = _ReloadableClient()
|
||||
orch = _build_orchestrator(soulseek=a, hifi=b)
|
||||
|
||||
orch.reload_instances()
|
||||
assert a.reload_called is True
|
||||
assert b.reload_called is True
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Singleton factory (matches Cin's get_metadata_engine pattern)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Cin bug 2: hybrid_order alias normalization
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_resolve_source_chain_normalizes_legacy_aliases():
|
||||
"""Cin's bug 2: hybrid_order config containing the legacy alias
|
||||
'deezer_dl' was silently dropped because the canonical-name
|
||||
membership check rejected it. Orchestrator must normalize via
|
||||
the registry alias map first."""
|
||||
orch = _build_orchestrator(
|
||||
soulseek=_FakeClient(),
|
||||
deezer_dl=_FakeClient(),
|
||||
youtube=_FakeClient(),
|
||||
)
|
||||
orch.hybrid_order = ['deezer_dl', 'soulseek', 'youtube']
|
||||
orch.hybrid_primary = None
|
||||
orch.hybrid_secondary = None
|
||||
|
||||
chain = orch._resolve_source_chain()
|
||||
assert chain == ['deezer', 'soulseek', 'youtube']
|
||||
|
||||
|
||||
def test_resolve_source_chain_dedupes_alias_and_canonical():
|
||||
"""If both 'deezer' and 'deezer_dl' appear, dedupe to single entry."""
|
||||
orch = _build_orchestrator(
|
||||
soulseek=_FakeClient(),
|
||||
deezer_dl=_FakeClient(),
|
||||
)
|
||||
orch.hybrid_order = ['deezer_dl', 'deezer', 'soulseek']
|
||||
orch.hybrid_primary = None
|
||||
orch.hybrid_secondary = None
|
||||
|
||||
chain = orch._resolve_source_chain()
|
||||
assert chain == ['deezer', 'soulseek']
|
||||
|
||||
|
||||
def test_resolve_source_chain_drops_unknown_names():
|
||||
orch = _build_orchestrator(soulseek=_FakeClient(), youtube=_FakeClient())
|
||||
orch.hybrid_order = ['nonsense', 'soulseek', 'also_fake', 'youtube']
|
||||
orch.hybrid_primary = None
|
||||
orch.hybrid_secondary = None
|
||||
|
||||
chain = orch._resolve_source_chain()
|
||||
assert chain == ['soulseek', 'youtube']
|
||||
|
||||
|
||||
def test_get_download_orchestrator_returns_set_singleton():
|
||||
"""When set_download_orchestrator has been called (web_server.py
|
||||
does this at boot), get_download_orchestrator returns the
|
||||
installed instance instead of building a fresh one."""
|
||||
from core.download_orchestrator import (
|
||||
get_download_orchestrator,
|
||||
set_download_orchestrator,
|
||||
)
|
||||
|
||||
orch = _build_orchestrator(soulseek=_FakeClient())
|
||||
set_download_orchestrator(orch)
|
||||
try:
|
||||
assert get_download_orchestrator() is orch
|
||||
finally:
|
||||
set_download_orchestrator(None)
|
||||
|
|
|
|||
|
|
@ -101,7 +101,7 @@ def _build_deps(
|
|||
on_complete=None,
|
||||
):
|
||||
deps = dc.CandidatesDeps(
|
||||
soulseek_client=soulseek or _FakeSoulseek(),
|
||||
download_orchestrator=soulseek or _FakeSoulseek(),
|
||||
spotify_client=spotify or _FakeSpotify(),
|
||||
run_async=_run_async,
|
||||
get_database=lambda: db or _FakeDB(),
|
||||
|
|
@ -136,7 +136,7 @@ def test_first_candidate_starts_download_and_returns_true():
|
|||
result = dc.attempt_download_with_candidates("t1", candidates, track, batch_id="b1", deps=deps)
|
||||
|
||||
assert result is True
|
||||
assert deps.soulseek_client.download_calls == [("user1", "best.flac", 1000)]
|
||||
assert deps.download_orchestrator.download_calls == [("user1", "best.flac", 1000)]
|
||||
assert download_tasks["t1"]["download_id"] == "dl-1"
|
||||
assert "user1::best.flac" in matched_downloads_context
|
||||
|
||||
|
|
@ -155,7 +155,7 @@ def test_candidates_tried_in_confidence_order():
|
|||
dc.attempt_download_with_candidates("t2", candidates, track, batch_id=None, deps=deps)
|
||||
|
||||
# First call should be the highest-confidence one
|
||||
assert deps.soulseek_client.download_calls[0][1] == "high.flac"
|
||||
assert deps.download_orchestrator.download_calls[0][1] == "high.flac"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
|
|
@ -175,8 +175,8 @@ def test_already_tried_source_skipped():
|
|||
dc.attempt_download_with_candidates("t3", candidates, track, batch_id=None, deps=deps)
|
||||
|
||||
# First candidate skipped (already used), second one tried
|
||||
assert len(deps.soulseek_client.download_calls) == 1
|
||||
assert deps.soulseek_client.download_calls[0][1] == "fresh.flac"
|
||||
assert len(deps.download_orchestrator.download_calls) == 1
|
||||
assert deps.download_orchestrator.download_calls[0][1] == "fresh.flac"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
|
|
@ -196,7 +196,7 @@ def test_blacklisted_source_skipped():
|
|||
|
||||
dc.attempt_download_with_candidates("t4", candidates, track, batch_id=None, deps=deps)
|
||||
|
||||
assert deps.soulseek_client.download_calls[0][1] == "ok.flac"
|
||||
assert deps.download_orchestrator.download_calls[0][1] == "ok.flac"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
|
|
@ -213,7 +213,7 @@ def test_cancellation_before_attempt_returns_false():
|
|||
result = dc.attempt_download_with_candidates("t5", candidates, track, batch_id=None, deps=deps)
|
||||
|
||||
assert result is False
|
||||
assert deps.soulseek_client.download_calls == []
|
||||
assert deps.download_orchestrator.download_calls == []
|
||||
|
||||
|
||||
def test_task_deleted_returns_false():
|
||||
|
|
@ -238,7 +238,7 @@ def test_active_download_id_skips_new_download():
|
|||
dc.attempt_download_with_candidates("t6", candidates, track, batch_id=None, deps=deps)
|
||||
|
||||
# Both candidates skipped (download_id already present)
|
||||
assert deps.soulseek_client.download_calls == []
|
||||
assert deps.download_orchestrator.download_calls == []
|
||||
|
||||
|
||||
def test_cancellation_after_download_starts_calls_cancel_and_lifecycle():
|
||||
|
|
@ -266,7 +266,7 @@ def test_cancellation_after_download_starts_calls_cancel_and_lifecycle():
|
|||
|
||||
assert result is False
|
||||
# cancel_download was called for the in-flight transfer
|
||||
assert deps.soulseek_client.cancel_calls
|
||||
assert deps.download_orchestrator.cancel_calls
|
||||
# on_download_completed fired with success=False to free the worker slot
|
||||
assert completion_calls == [("b7", "t7", False)]
|
||||
|
||||
|
|
@ -276,7 +276,7 @@ def test_cancellation_after_download_starts_calls_cancel_and_lifecycle():
|
|||
# ---------------------------------------------------------------------------
|
||||
|
||||
def test_all_candidates_failed_returns_false():
|
||||
"""If soulseek_client.download returns None (failure) for all candidates, returns False."""
|
||||
"""If download_orchestrator.download returns None (failure) for all candidates, returns False."""
|
||||
soulseek = _FakeSoulseek(download_id=None)
|
||||
deps = _build_deps(soulseek=soulseek)
|
||||
_seed_task("t8")
|
||||
|
|
@ -411,5 +411,5 @@ def test_candidates_with_equal_confidence_both_tried():
|
|||
dc.attempt_download_with_candidates("t13", candidates, track, batch_id=None, deps=deps)
|
||||
|
||||
# First one wins — second never tried because download succeeded
|
||||
assert len(deps.soulseek_client.download_calls) == 1
|
||||
assert deps.soulseek_client.download_calls[0][1] == "a.flac"
|
||||
assert len(deps.download_orchestrator.download_calls) == 1
|
||||
assert deps.download_orchestrator.download_calls[0][1] == "a.flac"
|
||||
|
|
|
|||
|
|
@ -178,7 +178,7 @@ def _build_deps(
|
|||
):
|
||||
return mw.MasterDeps(
|
||||
config_manager=config or _FakeConfig(),
|
||||
soulseek_client=soulseek or _FakeSoulseekWrapper(_FakeSoulseek()),
|
||||
download_orchestrator=soulseek or _FakeSoulseekWrapper(_FakeSoulseek()),
|
||||
run_async=run_async or _make_run_async(),
|
||||
mb_worker=mb_worker,
|
||||
mb_release_cache=mb_release_cache if mb_release_cache is not None else {},
|
||||
|
|
|
|||
|
|
@ -49,7 +49,7 @@ class _Recorder:
|
|||
def _build_deps(
|
||||
*,
|
||||
config=None,
|
||||
soulseek_client=None,
|
||||
download_orchestrator=None,
|
||||
run_async=None,
|
||||
docker_resolve_path=None,
|
||||
extract_filename=None,
|
||||
|
|
@ -64,7 +64,7 @@ def _build_deps(
|
|||
rec = _Recorder()
|
||||
return pp.PostProcessDeps(
|
||||
config_manager=config or _FakeConfig(),
|
||||
soulseek_client=soulseek_client,
|
||||
download_orchestrator=download_orchestrator,
|
||||
run_async=run_async or (lambda c: None),
|
||||
docker_resolve_path=docker_resolve_path or (lambda p: p),
|
||||
extract_filename=extract_filename or (lambda f: os.path.basename(f) if f else ''),
|
||||
|
|
@ -315,7 +315,7 @@ def test_youtube_task_uses_get_download_status_to_resolve_path(monkeypatch):
|
|||
monkeypatch.setattr(pp.os.path, 'exists', lambda p: p == '/downloads/Money.mp3')
|
||||
|
||||
deps, rec = _build_deps(
|
||||
soulseek_client=_FakeYTClient(),
|
||||
download_orchestrator=_FakeYTClient(),
|
||||
run_async=lambda coro: coro, # not async — direct call
|
||||
)
|
||||
pp.run_post_processing_worker('t1', 'b1', deps)
|
||||
|
|
|
|||
|
|
@ -31,13 +31,28 @@ class _Recorder:
|
|||
|
||||
|
||||
class _FakeClient:
|
||||
"""Stub soulseek client. `mode` defaults to non-hybrid."""
|
||||
"""Stub download orchestrator. `mode` defaults to non-hybrid.
|
||||
Per-source stubs passed via ``subclients`` are surfaced through
|
||||
``client(name)`` (the registry-backed accessor on the real
|
||||
orchestrator); plain attrs (hybrid_order, hybrid_primary, etc.)
|
||||
are set as attributes so getattr() lookups still resolve them."""
|
||||
_CLIENT_NAMES = {'soulseek', 'youtube', 'tidal', 'qobuz', 'hifi',
|
||||
'deezer_dl', 'lidarr', 'soundcloud'}
|
||||
|
||||
def __init__(self, results=None, mode='soulseek', subclients=None):
|
||||
self._results = results if results is not None else []
|
||||
self.mode = mode
|
||||
self.search_calls = []
|
||||
self._client_map = {}
|
||||
for k, v in (subclients or {}).items():
|
||||
setattr(self, k, v)
|
||||
if k in self._CLIENT_NAMES:
|
||||
self._client_map[k] = v
|
||||
else:
|
||||
# config-style attrs (hybrid_order, hybrid_primary, etc.)
|
||||
setattr(self, k, v)
|
||||
|
||||
def client(self, name):
|
||||
return self._client_map.get(name)
|
||||
|
||||
async def search(self, query, timeout=30):
|
||||
self.search_calls.append((query, timeout))
|
||||
|
|
@ -76,7 +91,7 @@ def _build_deps(
|
|||
):
|
||||
rec = _Recorder()
|
||||
return tw.TaskWorkerDeps(
|
||||
soulseek_client=soulseek or _FakeClient(),
|
||||
download_orchestrator=soulseek or _FakeClient(),
|
||||
matching_engine=matching or _FakeMatchEngine(),
|
||||
run_async=_sync_run_async,
|
||||
try_source_reuse=try_source_reuse,
|
||||
|
|
|
|||
92
tests/downloads/test_downloads_validation.py
Normal file
92
tests/downloads/test_downloads_validation.py
Normal file
|
|
@ -0,0 +1,92 @@
|
|||
"""Tests for core/downloads/validation.py — SoundCloud preview filter.
|
||||
|
||||
The SoundCloud anonymous tier serves a ~30s preview clip for tracks
|
||||
gated behind Go+ / login. ``filter_soundcloud_previews`` drops these
|
||||
candidates before they reach the matcher, the modal cache, or the
|
||||
manual-pick download path.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
from typing import Optional
|
||||
|
||||
from core.downloads.validation import filter_soundcloud_previews
|
||||
|
||||
|
||||
@dataclass
|
||||
class _Track:
|
||||
duration_ms: int
|
||||
|
||||
|
||||
@dataclass
|
||||
class _Candidate:
|
||||
username: str
|
||||
duration: Optional[int] # milliseconds
|
||||
title: str = ''
|
||||
|
||||
|
||||
def test_drops_soundcloud_30s_preview_when_expected_long():
|
||||
"""A 30s SC candidate against a 5-minute expected track is the
|
||||
canonical preview-snippet case — must be dropped."""
|
||||
expected = _Track(duration_ms=338_000) # ~5:38
|
||||
cands = [
|
||||
_Candidate(username='soundcloud', duration=30_000, title='Preview'),
|
||||
_Candidate(username='soundcloud', duration=338_000, title='Real'),
|
||||
]
|
||||
result = filter_soundcloud_previews(cands, expected)
|
||||
assert len(result) == 1
|
||||
assert result[0].title == 'Real'
|
||||
|
||||
|
||||
def test_drops_under_half_expected_duration():
|
||||
"""SC candidate at 100s against 300s expected = clearly truncated /
|
||||
wrong content. Must be dropped even if not at the 30s boundary."""
|
||||
expected = _Track(duration_ms=300_000)
|
||||
cand = _Candidate(username='soundcloud', duration=100_000)
|
||||
assert filter_soundcloud_previews([cand], expected) == []
|
||||
|
||||
|
||||
def test_keeps_soundcloud_when_expected_track_is_short():
|
||||
"""Genuinely short SC tracks (intros, sound effects, sub-minute
|
||||
songs) must pass through when the expected track is also short.
|
||||
Filter only kicks in when expected > 60s."""
|
||||
expected = _Track(duration_ms=45_000) # 45s expected
|
||||
cand = _Candidate(username='soundcloud', duration=30_000)
|
||||
result = filter_soundcloud_previews([cand], expected)
|
||||
assert result == [cand]
|
||||
|
||||
|
||||
def test_does_not_filter_non_soundcloud_sources():
|
||||
"""A 30s candidate from another streaming source isn't a SoundCloud
|
||||
preview — leave it for the generic matching engine to score."""
|
||||
expected = _Track(duration_ms=338_000)
|
||||
yt = _Candidate(username='youtube', duration=30_000)
|
||||
tidal = _Candidate(username='tidal', duration=30_000)
|
||||
assert filter_soundcloud_previews([yt, tidal], expected) == [yt, tidal]
|
||||
|
||||
|
||||
def test_returns_input_unchanged_without_expected_duration():
|
||||
"""Without a Spotify-track / expected duration we can't reason
|
||||
about previews — pass everything through."""
|
||||
cands = [
|
||||
_Candidate(username='soundcloud', duration=30_000),
|
||||
_Candidate(username='soundcloud', duration=300_000),
|
||||
]
|
||||
assert filter_soundcloud_previews(cands, None) == cands
|
||||
assert filter_soundcloud_previews(cands, _Track(duration_ms=0)) == cands
|
||||
|
||||
|
||||
def test_empty_input_returns_empty_list():
|
||||
assert filter_soundcloud_previews([], _Track(duration_ms=200_000)) == []
|
||||
|
||||
|
||||
def test_keeps_soundcloud_candidate_at_threshold():
|
||||
"""Boundary check: 35s candidate against 200s expected — exactly
|
||||
at the 35s preview boundary, but 35s is also above
|
||||
expected*0.5 (100s) check (35 < 100, so still drops). Use a
|
||||
higher value to confirm the just-above threshold passes."""
|
||||
expected = _Track(duration_ms=200_000) # 200s
|
||||
# 110s passes both checks: > 35s AND > 100s (half of 200s)
|
||||
cand = _Candidate(username='soundcloud', duration=110_000)
|
||||
assert filter_soundcloud_previews([cand], expected) == [cand]
|
||||
104
tests/downloads/test_hifi_pinning.py
Normal file
104
tests/downloads/test_hifi_pinning.py
Normal file
|
|
@ -0,0 +1,104 @@
|
|||
"""Phase A pinning tests for HiFiClient — UPDATED for Phase C5."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import threading
|
||||
from pathlib import Path
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
|
||||
from core.download_engine import DownloadEngine
|
||||
from core.hifi_client import HiFiClient
|
||||
|
||||
|
||||
def _run_async(coro):
|
||||
loop = asyncio.new_event_loop()
|
||||
try:
|
||||
return loop.run_until_complete(coro)
|
||||
finally:
|
||||
loop.close()
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def hifi_client_with_engine():
|
||||
client = HiFiClient.__new__(HiFiClient)
|
||||
client.download_path = Path('./test_hifi_downloads')
|
||||
client.shutdown_check = None
|
||||
client._engine = None
|
||||
engine = DownloadEngine()
|
||||
client.set_engine(engine)
|
||||
return client, engine
|
||||
|
||||
|
||||
def test_download_returns_none_for_invalid_filename_format(hifi_client_with_engine):
|
||||
client, _ = hifi_client_with_engine
|
||||
result = _run_async(client.download('hifi', 'no-separator', 0))
|
||||
assert result is None
|
||||
|
||||
|
||||
def test_download_returns_none_for_non_integer_track_id(hifi_client_with_engine):
|
||||
client, _ = hifi_client_with_engine
|
||||
result = _run_async(client.download('hifi', 'not-int||title', 0))
|
||||
assert result is None
|
||||
|
||||
|
||||
def test_download_raises_when_engine_not_wired():
|
||||
"""Defensive: client without engine reference must raise so the
|
||||
orchestrator's download_with_fallback surfaces the error and
|
||||
moves on to the next source. Returning None silently would drop
|
||||
the download with no user feedback (per JohnBaumb)."""
|
||||
import pytest
|
||||
client = HiFiClient.__new__(HiFiClient)
|
||||
client._engine = None
|
||||
with pytest.raises(RuntimeError, match="engine reference"):
|
||||
_run_async(client.download('hifi', 'v||t', 0))
|
||||
|
||||
|
||||
def test_download_returns_uuid_for_valid_filename(hifi_client_with_engine):
|
||||
client, _ = hifi_client_with_engine
|
||||
with patch.object(client, '_download_sync', return_value='/tmp/x.flac'):
|
||||
result = _run_async(client.download('hifi', '12345||Some Song', 0))
|
||||
assert result is not None
|
||||
assert len(result) == 36
|
||||
|
||||
|
||||
def test_download_populates_engine_record_with_initial_state(hifi_client_with_engine):
|
||||
client, engine = hifi_client_with_engine
|
||||
started = threading.Event()
|
||||
release = threading.Event()
|
||||
|
||||
def slow_impl(*args, **kwargs):
|
||||
started.set()
|
||||
release.wait(timeout=1.0)
|
||||
return '/tmp/done.flac'
|
||||
|
||||
with patch.object(client, '_download_sync', side_effect=slow_impl):
|
||||
download_id = _run_async(client.download('hifi', '999||My HiFi Song', 0))
|
||||
started.wait(timeout=1.0)
|
||||
record = engine.get_record('hifi', download_id)
|
||||
assert record['filename'] == '999||My HiFi Song'
|
||||
assert record['username'] == 'hifi'
|
||||
assert record['track_id'] == 999
|
||||
assert record['display_name'] == 'My HiFi Song'
|
||||
release.set()
|
||||
|
||||
|
||||
def test_get_all_downloads_reads_engine_records(hifi_client_with_engine):
|
||||
client, engine = hifi_client_with_engine
|
||||
engine.add_record('hifi', 'dl-1', {
|
||||
'id': 'dl-1', 'filename': '111||A', 'username': 'hifi',
|
||||
'state': 'InProgress, Downloading', 'progress': 50.0,
|
||||
})
|
||||
result = _run_async(client.get_all_downloads())
|
||||
assert len(result) == 1
|
||||
assert result[0].id == 'dl-1'
|
||||
|
||||
|
||||
def test_cancel_download_marks_cancelled(hifi_client_with_engine):
|
||||
client, engine = hifi_client_with_engine
|
||||
engine.add_record('hifi', 'dl-1', {'id': 'dl-1', 'state': 'InProgress, Downloading'})
|
||||
ok = _run_async(client.cancel_download('dl-1', None, remove=False))
|
||||
assert ok is True
|
||||
assert engine.get_record('hifi', 'dl-1')['state'] == 'Cancelled'
|
||||
138
tests/downloads/test_lidarr_pinning.py
Normal file
138
tests/downloads/test_lidarr_pinning.py
Normal file
|
|
@ -0,0 +1,138 @@
|
|||
"""Phase A pinning tests for LidarrDownloadClient's download lifecycle.
|
||||
|
||||
Lidarr is the special case in the dispatcher — it's an
|
||||
ALBUM-grabber, not a track-grabber. When the user asks for a
|
||||
track, Lidarr grabs the whole album, then we pick the wanted
|
||||
track out (logic at the end of `_download_thread_worker`).
|
||||
|
||||
Engine refactor's plugin contract must accommodate album-only
|
||||
sources OR Lidarr stays special. Pinning the current contract
|
||||
forces the design decision to be conscious during Phase G.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import threading
|
||||
from pathlib import Path
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
|
||||
from core.lidarr_download_client import LidarrDownloadClient
|
||||
|
||||
|
||||
def _run_async(coro):
|
||||
loop = asyncio.new_event_loop()
|
||||
try:
|
||||
return loop.run_until_complete(coro)
|
||||
finally:
|
||||
loop.close()
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def lidarr_client():
|
||||
client = LidarrDownloadClient.__new__(LidarrDownloadClient)
|
||||
client.download_path = Path('./test_lidarr_downloads')
|
||||
client.shutdown_check = None
|
||||
client.active_downloads = {}
|
||||
client._download_lock = threading.Lock()
|
||||
client._url = 'http://lidarr.test'
|
||||
client._api_key = 'test-key'
|
||||
return client
|
||||
|
||||
|
||||
def test_download_returns_none_when_not_configured():
|
||||
"""Pinning: no Lidarr URL/key → None. Orchestrator hybrid skip
|
||||
behavior depends on this."""
|
||||
client = LidarrDownloadClient.__new__(LidarrDownloadClient)
|
||||
client._url = ''
|
||||
client._api_key = ''
|
||||
result = _run_async(client.download('lidarr', '12345||Album Name', 0))
|
||||
assert result is None
|
||||
|
||||
|
||||
def test_download_returns_uuid_for_valid_filename(lidarr_client):
|
||||
"""Pinning: valid filename → UUID download_id."""
|
||||
with patch('core.lidarr_download_client.threading.Thread') as fake:
|
||||
fake.return_value.start = lambda: None
|
||||
result = _run_async(lidarr_client.download(
|
||||
'lidarr', '12345||Some Album', 0,
|
||||
))
|
||||
assert result is not None
|
||||
assert len(result) == 36
|
||||
|
||||
|
||||
def test_download_parses_album_foreign_id_from_filename(lidarr_client):
|
||||
"""Pinning: filename format is ``album_foreign_id||display`` where
|
||||
`album_foreign_id` is the MusicBrainz album MBID Lidarr lookups
|
||||
use. Engine refactor's plugin contract must respect that this
|
||||
is an ALBUM identifier, not a track."""
|
||||
with patch('core.lidarr_download_client.threading.Thread') as fake:
|
||||
fake.return_value.start = lambda: None
|
||||
download_id = _run_async(lidarr_client.download(
|
||||
'lidarr', 'mbid-album-123||Some Album by Artist', 0,
|
||||
))
|
||||
|
||||
record = lidarr_client.active_downloads[download_id]
|
||||
assert record['album_foreign_id'] == 'mbid-album-123'
|
||||
assert record['display_name'] == 'Some Album by Artist'
|
||||
|
||||
|
||||
def test_download_handles_filename_without_separator(lidarr_client):
|
||||
"""Pinning: defensive — filename without `||` still produces a
|
||||
download record (album_foreign_id stays empty, display_name is
|
||||
the whole filename). Lidarr's worker tries lookup-by-display."""
|
||||
with patch('core.lidarr_download_client.threading.Thread') as fake:
|
||||
fake.return_value.start = lambda: None
|
||||
download_id = _run_async(lidarr_client.download(
|
||||
'lidarr', 'just-some-display-name', 0,
|
||||
))
|
||||
|
||||
assert download_id is not None
|
||||
record = lidarr_client.active_downloads[download_id]
|
||||
assert record['album_foreign_id'] == ''
|
||||
assert record['display_name'] == 'just-some-display-name'
|
||||
|
||||
|
||||
def test_download_populates_active_downloads_with_album_oriented_state(lidarr_client):
|
||||
"""Pinning: Lidarr's state-dict is SMALLER than streaming sources
|
||||
(no track_id, no transferred/speed/time_remaining — Lidarr
|
||||
polls Lidarr's queue API for those, doesn't track byte-level
|
||||
progress locally). Engine extraction must accommodate the
|
||||
smaller schema."""
|
||||
with patch('core.lidarr_download_client.threading.Thread') as fake:
|
||||
fake.return_value.start = lambda: None
|
||||
download_id = _run_async(lidarr_client.download(
|
||||
'lidarr', 'mbid-1||Album', 0,
|
||||
))
|
||||
|
||||
record = lidarr_client.active_downloads[download_id]
|
||||
assert record['id'] == download_id
|
||||
assert record['username'] == 'lidarr'
|
||||
assert record['state'] == 'Initializing'
|
||||
assert record['progress'] == 0.0
|
||||
assert record['album_foreign_id'] == 'mbid-1'
|
||||
assert record['file_path'] is None
|
||||
|
||||
|
||||
def test_download_spawns_daemon_thread_targeting_worker(lidarr_client):
|
||||
"""Pinning: thread target is `_download_thread_worker(download_id,
|
||||
album_foreign_id, display_name)` — 3 args, not 4 like streaming
|
||||
sources. Lidarr doesn't need original_filename because the album
|
||||
foreign id IS the unique key."""
|
||||
captured_kwargs = {}
|
||||
|
||||
def capture_thread(*args, **kwargs):
|
||||
captured_kwargs.update(kwargs)
|
||||
return type('FakeThread', (), {'start': lambda self: None})()
|
||||
|
||||
with patch('core.lidarr_download_client.threading.Thread', side_effect=capture_thread):
|
||||
_run_async(lidarr_client.download('lidarr', 'mbid-x||Album', 0))
|
||||
|
||||
assert captured_kwargs.get('daemon') is True
|
||||
assert captured_kwargs.get('target') == lidarr_client._download_thread_worker
|
||||
args = captured_kwargs.get('args', ())
|
||||
assert len(args) == 3 # 3-arg signature
|
||||
assert args[1] == 'mbid-x'
|
||||
assert args[2] == 'Album'
|
||||
121
tests/downloads/test_qobuz_pinning.py
Normal file
121
tests/downloads/test_qobuz_pinning.py
Normal file
|
|
@ -0,0 +1,121 @@
|
|||
"""Phase A pinning tests for QobuzClient — UPDATED for Phase C4.
|
||||
|
||||
Post-C4 the client uses engine.worker for thread + state management.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import threading
|
||||
from pathlib import Path
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
|
||||
from core.download_engine import DownloadEngine
|
||||
from core.qobuz_client import QobuzClient
|
||||
|
||||
|
||||
def _run_async(coro):
|
||||
loop = asyncio.new_event_loop()
|
||||
try:
|
||||
return loop.run_until_complete(coro)
|
||||
finally:
|
||||
loop.close()
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def qobuz_client_with_engine():
|
||||
client = QobuzClient.__new__(QobuzClient)
|
||||
client.download_path = Path('./test_qobuz_downloads')
|
||||
client.shutdown_check = None
|
||||
client._engine = None
|
||||
engine = DownloadEngine()
|
||||
client.set_engine(engine)
|
||||
return client, engine
|
||||
|
||||
|
||||
def test_download_returns_none_for_invalid_filename_format(qobuz_client_with_engine):
|
||||
client, _ = qobuz_client_with_engine
|
||||
result = _run_async(client.download('qobuz', 'no-separator', 0))
|
||||
assert result is None
|
||||
|
||||
|
||||
def test_download_returns_none_for_non_integer_track_id(qobuz_client_with_engine):
|
||||
client, _ = qobuz_client_with_engine
|
||||
result = _run_async(client.download('qobuz', 'not-int||title', 0))
|
||||
assert result is None
|
||||
|
||||
|
||||
def test_download_raises_when_engine_not_wired():
|
||||
"""Defensive: client without engine reference must raise so the
|
||||
orchestrator's download_with_fallback surfaces the error and
|
||||
moves on to the next source. Returning None silently would drop
|
||||
the download with no user feedback (per JohnBaumb)."""
|
||||
import pytest
|
||||
client = QobuzClient.__new__(QobuzClient)
|
||||
client._engine = None
|
||||
with pytest.raises(RuntimeError, match="engine reference"):
|
||||
_run_async(client.download('qobuz', 'v||t', 0))
|
||||
|
||||
|
||||
def test_download_returns_uuid_for_valid_filename(qobuz_client_with_engine):
|
||||
client, _ = qobuz_client_with_engine
|
||||
with patch.object(client, '_download_sync', return_value='/tmp/x.flac'):
|
||||
result = _run_async(client.download('qobuz', '12345||Some Song', 0))
|
||||
assert result is not None
|
||||
assert len(result) == 36
|
||||
|
||||
|
||||
def test_download_populates_engine_record_with_initial_state(qobuz_client_with_engine):
|
||||
client, engine = qobuz_client_with_engine
|
||||
started = threading.Event()
|
||||
release = threading.Event()
|
||||
|
||||
def slow_impl(*args, **kwargs):
|
||||
started.set()
|
||||
release.wait(timeout=1.0)
|
||||
return '/tmp/done.flac'
|
||||
|
||||
with patch.object(client, '_download_sync', side_effect=slow_impl):
|
||||
download_id = _run_async(client.download('qobuz', '999||My Qobuz Song', 0))
|
||||
started.wait(timeout=1.0)
|
||||
record = engine.get_record('qobuz', download_id)
|
||||
|
||||
assert record is not None
|
||||
assert record['filename'] == '999||My Qobuz Song'
|
||||
assert record['username'] == 'qobuz'
|
||||
assert record['track_id'] == 999
|
||||
assert record['display_name'] == 'My Qobuz Song'
|
||||
release.set()
|
||||
|
||||
|
||||
def test_get_all_downloads_reads_engine_records(qobuz_client_with_engine):
|
||||
client, engine = qobuz_client_with_engine
|
||||
engine.add_record('qobuz', 'dl-1', {
|
||||
'id': 'dl-1', 'filename': '111||A', 'username': 'qobuz',
|
||||
'state': 'InProgress, Downloading', 'progress': 50.0,
|
||||
})
|
||||
result = _run_async(client.get_all_downloads())
|
||||
assert len(result) == 1
|
||||
assert result[0].id == 'dl-1'
|
||||
|
||||
|
||||
def test_cancel_download_marks_cancelled(qobuz_client_with_engine):
|
||||
client, engine = qobuz_client_with_engine
|
||||
engine.add_record('qobuz', 'dl-1', {'id': 'dl-1', 'state': 'InProgress, Downloading'})
|
||||
|
||||
ok = _run_async(client.cancel_download('dl-1', None, remove=False))
|
||||
assert ok is True
|
||||
assert engine.get_record('qobuz', 'dl-1')['state'] == 'Cancelled'
|
||||
|
||||
|
||||
def test_clear_all_completed_drops_only_terminal_records(qobuz_client_with_engine):
|
||||
client, engine = qobuz_client_with_engine
|
||||
engine.add_record('qobuz', 'done', {'id': 'done', 'state': 'Completed, Succeeded'})
|
||||
engine.add_record('qobuz', 'live', {'id': 'live', 'state': 'InProgress, Downloading'})
|
||||
|
||||
_run_async(client.clear_all_completed_downloads())
|
||||
|
||||
assert engine.get_record('qobuz', 'done') is None
|
||||
assert engine.get_record('qobuz', 'live') is not None
|
||||
105
tests/downloads/test_rate_limit_policy.py
Normal file
105
tests/downloads/test_rate_limit_policy.py
Normal file
|
|
@ -0,0 +1,105 @@
|
|||
"""Tests for the per-source RateLimitPolicy declaration mechanism (Phase E1)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from core.download_engine import DownloadEngine, RateLimitPolicy
|
||||
from core.download_engine.rate_limit import DEFAULT_POLICY, resolve_policy
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# resolve_policy
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_resolve_policy_returns_default_when_plugin_declares_nothing():
|
||||
plugin = object()
|
||||
assert resolve_policy(plugin) is DEFAULT_POLICY
|
||||
|
||||
|
||||
def test_resolve_policy_reads_class_attribute():
|
||||
class _Plugin:
|
||||
RATE_LIMIT_POLICY = RateLimitPolicy(download_delay_seconds=5.0)
|
||||
|
||||
policy = resolve_policy(_Plugin())
|
||||
assert policy.download_delay_seconds == 5.0
|
||||
|
||||
|
||||
def test_resolve_policy_prefers_method_over_class_attribute():
|
||||
class _Plugin:
|
||||
RATE_LIMIT_POLICY = RateLimitPolicy(download_delay_seconds=1.0)
|
||||
|
||||
def rate_limit_policy(self):
|
||||
return RateLimitPolicy(download_delay_seconds=10.0)
|
||||
|
||||
assert resolve_policy(_Plugin()).download_delay_seconds == 10.0
|
||||
|
||||
|
||||
def test_resolve_policy_falls_back_to_default_when_method_returns_garbage():
|
||||
class _Plugin:
|
||||
def rate_limit_policy(self):
|
||||
return "not a policy object"
|
||||
|
||||
assert resolve_policy(_Plugin()) is DEFAULT_POLICY
|
||||
|
||||
|
||||
def test_resolve_policy_falls_back_to_default_when_method_raises():
|
||||
class _Plugin:
|
||||
def rate_limit_policy(self):
|
||||
raise RuntimeError("boom")
|
||||
|
||||
assert resolve_policy(_Plugin()) is DEFAULT_POLICY
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Engine applies policy on register
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_engine_applies_declared_policy_on_register():
|
||||
"""Pinning: when a plugin is registered, its declared
|
||||
RateLimitPolicy is pushed into the worker's per-source semaphore +
|
||||
delay registry. Future dispatches use those values."""
|
||||
class _ThrottledPlugin:
|
||||
RATE_LIMIT_POLICY = RateLimitPolicy(download_concurrency=1, download_delay_seconds=2.5)
|
||||
|
||||
engine = DownloadEngine()
|
||||
engine.register_plugin('throttled', _ThrottledPlugin())
|
||||
|
||||
assert engine.worker._get_delay('throttled') == 2.5
|
||||
|
||||
|
||||
def test_engine_applies_default_policy_when_plugin_declares_nothing():
|
||||
"""Plugins without a declaration get the conservative default
|
||||
(concurrency=1, delay=0)."""
|
||||
class _DefaultPlugin:
|
||||
pass
|
||||
|
||||
engine = DownloadEngine()
|
||||
engine.register_plugin('default', _DefaultPlugin())
|
||||
|
||||
assert engine.worker._get_delay('default') == 0.0
|
||||
|
||||
|
||||
def test_set_engine_callback_runs_after_policy_applied():
|
||||
"""Pinning: set_engine fires AFTER policy registration, so
|
||||
config-driven sources can override their declared policy.
|
||||
YouTube uses this — set_engine reads the user-tunable
|
||||
youtube.download_delay config and overrides the declared default."""
|
||||
fired_at: list = []
|
||||
|
||||
class _Plugin:
|
||||
RATE_LIMIT_POLICY = RateLimitPolicy(download_delay_seconds=1.0)
|
||||
|
||||
def set_engine(self, engine):
|
||||
# Capture worker state at the moment set_engine fires.
|
||||
fired_at.append(engine.worker._get_delay('flexible'))
|
||||
# Then override.
|
||||
engine.worker.set_delay('flexible', 99.0)
|
||||
|
||||
engine = DownloadEngine()
|
||||
engine.register_plugin('flexible', _Plugin())
|
||||
|
||||
# The class-attribute value was applied first.
|
||||
assert fired_at == [1.0]
|
||||
# Then set_engine overrode it.
|
||||
assert engine.worker._get_delay('flexible') == 99.0
|
||||
258
tests/downloads/test_soulseek_pinning.py
Normal file
258
tests/downloads/test_soulseek_pinning.py
Normal file
|
|
@ -0,0 +1,258 @@
|
|||
"""Phase A pinning tests for SoulseekClient's download lifecycle.
|
||||
|
||||
These tests pin the OBSERVABLE BEHAVIOR of `SoulseekClient.download` /
|
||||
`get_all_downloads` / `cancel_download` so the upcoming download
|
||||
engine refactor (which lifts shared state + thread workers + search
|
||||
retry into a central engine) can't drift the per-source contract.
|
||||
|
||||
The contract these tests pin is what the engine will call into via
|
||||
`plugin.download_raw(target_id)` / `plugin.cancel_raw(target_id)`
|
||||
after the refactor lands. If a future commit breaks any of these
|
||||
expectations, the diff fails fast — long before a real download
|
||||
attempt against a live slskd would have surfaced the bug.
|
||||
|
||||
NOTE: Soulseek is structurally different from the streaming sources.
|
||||
It has NO local thread worker — slskd manages downloads server-side
|
||||
and the client just polls for state. So Soulseek skips most of the
|
||||
engine refactor's thread-extraction work; what stays critical is
|
||||
the slskd HTTP API contract (endpoints, payload shape, id
|
||||
extraction). That's what these tests pin.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
from pathlib import Path
|
||||
from unittest.mock import AsyncMock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
from core.soulseek_client import SoulseekClient
|
||||
|
||||
|
||||
def _run_async(coro):
|
||||
loop = asyncio.new_event_loop()
|
||||
try:
|
||||
return loop.run_until_complete(coro)
|
||||
finally:
|
||||
loop.close()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Configuration / lifecycle
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_is_configured_returns_false_when_no_base_url():
|
||||
"""Pinning: an unconfigured client (no slskd URL set) reports
|
||||
is_configured() == False. The orchestrator's hybrid fallback +
|
||||
every consumer that gates on is_configured() depends on this."""
|
||||
client = SoulseekClient.__new__(SoulseekClient)
|
||||
client.base_url = None
|
||||
client.api_key = None
|
||||
assert client.is_configured() is False
|
||||
|
||||
|
||||
def test_is_configured_returns_true_when_base_url_set():
|
||||
"""Pinning: configured client (slskd URL present) reports True."""
|
||||
client = SoulseekClient.__new__(SoulseekClient)
|
||||
client.base_url = 'http://localhost:5030'
|
||||
client.api_key = 'test-key'
|
||||
assert client.is_configured() is True
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# download()
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def configured_client():
|
||||
"""A SoulseekClient with the slskd URL set but no real network. Tests
|
||||
individually patch `_make_request` to return whatever shape they
|
||||
want to exercise."""
|
||||
client = SoulseekClient.__new__(SoulseekClient)
|
||||
client.base_url = 'http://localhost:5030'
|
||||
client.api_key = 'test-key'
|
||||
client.download_path = Path('./test_downloads')
|
||||
return client
|
||||
|
||||
|
||||
def test_download_returns_none_when_not_configured():
|
||||
"""Pinning: an unconfigured client refuses downloads — returns
|
||||
None silently rather than raising. Used as the soft-fail signal
|
||||
by the orchestrator's per-source fallback chain."""
|
||||
client = SoulseekClient.__new__(SoulseekClient)
|
||||
client.base_url = None
|
||||
result = _run_async(client.download('user', 'song.flac', 1024))
|
||||
assert result is None
|
||||
|
||||
|
||||
def test_download_hits_transfers_downloads_username_endpoint(configured_client):
|
||||
"""Pinning: the primary download endpoint is
|
||||
`transfers/downloads/<username>` POST. This shape was chosen to
|
||||
match slskd's web-interface API exactly. Changing it breaks
|
||||
every download against current slskd builds."""
|
||||
captured = []
|
||||
|
||||
async def fake_request(method, endpoint, json=None, **kwargs):
|
||||
captured.append((method, endpoint, json))
|
||||
return {'id': 'dl-id-from-slskd'}
|
||||
|
||||
with patch.object(configured_client, '_make_request', side_effect=fake_request):
|
||||
result = _run_async(configured_client.download('user', 'song.flac', 1024))
|
||||
|
||||
assert result == 'dl-id-from-slskd'
|
||||
method, endpoint, payload = captured[0]
|
||||
assert method == 'POST'
|
||||
assert endpoint == 'transfers/downloads/user'
|
||||
# Payload is the slskd web-interface array format.
|
||||
assert isinstance(payload, list)
|
||||
assert payload[0]['filename'] == 'song.flac'
|
||||
assert payload[0]['size'] == 1024
|
||||
|
||||
|
||||
def test_download_extracts_id_from_dict_response(configured_client):
|
||||
"""Pinning: when slskd returns `{id: ...}`, that's the
|
||||
download_id the orchestrator uses to track the download."""
|
||||
with patch.object(configured_client, '_make_request',
|
||||
AsyncMock(return_value={'id': 'abc123'})):
|
||||
result = _run_async(configured_client.download('user', 'song.flac', 1024))
|
||||
assert result == 'abc123'
|
||||
|
||||
|
||||
def test_download_extracts_id_from_list_response(configured_client):
|
||||
"""Pinning: slskd sometimes returns a list of file objects.
|
||||
The first item's id is the download_id."""
|
||||
with patch.object(configured_client, '_make_request',
|
||||
AsyncMock(return_value=[{'id': 'list-id'}, {'id': 'second'}])):
|
||||
result = _run_async(configured_client.download('user', 'song.flac', 1024))
|
||||
assert result == 'list-id'
|
||||
|
||||
|
||||
def test_download_falls_back_to_filename_when_no_id_in_response(configured_client):
|
||||
"""Pinning: defensive — older slskd builds returned 201 Created
|
||||
with no id field. The client uses the filename as the download
|
||||
identifier in that case so downstream tracking still works."""
|
||||
with patch.object(configured_client, '_make_request',
|
||||
AsyncMock(return_value={'status': 'queued'})):
|
||||
result = _run_async(configured_client.download('user', 'song.flac', 1024))
|
||||
assert result == 'song.flac'
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# get_all_downloads()
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_get_all_downloads_returns_empty_when_not_configured():
|
||||
client = SoulseekClient.__new__(SoulseekClient)
|
||||
client.base_url = None
|
||||
result = _run_async(client.get_all_downloads())
|
||||
assert result == []
|
||||
|
||||
|
||||
def test_get_all_downloads_parses_nested_user_directory_files_response(configured_client):
|
||||
"""Pinning: slskd's `transfers/downloads` returns
|
||||
`[{username, directories: [{files: [...]}]}]`. The client
|
||||
flattens that into a list of DownloadStatus objects, one per
|
||||
file. Engine refactor's state aggregation depends on this shape."""
|
||||
fake_response = [
|
||||
{
|
||||
'username': 'peer1',
|
||||
'directories': [{
|
||||
'files': [
|
||||
{'id': 'f1', 'filename': 'a.flac', 'state': 'InProgress',
|
||||
'size': 100, 'bytesTransferred': 50, 'averageSpeed': 1024},
|
||||
{'id': 'f2', 'filename': 'b.flac', 'state': 'Completed, Succeeded',
|
||||
'size': 200, 'bytesTransferred': 200, 'averageSpeed': 2048},
|
||||
],
|
||||
}],
|
||||
},
|
||||
]
|
||||
|
||||
with patch.object(configured_client, '_make_request',
|
||||
AsyncMock(return_value=fake_response)):
|
||||
result = _run_async(configured_client.get_all_downloads())
|
||||
|
||||
assert len(result) == 2
|
||||
assert result[0].id == 'f1'
|
||||
assert result[0].username == 'peer1'
|
||||
assert result[0].state == 'InProgress'
|
||||
assert result[1].id == 'f2'
|
||||
# Pinning: 'Completed' state forces progress=100 regardless of source data.
|
||||
assert result[1].progress == 100.0
|
||||
|
||||
|
||||
def test_get_all_downloads_endpoint_is_transfers_downloads(configured_client):
|
||||
"""Pinning: the listing endpoint is `transfers/downloads` (no
|
||||
username). The 404'd `users/.../downloads` variant was tried
|
||||
once and removed — keep it gone."""
|
||||
captured = []
|
||||
|
||||
async def fake_request(method, endpoint, **kwargs):
|
||||
captured.append((method, endpoint))
|
||||
return []
|
||||
|
||||
with patch.object(configured_client, '_make_request', side_effect=fake_request):
|
||||
_run_async(configured_client.get_all_downloads())
|
||||
|
||||
assert captured == [('GET', 'transfers/downloads')]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# cancel_download()
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_cancel_download_returns_false_when_not_configured():
|
||||
client = SoulseekClient.__new__(SoulseekClient)
|
||||
client.base_url = None
|
||||
result = _run_async(client.cancel_download('dl-id', 'user', remove=False))
|
||||
assert result is False
|
||||
|
||||
|
||||
def test_cancel_download_looks_up_username_when_not_provided(configured_client):
|
||||
"""Pinning: orchestrator may call cancel_download without a
|
||||
username hint. The client falls back to scanning all downloads
|
||||
to find which peer owns it. Engine refactor must preserve this
|
||||
so existing API endpoints that don't pass username keep working."""
|
||||
fake_listing = [
|
||||
{
|
||||
'username': 'peer-owner',
|
||||
'directories': [{
|
||||
'files': [{'id': 'target-dl', 'filename': 'x.flac',
|
||||
'state': 'InProgress', 'size': 0,
|
||||
'bytesTransferred': 0, 'averageSpeed': 0}],
|
||||
}],
|
||||
},
|
||||
]
|
||||
|
||||
captured_endpoints = []
|
||||
|
||||
async def fake_request(method, endpoint, **kwargs):
|
||||
captured_endpoints.append((method, endpoint))
|
||||
if method == 'GET' and endpoint == 'transfers/downloads':
|
||||
return fake_listing
|
||||
# The DELETE for cancel — return success
|
||||
return True
|
||||
|
||||
with patch.object(configured_client, '_make_request', side_effect=fake_request):
|
||||
_run_async(configured_client.cancel_download('target-dl', None, remove=False))
|
||||
|
||||
# The lookup hit get_all_downloads first, then the DELETE used the discovered username.
|
||||
assert ('GET', 'transfers/downloads') in captured_endpoints
|
||||
delete_calls = [(m, e) for m, e in captured_endpoints if m == 'DELETE']
|
||||
assert delete_calls, "Expected at least one DELETE after username lookup"
|
||||
# The cancel endpoint URL contains the discovered username.
|
||||
assert any('peer-owner' in e for _, e in delete_calls)
|
||||
|
||||
|
||||
def test_cancel_download_returns_false_when_username_lookup_fails(configured_client):
|
||||
"""Pinning: if the download_id isn't in the active list, return
|
||||
False rather than raising. Orchestrator treats False as "couldn't
|
||||
cancel" and continues; an exception would propagate to the user."""
|
||||
with patch.object(configured_client, '_make_request',
|
||||
AsyncMock(return_value=[])):
|
||||
result = _run_async(configured_client.cancel_download('missing-id', None))
|
||||
assert result is False
|
||||
131
tests/downloads/test_soundcloud_pinning.py
Normal file
131
tests/downloads/test_soundcloud_pinning.py
Normal file
|
|
@ -0,0 +1,131 @@
|
|||
"""Phase A pinning tests for SoundcloudClient — UPDATED for Phase C7.
|
||||
|
||||
SoundCloud's quirk: 3-part filename `track_id||permalink_url||display_name`
|
||||
because yt-dlp consumes the URL, not the track_id, to actually download.
|
||||
The engine record holds both fields so the worker can call
|
||||
`_download_sync(download_id, permalink_url, display_name)` correctly.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import threading
|
||||
from pathlib import Path
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
|
||||
from core.download_engine import DownloadEngine
|
||||
from core.soundcloud_client import SoundcloudClient
|
||||
|
||||
|
||||
def _run_async(coro):
|
||||
loop = asyncio.new_event_loop()
|
||||
try:
|
||||
return loop.run_until_complete(coro)
|
||||
finally:
|
||||
loop.close()
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def sc_client_with_engine():
|
||||
client = SoundcloudClient.__new__(SoundcloudClient)
|
||||
client.download_path = Path('./test_sc_downloads')
|
||||
client.shutdown_check = None
|
||||
client._engine = None
|
||||
engine = DownloadEngine()
|
||||
client.set_engine(engine)
|
||||
return client, engine
|
||||
|
||||
|
||||
def test_download_returns_none_for_filename_with_too_few_parts(sc_client_with_engine):
|
||||
client, _ = sc_client_with_engine
|
||||
result = _run_async(client.download('soundcloud', 'just-id-no-url', 0))
|
||||
assert result is None
|
||||
|
||||
|
||||
def test_download_returns_none_for_empty_track_id_or_url(sc_client_with_engine):
|
||||
client, _ = sc_client_with_engine
|
||||
assert _run_async(client.download('soundcloud', '||https://x.com/y', 0)) is None
|
||||
assert _run_async(client.download('soundcloud', 'track123||', 0)) is None
|
||||
|
||||
|
||||
def test_download_raises_when_engine_not_wired():
|
||||
"""Defensive: client without engine reference must raise so the
|
||||
orchestrator's download_with_fallback surfaces the error and
|
||||
moves on to the next source. Returning None silently would drop
|
||||
the download with no user feedback (per JohnBaumb)."""
|
||||
import pytest
|
||||
client = SoundcloudClient.__new__(SoundcloudClient)
|
||||
client._engine = None
|
||||
with pytest.raises(RuntimeError, match="engine reference"):
|
||||
_run_async(client.download('soundcloud', 'v||t', 0))
|
||||
|
||||
|
||||
def test_download_accepts_three_part_filename_with_display(sc_client_with_engine):
|
||||
"""Pinning: 3-part filename `track_id||permalink_url||display`
|
||||
is the canonical form. All three fields go into the engine record."""
|
||||
client, engine = sc_client_with_engine
|
||||
started = threading.Event()
|
||||
release = threading.Event()
|
||||
|
||||
def slow_impl(*args, **kwargs):
|
||||
started.set()
|
||||
release.wait(timeout=1.0)
|
||||
return '/tmp/done.mp3'
|
||||
|
||||
with patch.object(client, '_download_sync', side_effect=slow_impl):
|
||||
download_id = _run_async(client.download(
|
||||
'soundcloud',
|
||||
'sc-12345||https://soundcloud.com/artist/song||Some Display Title',
|
||||
0,
|
||||
))
|
||||
started.wait(timeout=1.0)
|
||||
record = engine.get_record('soundcloud', download_id)
|
||||
|
||||
assert record['track_id'] == 'sc-12345'
|
||||
assert record['permalink_url'] == 'https://soundcloud.com/artist/song'
|
||||
assert record['display_name'] == 'Some Display Title'
|
||||
release.set()
|
||||
|
||||
|
||||
def test_download_falls_back_display_name_to_track_id_when_two_part(sc_client_with_engine):
|
||||
"""Pinning: 2-part filename → display name = track_id."""
|
||||
client, engine = sc_client_with_engine
|
||||
started = threading.Event()
|
||||
release = threading.Event()
|
||||
|
||||
def slow_impl(*args, **kwargs):
|
||||
started.set()
|
||||
release.wait(timeout=1.0)
|
||||
return '/tmp/x.mp3'
|
||||
|
||||
with patch.object(client, '_download_sync', side_effect=slow_impl):
|
||||
download_id = _run_async(client.download(
|
||||
'soundcloud', 'sc-99||https://soundcloud.com/x/y', 0,
|
||||
))
|
||||
started.wait(timeout=1.0)
|
||||
assert engine.get_record('soundcloud', download_id)['display_name'] == 'sc-99'
|
||||
release.set()
|
||||
|
||||
|
||||
def test_download_populates_engine_record_with_initial_state(sc_client_with_engine):
|
||||
client, engine = sc_client_with_engine
|
||||
started = threading.Event()
|
||||
release = threading.Event()
|
||||
|
||||
def slow_impl(*args, **kwargs):
|
||||
started.set()
|
||||
release.wait(timeout=1.0)
|
||||
return '/tmp/x.mp3'
|
||||
|
||||
with patch.object(client, '_download_sync', side_effect=slow_impl):
|
||||
download_id = _run_async(client.download(
|
||||
'soundcloud', 'sc-1||https://soundcloud.com/x||Title', 0,
|
||||
))
|
||||
started.wait(timeout=1.0)
|
||||
record = engine.get_record('soundcloud', download_id)
|
||||
assert record['username'] == 'soundcloud'
|
||||
assert record['state'] in ('Initializing', 'InProgress, Downloading')
|
||||
assert 'permalink_url' in record
|
||||
release.set()
|
||||
172
tests/downloads/test_tidal_pinning.py
Normal file
172
tests/downloads/test_tidal_pinning.py
Normal file
|
|
@ -0,0 +1,172 @@
|
|||
"""Phase A pinning tests for TidalDownloadClient — UPDATED for Phase C3.
|
||||
|
||||
Post-C3 the client no longer owns its own ``active_downloads`` dict
|
||||
or thread spawn — both moved into the engine's BackgroundDownloadWorker.
|
||||
Pinning tests now read state from ``engine.get_record('tidal', ...)``
|
||||
instead of ``client.active_downloads[...]``.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import threading
|
||||
from pathlib import Path
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
|
||||
from core.download_engine import DownloadEngine
|
||||
from core.tidal_download_client import TidalDownloadClient
|
||||
|
||||
|
||||
def _run_async(coro):
|
||||
loop = asyncio.new_event_loop()
|
||||
try:
|
||||
return loop.run_until_complete(coro)
|
||||
finally:
|
||||
loop.close()
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def tidal_client_with_engine():
|
||||
client = TidalDownloadClient.__new__(TidalDownloadClient)
|
||||
client.download_path = Path('./test_tidal_downloads')
|
||||
client.shutdown_check = None
|
||||
client.session = None
|
||||
client._device_auth_future = None
|
||||
client._device_auth_link = None
|
||||
client._engine = None
|
||||
|
||||
engine = DownloadEngine()
|
||||
client.set_engine(engine)
|
||||
return client, engine
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# is_configured / is_authenticated
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_is_authenticated_false_when_no_session(tidal_client_with_engine):
|
||||
client, _ = tidal_client_with_engine
|
||||
assert client.is_authenticated() is False
|
||||
|
||||
|
||||
def test_is_authenticated_false_when_session_check_login_raises(tidal_client_with_engine):
|
||||
client, _ = tidal_client_with_engine
|
||||
fake_session = type('FakeSession', (), {
|
||||
'check_login': lambda self: (_ for _ in ()).throw(RuntimeError("expired")),
|
||||
})()
|
||||
client.session = fake_session
|
||||
assert client.is_authenticated() is False
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# download() — filename parsing + id contract
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_download_returns_none_for_invalid_filename_format(tidal_client_with_engine):
|
||||
client, _ = tidal_client_with_engine
|
||||
result = _run_async(client.download('tidal', 'no-separator', 0))
|
||||
assert result is None
|
||||
|
||||
|
||||
def test_download_returns_none_for_non_integer_track_id(tidal_client_with_engine):
|
||||
client, _ = tidal_client_with_engine
|
||||
result = _run_async(client.download('tidal', 'not-int||title', 0))
|
||||
assert result is None
|
||||
|
||||
|
||||
def test_download_raises_when_engine_not_wired():
|
||||
"""Defensive: client without engine reference must raise so the
|
||||
orchestrator's download_with_fallback surfaces the error and
|
||||
moves on to the next source. Returning None silently would drop
|
||||
the download with no user feedback (per JohnBaumb)."""
|
||||
import pytest
|
||||
client = TidalDownloadClient.__new__(TidalDownloadClient)
|
||||
client._engine = None
|
||||
with pytest.raises(RuntimeError, match="engine reference"):
|
||||
_run_async(client.download('tidal', 'v||t', 0))
|
||||
|
||||
|
||||
def test_download_returns_uuid_for_valid_filename(tidal_client_with_engine):
|
||||
client, _ = tidal_client_with_engine
|
||||
with patch.object(client, '_download_sync', return_value='/tmp/x.flac'):
|
||||
result = _run_async(client.download('tidal', '12345||Some Song', 0))
|
||||
assert result is not None
|
||||
assert len(result) == 36
|
||||
|
||||
|
||||
def test_download_populates_engine_record_with_initial_state(tidal_client_with_engine):
|
||||
client, engine = tidal_client_with_engine
|
||||
started = threading.Event()
|
||||
release = threading.Event()
|
||||
|
||||
def slow_impl(*args, **kwargs):
|
||||
started.set()
|
||||
release.wait(timeout=1.0)
|
||||
return '/tmp/done.flac'
|
||||
|
||||
with patch.object(client, '_download_sync', side_effect=slow_impl):
|
||||
download_id = _run_async(client.download('tidal', '999||My Tidal Song', 0))
|
||||
started.wait(timeout=1.0)
|
||||
record = engine.get_record('tidal', download_id)
|
||||
|
||||
assert record is not None
|
||||
assert record['id'] == download_id
|
||||
assert record['filename'] == '999||My Tidal Song'
|
||||
assert record['username'] == 'tidal'
|
||||
assert record['state'] in ('Initializing', 'InProgress, Downloading')
|
||||
assert record['progress'] == 0.0
|
||||
assert record['track_id'] == 999 # parsed as int
|
||||
assert record['display_name'] == 'My Tidal Song'
|
||||
assert record['file_path'] is None
|
||||
release.set()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Query / cancel — engine-backed reads
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_get_all_downloads_reads_engine_records(tidal_client_with_engine):
|
||||
client, engine = tidal_client_with_engine
|
||||
engine.add_record('tidal', 'dl-1', {
|
||||
'id': 'dl-1', 'filename': '111||Song A', 'username': 'tidal',
|
||||
'state': 'InProgress, Downloading', 'progress': 50.0,
|
||||
'size': 1000, 'transferred': 500, 'speed': 100,
|
||||
})
|
||||
engine.add_record('tidal', 'dl-2', {
|
||||
'id': 'dl-2', 'filename': '222||Song B', 'username': 'tidal',
|
||||
'state': 'Completed, Succeeded', 'progress': 100.0,
|
||||
'size': 2000, 'transferred': 2000, 'speed': 0,
|
||||
})
|
||||
result = _run_async(client.get_all_downloads())
|
||||
assert len(result) == 2
|
||||
assert {r.id for r in result} == {'dl-1', 'dl-2'}
|
||||
assert {r.username for r in result} == {'tidal'}
|
||||
|
||||
|
||||
def test_cancel_download_marks_cancelled(tidal_client_with_engine):
|
||||
client, engine = tidal_client_with_engine
|
||||
engine.add_record('tidal', 'dl-1', {'id': 'dl-1', 'state': 'InProgress, Downloading'})
|
||||
|
||||
ok = _run_async(client.cancel_download('dl-1', None, remove=False))
|
||||
assert ok is True
|
||||
assert engine.get_record('tidal', 'dl-1')['state'] == 'Cancelled'
|
||||
|
||||
ok = _run_async(client.cancel_download('dl-1', None, remove=True))
|
||||
assert ok is True
|
||||
assert engine.get_record('tidal', 'dl-1') is None
|
||||
|
||||
|
||||
def test_clear_all_completed_drops_only_terminal_records(tidal_client_with_engine):
|
||||
client, engine = tidal_client_with_engine
|
||||
engine.add_record('tidal', 'done', {'id': 'done', 'state': 'Completed, Succeeded'})
|
||||
engine.add_record('tidal', 'live', {'id': 'live', 'state': 'InProgress, Downloading'})
|
||||
|
||||
_run_async(client.clear_all_completed_downloads())
|
||||
|
||||
assert engine.get_record('tidal', 'done') is None
|
||||
assert engine.get_record('tidal', 'live') is not None
|
||||
212
tests/downloads/test_youtube_pinning.py
Normal file
212
tests/downloads/test_youtube_pinning.py
Normal file
|
|
@ -0,0 +1,212 @@
|
|||
"""Phase A pinning tests for YouTubeClient — UPDATED for Phase C2.
|
||||
|
||||
Post-C2 the client no longer owns its own ``active_downloads`` dict
|
||||
or thread spawn — both moved into the engine's BackgroundDownloadWorker.
|
||||
These tests still pin the same OBSERVABLE CONTRACT (filename
|
||||
encoding, UUID download_id, initial-record schema, source-specific
|
||||
extras like video_id/url/title) but read state from
|
||||
``engine.get_record(...)`` instead of ``client.active_downloads[...]``.
|
||||
|
||||
What pre-C2 pinning tests caught and what these still catch:
|
||||
- Filename format: `video_id||title` ✓
|
||||
- Invalid filename → None ✓
|
||||
- UUID download_id format ✓
|
||||
- Per-download record schema (id, filename, username, state,
|
||||
progress, video_id, url, title, file_path) ✓
|
||||
- Source name in record's username slot is `'youtube'` ✓
|
||||
|
||||
What dropped (covered by other tests):
|
||||
- Direct thread-spawn assertion (engine.worker has its own tests).
|
||||
- `_download_thread_worker` target (gone from client; engine owns
|
||||
it).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import threading
|
||||
from pathlib import Path
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
|
||||
from core.download_engine import DownloadEngine
|
||||
from core.youtube_client import YouTubeClient
|
||||
|
||||
|
||||
def _run_async(coro):
|
||||
loop = asyncio.new_event_loop()
|
||||
try:
|
||||
return loop.run_until_complete(coro)
|
||||
finally:
|
||||
loop.close()
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def yt_client_with_engine():
|
||||
"""A bare YouTubeClient wired into a real engine. The engine
|
||||
callback is invoked manually since we bypass orchestrator init."""
|
||||
client = YouTubeClient.__new__(YouTubeClient)
|
||||
client.download_path = Path('./test_yt_downloads')
|
||||
client.shutdown_check = None
|
||||
client.matching_engine = None
|
||||
client._download_delay = 3
|
||||
client.current_download_id = None
|
||||
client.current_download_progress = {
|
||||
'status': 'idle', 'percent': 0.0, 'downloaded_bytes': 0,
|
||||
'total_bytes': 0, 'speed': 0, 'eta': 0, 'filename': '',
|
||||
}
|
||||
client.progress_callback = None
|
||||
client.download_opts = {}
|
||||
client._engine = None
|
||||
|
||||
engine = DownloadEngine()
|
||||
client.set_engine(engine)
|
||||
return client, engine
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# download() — filename parsing + id contract
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_download_returns_none_for_invalid_filename_format(yt_client_with_engine):
|
||||
"""Pinning: missing `||` → None (not exception)."""
|
||||
client, _ = yt_client_with_engine
|
||||
result = _run_async(client.download('youtube', 'no-separator', 0))
|
||||
assert result is None
|
||||
|
||||
|
||||
def test_download_raises_when_engine_not_wired():
|
||||
"""Defensive: client without engine reference must raise so the
|
||||
orchestrator's download_with_fallback surfaces the error and
|
||||
moves on to the next source. Returning None silently would drop
|
||||
the download with no user feedback (per JohnBaumb)."""
|
||||
import pytest
|
||||
client = YouTubeClient.__new__(YouTubeClient)
|
||||
client._engine = None
|
||||
with pytest.raises(RuntimeError, match="engine reference"):
|
||||
_run_async(client.download('youtube', 'v||t', 0))
|
||||
|
||||
|
||||
def test_download_returns_uuid_download_id_for_valid_filename(yt_client_with_engine):
|
||||
"""Pinning: valid `video_id||title` → UUID download_id."""
|
||||
client, engine = yt_client_with_engine
|
||||
|
||||
# Patch _download_sync so the worker thread's impl returns
|
||||
# without doing real yt-dlp work.
|
||||
with patch.object(client, '_download_sync', return_value='/tmp/x.mp3'):
|
||||
result = _run_async(client.download('youtube', 'abc123||Some Song', 0))
|
||||
|
||||
assert result is not None
|
||||
assert len(result) == 36
|
||||
assert result.count('-') == 4
|
||||
|
||||
|
||||
def test_download_populates_engine_record_with_initial_state(yt_client_with_engine):
|
||||
"""Pinning: per-download record schema. STATE LOCATION CHANGED
|
||||
in C2 (now in engine), but the SHAPE of the record is the same
|
||||
— frontend / status APIs / context-key matching depend on these
|
||||
keys."""
|
||||
client, engine = yt_client_with_engine
|
||||
|
||||
# Hold the impl so we can read 'Initializing' / 'InProgress' state
|
||||
# before the worker completes.
|
||||
started = threading.Event()
|
||||
release = threading.Event()
|
||||
|
||||
def slow_impl(*args, **kwargs):
|
||||
started.set()
|
||||
release.wait(timeout=1.0)
|
||||
return '/tmp/done.mp3'
|
||||
|
||||
with patch.object(client, '_download_sync', side_effect=slow_impl):
|
||||
download_id = _run_async(
|
||||
client.download('youtube', 'video123||My Title', 5000)
|
||||
)
|
||||
started.wait(timeout=1.0)
|
||||
record = engine.get_record('youtube', download_id)
|
||||
|
||||
assert record is not None
|
||||
assert record['id'] == download_id
|
||||
assert record['filename'] == 'video123||My Title' # ORIGINAL form
|
||||
assert record['username'] == 'youtube'
|
||||
assert record['state'] in ('Initializing', 'InProgress, Downloading')
|
||||
assert record['progress'] == 0.0
|
||||
assert record['file_path'] is None
|
||||
# Source-specific extras must merge into the record.
|
||||
assert record['video_id'] == 'video123'
|
||||
assert record['url'] == 'https://www.youtube.com/watch?v=video123'
|
||||
assert record['title'] == 'My Title'
|
||||
|
||||
release.set()
|
||||
|
||||
|
||||
def test_set_engine_configures_worker_delay(yt_client_with_engine):
|
||||
"""Pinning: when engine is wired, the YouTube download_delay
|
||||
config (3s default) propagates to the worker so successive
|
||||
downloads serialize with the same gap they did pre-C2."""
|
||||
client, engine = yt_client_with_engine
|
||||
# Default delay is 3s.
|
||||
assert engine.worker._get_delay('youtube') == 3.0
|
||||
|
||||
|
||||
def test_rate_limit_policy_reflects_configured_delay(yt_client_with_engine):
|
||||
"""Pinning (Phase E): YouTube's rate_limit_policy() returns a
|
||||
RateLimitPolicy with the configured download_delay (3s default
|
||||
from `youtube.download_delay` config). Engine reads this at
|
||||
register_plugin time."""
|
||||
client, _ = yt_client_with_engine
|
||||
policy = client.rate_limit_policy()
|
||||
assert policy.download_delay_seconds == 3.0
|
||||
assert policy.download_concurrency == 1
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Query / cancel — engine-backed reads
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_get_all_downloads_reads_engine_records(yt_client_with_engine):
|
||||
client, engine = yt_client_with_engine
|
||||
|
||||
# Seed engine with a fake record to mirror what dispatch would do.
|
||||
engine.add_record('youtube', 'dl-1', {
|
||||
'id': 'dl-1', 'filename': 'v||t', 'username': 'youtube',
|
||||
'state': 'InProgress, Downloading', 'progress': 50.0,
|
||||
'size': 1000, 'transferred': 500, 'speed': 100,
|
||||
})
|
||||
result = _run_async(client.get_all_downloads())
|
||||
assert len(result) == 1
|
||||
assert result[0].id == 'dl-1'
|
||||
assert result[0].state == 'InProgress, Downloading'
|
||||
|
||||
|
||||
def test_cancel_download_marks_cancelled_and_optionally_removes(yt_client_with_engine):
|
||||
client, engine = yt_client_with_engine
|
||||
|
||||
engine.add_record('youtube', 'dl-1', {
|
||||
'id': 'dl-1', 'filename': 'v||t', 'username': 'youtube',
|
||||
'state': 'InProgress, Downloading', 'progress': 50.0,
|
||||
})
|
||||
|
||||
ok = _run_async(client.cancel_download('dl-1', None, remove=False))
|
||||
assert ok is True
|
||||
assert engine.get_record('youtube', 'dl-1')['state'] == 'Cancelled'
|
||||
|
||||
ok = _run_async(client.cancel_download('dl-1', None, remove=True))
|
||||
assert ok is True
|
||||
assert engine.get_record('youtube', 'dl-1') is None
|
||||
|
||||
|
||||
def test_clear_all_completed_drops_only_terminal_records(yt_client_with_engine):
|
||||
client, engine = yt_client_with_engine
|
||||
engine.add_record('youtube', 'done', {'id': 'done', 'state': 'Completed, Succeeded'})
|
||||
engine.add_record('youtube', 'erred', {'id': 'erred', 'state': 'Errored'})
|
||||
engine.add_record('youtube', 'live', {'id': 'live', 'state': 'InProgress, Downloading'})
|
||||
|
||||
_run_async(client.clear_all_completed_downloads())
|
||||
|
||||
assert engine.get_record('youtube', 'done') is None
|
||||
assert engine.get_record('youtube', 'erred') is None
|
||||
assert engine.get_record('youtube', 'live') is not None
|
||||
|
|
@ -129,7 +129,7 @@ def _build_deps(**overrides):
|
|||
spotify_client=None,
|
||||
hydrabase_client=None,
|
||||
hydrabase_worker=None,
|
||||
soulseek_client=None,
|
||||
download_orchestrator=None,
|
||||
fix_artist_image_url=lambda u: f'FIXED::{u}' if u else None,
|
||||
is_hydrabase_active=lambda: False,
|
||||
get_metadata_fallback_source=lambda: 'spotify',
|
||||
|
|
@ -462,24 +462,27 @@ class _FakeYouTube:
|
|||
|
||||
class _FakeSoulseekWithYT:
|
||||
def __init__(self, youtube):
|
||||
self.youtube = youtube
|
||||
self._youtube = youtube
|
||||
|
||||
def client(self, name):
|
||||
return self._youtube if name == 'youtube' else None
|
||||
|
||||
|
||||
def test_resolve_youtube_videos_returns_subclient():
|
||||
yt = _FakeYouTube()
|
||||
deps = _build_deps(soulseek_client=_FakeSoulseekWithYT(yt))
|
||||
deps = _build_deps(download_orchestrator=_FakeSoulseekWithYT(yt))
|
||||
assert orchestrator.resolve_youtube_videos_client(deps) is yt
|
||||
|
||||
|
||||
def test_resolve_youtube_videos_no_soulseek_returns_none():
|
||||
deps = _build_deps(soulseek_client=None)
|
||||
deps = _build_deps(download_orchestrator=None)
|
||||
assert orchestrator.resolve_youtube_videos_client(deps) is None
|
||||
|
||||
|
||||
def test_resolve_youtube_videos_no_youtube_attr_returns_none():
|
||||
class _NoYT:
|
||||
pass
|
||||
deps = _build_deps(soulseek_client=_NoYT())
|
||||
deps = _build_deps(download_orchestrator=_NoYT())
|
||||
assert orchestrator.resolve_youtube_videos_client(deps) is None
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -53,15 +53,20 @@ class _FakeStreamClient:
|
|||
|
||||
class _FakeSoulseek:
|
||||
def __init__(self, youtube=None, tidal=None, qobuz=None, hifi=None, deezer_dl=None, lidarr=None, results_per_query=None):
|
||||
self.youtube = youtube
|
||||
self.tidal = tidal
|
||||
self.qobuz = qobuz
|
||||
self.hifi = hifi
|
||||
self.deezer_dl = deezer_dl
|
||||
self.lidarr = lidarr
|
||||
self._clients = {
|
||||
'youtube': youtube,
|
||||
'tidal': tidal,
|
||||
'qobuz': qobuz,
|
||||
'hifi': hifi,
|
||||
'deezer_dl': deezer_dl,
|
||||
'lidarr': lidarr,
|
||||
}
|
||||
self._results = results_per_query or {}
|
||||
self.search_calls = []
|
||||
|
||||
def client(self, name):
|
||||
return self._clients.get(name)
|
||||
|
||||
async def search(self, query, timeout=15):
|
||||
self.search_calls.append(query)
|
||||
return self._results.get(query, ([], []))
|
||||
|
|
@ -164,7 +169,7 @@ def test_stream_finds_match_on_first_query():
|
|||
result = stream.stream_search_track(
|
||||
track_name='Money', artist_name='Pink Floyd', album_name=None,
|
||||
duration_ms=180000,
|
||||
config_manager=cfg, soulseek_client=soul, matching_engine=engine,
|
||||
config_manager=cfg, download_orchestrator=soul, matching_engine=engine,
|
||||
run_async=_run_async,
|
||||
)
|
||||
assert result is not None
|
||||
|
|
@ -185,7 +190,7 @@ def test_stream_walks_to_second_query_on_no_match():
|
|||
result = stream.stream_search_track(
|
||||
track_name='Money (Remastered)', artist_name='Pink Floyd', album_name=None,
|
||||
duration_ms=180000,
|
||||
config_manager=cfg, soulseek_client=soul, matching_engine=engine,
|
||||
config_manager=cfg, download_orchestrator=soul, matching_engine=engine,
|
||||
run_async=_run_async,
|
||||
)
|
||||
assert result is not None
|
||||
|
|
@ -204,7 +209,7 @@ def test_stream_returns_none_when_no_matches():
|
|||
result = stream.stream_search_track(
|
||||
track_name='Money', artist_name='Pink Floyd', album_name=None,
|
||||
duration_ms=180000,
|
||||
config_manager=cfg, soulseek_client=soul, matching_engine=engine,
|
||||
config_manager=cfg, download_orchestrator=soul, matching_engine=engine,
|
||||
run_async=_run_async,
|
||||
)
|
||||
assert result is None
|
||||
|
|
@ -212,7 +217,7 @@ def test_stream_returns_none_when_no_matches():
|
|||
|
||||
def test_stream_falls_back_to_default_soulseek_when_no_direct_client():
|
||||
# Effective mode = 'youtube' (coerced from soulseek), but soul.youtube is None
|
||||
# → code falls through to soulseek_client.search directly. Streaming-mode
|
||||
# → code falls through to download_orchestrator.search directly. Streaming-mode
|
||||
# query gen → "Pink Floyd Money".
|
||||
soul = _FakeSoulseek(results_per_query={'Pink Floyd Money': ([object()], [])})
|
||||
cfg = _FakeConfig({
|
||||
|
|
@ -224,7 +229,7 @@ def test_stream_falls_back_to_default_soulseek_when_no_direct_client():
|
|||
result = stream.stream_search_track(
|
||||
track_name='Money', artist_name='Pink Floyd', album_name=None,
|
||||
duration_ms=180000,
|
||||
config_manager=cfg, soulseek_client=soul, matching_engine=engine,
|
||||
config_manager=cfg, download_orchestrator=soul, matching_engine=engine,
|
||||
run_async=_run_async,
|
||||
)
|
||||
assert result is not None
|
||||
|
|
@ -252,7 +257,7 @@ def test_stream_continues_past_per_query_exception():
|
|||
result = stream.stream_search_track(
|
||||
track_name='Money (Live)', artist_name='Pink Floyd', album_name=None,
|
||||
duration_ms=180000,
|
||||
config_manager=cfg, soulseek_client=soul, matching_engine=engine,
|
||||
config_manager=cfg, download_orchestrator=soul, matching_engine=engine,
|
||||
run_async=_run_async,
|
||||
)
|
||||
# First query raised, second succeeded
|
||||
|
|
|
|||
|
|
@ -10,7 +10,7 @@ from core.streaming import prepare as sp
|
|||
|
||||
|
||||
class _FakeSoulseek:
|
||||
"""Minimal soulseek_client stub for the stream-prep worker."""
|
||||
"""Minimal download_orchestrator stub for the stream-prep worker."""
|
||||
|
||||
def __init__(self, *, download_id='dl-1', all_downloads=None):
|
||||
self._download_id = download_id
|
||||
|
|
@ -43,7 +43,7 @@ def _build_deps(
|
|||
state = state if state is not None else {}
|
||||
deps = sp.PrepareStreamDeps(
|
||||
config_manager=type('C', (), {'get': lambda self, k, d=None: d})(),
|
||||
soulseek_client=soulseek or _FakeSoulseek(),
|
||||
download_orchestrator=soulseek or _FakeSoulseek(),
|
||||
stream_lock=threading.Lock(),
|
||||
project_root=project_root,
|
||||
docker_resolve_path=lambda p: p,
|
||||
|
|
@ -107,7 +107,7 @@ def test_stream_folder_cleared_before_download(tmp_path):
|
|||
# ---------------------------------------------------------------------------
|
||||
|
||||
def test_download_returns_none_marks_error(tmp_path):
|
||||
"""soulseek_client.download() returning None → state.error."""
|
||||
"""download_orchestrator.download() returning None → state.error."""
|
||||
sk = _FakeSoulseek(download_id=None)
|
||||
deps = _build_deps(soulseek=sk, project_root=str(tmp_path))
|
||||
|
||||
|
|
|
|||
|
|
@ -39,18 +39,18 @@ def orchestrator() -> DownloadOrchestrator:
|
|||
|
||||
|
||||
def test_orchestrator_constructs_soundcloud_client(orchestrator: DownloadOrchestrator) -> None:
|
||||
assert orchestrator.soundcloud is not None
|
||||
assert isinstance(orchestrator.soundcloud, SoundcloudClient)
|
||||
assert orchestrator.client('soundcloud') is not None
|
||||
assert isinstance(orchestrator.client('soundcloud'), SoundcloudClient)
|
||||
|
||||
|
||||
def test_client_lookup_resolves_soundcloud(orchestrator: DownloadOrchestrator) -> None:
|
||||
"""Verify the dict-based name → client lookup includes SoundCloud."""
|
||||
assert orchestrator._client('soundcloud') is orchestrator.soundcloud
|
||||
"""Verify the registry-backed name → client lookup includes SoundCloud."""
|
||||
assert orchestrator.client('soundcloud') is orchestrator.registry.get('soundcloud')
|
||||
|
||||
|
||||
def test_client_lookup_returns_none_for_unknown(orchestrator: DownloadOrchestrator) -> None:
|
||||
"""Sanity: unknown sources don't somehow resolve to SoundCloud."""
|
||||
assert orchestrator._client('made_up') is None
|
||||
assert orchestrator.client('made_up') is None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
|
|
@ -80,7 +80,7 @@ def test_download_routes_soundcloud_username_to_client(orchestrator: DownloadOrc
|
|||
async def _fake_download(username, filename, file_size=0):
|
||||
return sentinel
|
||||
|
||||
with patch.object(orchestrator.soundcloud, 'download', side_effect=_fake_download) as mock_dl:
|
||||
with patch.object(orchestrator.client('soundcloud'), 'download', side_effect=_fake_download) as mock_dl:
|
||||
result = _run(orchestrator.download(
|
||||
'soundcloud',
|
||||
'999||https://soundcloud.com/x/y||Display',
|
||||
|
|
@ -93,13 +93,13 @@ def test_download_routes_soundcloud_username_to_client(orchestrator: DownloadOrc
|
|||
def test_download_unknown_username_still_falls_to_soulseek(orchestrator: DownloadOrchestrator) -> None:
|
||||
"""Adding SoundCloud must not change the legacy Soulseek-fallback
|
||||
behavior for unrecognized usernames."""
|
||||
if orchestrator.soulseek is None:
|
||||
if orchestrator.client('soulseek') is None:
|
||||
pytest.skip("Soulseek client unavailable in this environment")
|
||||
|
||||
async def _fake_soulseek_download(username, filename, file_size=0):
|
||||
return 'soulseek-id'
|
||||
|
||||
with patch.object(orchestrator.soulseek, 'download', side_effect=_fake_soulseek_download) as mock_dl:
|
||||
with patch.object(orchestrator.client('soulseek'), 'download', side_effect=_fake_soulseek_download) as mock_dl:
|
||||
result = _run(orchestrator.download('some_random_user', 'file.mp3', 0))
|
||||
assert result == 'soulseek-id'
|
||||
mock_dl.assert_called_once()
|
||||
|
|
@ -122,8 +122,8 @@ def test_hybrid_search_iterates_soundcloud_when_in_order(orchestrator: DownloadO
|
|||
async def _fake_search(query, timeout=None, progress_callback=None):
|
||||
return ([fake_track], [])
|
||||
|
||||
with patch.object(orchestrator.soundcloud, 'search', side_effect=_fake_search), \
|
||||
patch.object(orchestrator.soundcloud, 'is_configured', return_value=True):
|
||||
with patch.object(orchestrator.client('soundcloud'), 'search', side_effect=_fake_search), \
|
||||
patch.object(orchestrator.client('soundcloud'), 'is_configured', return_value=True):
|
||||
tracks, albums = _run(orchestrator.search("any query"))
|
||||
|
||||
assert tracks == [fake_track]
|
||||
|
|
@ -136,7 +136,7 @@ def test_hybrid_search_skips_unconfigured_soundcloud(orchestrator: DownloadOrche
|
|||
orchestrator.mode = 'hybrid'
|
||||
orchestrator.hybrid_order = ['soundcloud', 'soulseek']
|
||||
|
||||
if orchestrator.soulseek is None:
|
||||
if orchestrator.client('soulseek') is None:
|
||||
pytest.skip("Soulseek client unavailable in this environment")
|
||||
|
||||
soulseek_track = MagicMock()
|
||||
|
|
@ -145,9 +145,9 @@ def test_hybrid_search_skips_unconfigured_soundcloud(orchestrator: DownloadOrche
|
|||
async def _fake_soulseek_search(query, timeout=None, progress_callback=None):
|
||||
return ([soulseek_track], [])
|
||||
|
||||
with patch.object(orchestrator.soundcloud, 'is_configured', return_value=False), \
|
||||
patch.object(orchestrator.soulseek, 'is_configured', return_value=True), \
|
||||
patch.object(orchestrator.soulseek, 'search', side_effect=_fake_soulseek_search):
|
||||
with patch.object(orchestrator.client('soundcloud'), 'is_configured', return_value=False), \
|
||||
patch.object(orchestrator.client('soulseek'), 'is_configured', return_value=True), \
|
||||
patch.object(orchestrator.client('soulseek'), 'search', side_effect=_fake_soulseek_search):
|
||||
tracks, _ = _run(orchestrator.search("any"))
|
||||
|
||||
assert tracks == [soulseek_track]
|
||||
|
|
@ -166,7 +166,7 @@ def test_get_all_downloads_walks_soundcloud(orchestrator: DownloadOrchestrator)
|
|||
async def _fake_get_all():
|
||||
return [fake_status]
|
||||
|
||||
with patch.object(orchestrator.soundcloud, 'get_all_downloads', side_effect=_fake_get_all):
|
||||
with patch.object(orchestrator.client('soundcloud'), 'get_all_downloads', side_effect=_fake_get_all):
|
||||
all_dl = _run(orchestrator.get_all_downloads())
|
||||
|
||||
assert any(d is fake_status for d in all_dl)
|
||||
|
|
@ -180,7 +180,7 @@ def test_get_download_status_finds_soundcloud_id(orchestrator: DownloadOrchestra
|
|||
async def _fake_get_status(download_id):
|
||||
return fake_status if download_id == 'sc-2' else None
|
||||
|
||||
with patch.object(orchestrator.soundcloud, 'get_download_status', side_effect=_fake_get_status):
|
||||
with patch.object(orchestrator.client('soundcloud'), 'get_download_status', side_effect=_fake_get_status):
|
||||
result = _run(orchestrator.get_download_status('sc-2'))
|
||||
|
||||
assert result is fake_status
|
||||
|
|
@ -192,7 +192,7 @@ def test_cancel_routes_soundcloud_username(orchestrator: DownloadOrchestrator) -
|
|||
async def _fake_cancel(download_id, username=None, remove=False):
|
||||
return True
|
||||
|
||||
with patch.object(orchestrator.soundcloud, 'cancel_download', side_effect=_fake_cancel) as mock_cancel:
|
||||
with patch.object(orchestrator.client('soundcloud'), 'cancel_download', side_effect=_fake_cancel) as mock_cancel:
|
||||
ok = _run(orchestrator.cancel_download('sc-3', username='soundcloud'))
|
||||
assert ok is True
|
||||
mock_cancel.assert_called_once()
|
||||
|
|
@ -210,7 +210,7 @@ def test_clear_completed_walks_soundcloud(orchestrator: DownloadOrchestrator) ->
|
|||
async def _fake_clear():
|
||||
return True
|
||||
|
||||
with patch.object(orchestrator.soundcloud, 'clear_all_completed_downloads', side_effect=_fake_clear) as mock_clear:
|
||||
with patch.object(orchestrator.client('soundcloud'), 'clear_all_completed_downloads', side_effect=_fake_clear) as mock_clear:
|
||||
_run(orchestrator.clear_all_completed_downloads())
|
||||
mock_clear.assert_called_once()
|
||||
|
||||
|
|
@ -228,8 +228,8 @@ def test_soundcloud_only_mode_uses_soundcloud(orchestrator: DownloadOrchestrator
|
|||
async def _fake_search(query, timeout=None, progress_callback=None):
|
||||
return ([MagicMock(username='soundcloud')], [])
|
||||
|
||||
with patch.object(orchestrator.soundcloud, 'search', side_effect=_fake_search) as mock_sc, \
|
||||
patch.object(orchestrator.soulseek, 'search', side_effect=AssertionError("soulseek must not be searched")):
|
||||
with patch.object(orchestrator.client('soundcloud'), 'search', side_effect=_fake_search) as mock_sc, \
|
||||
patch.object(orchestrator.client('soulseek'), 'search', side_effect=AssertionError("soulseek must not be searched")):
|
||||
tracks, _ = _run(orchestrator.search("any"))
|
||||
|
||||
assert len(tracks) == 1
|
||||
|
|
|
|||
163
tests/test_download_plugin_conformance.py
Normal file
163
tests/test_download_plugin_conformance.py
Normal file
|
|
@ -0,0 +1,163 @@
|
|||
"""Pin the structural conformance of every download source plugin
|
||||
class to ``DownloadSourcePlugin``.
|
||||
|
||||
Each registered source class MUST:
|
||||
- Implement every protocol method by name.
|
||||
- Mark async methods as `async def` so the orchestrator can `await`
|
||||
them uniformly.
|
||||
|
||||
When someone adds a new source (e.g. Usenet) and forgets one of
|
||||
these methods, this test fails at the contract — long before the
|
||||
first real download attempt would have raised AttributeError in
|
||||
production. When someone CHANGES the contract (adds a method to
|
||||
the protocol), this test forces every existing source to be
|
||||
updated.
|
||||
|
||||
Catches the smell that motivated the refactor in the first place:
|
||||
8 sources independently grew the same shape because every
|
||||
consumer site needed the same calls, but nothing enforced parity.
|
||||
|
||||
NOTE on test design: these tests check CLASSES, not instances.
|
||||
Instantiating real client classes (TidalDownloadClient, etc.) at
|
||||
fixture setup pollutes module-level state in tidalapi / spotipy
|
||||
imports and breaks downstream tests that rely on a clean import
|
||||
graph. Class-level checks are equally strict for structural
|
||||
conformance — the protocol only constrains the method surface, not
|
||||
runtime instance behavior.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import inspect
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
REQUIRED_SYNC_METHODS = {'is_configured'}
|
||||
|
||||
REQUIRED_ASYNC_METHODS = {
|
||||
'check_connection',
|
||||
'search',
|
||||
'download',
|
||||
'get_all_downloads',
|
||||
'get_download_status',
|
||||
'cancel_download',
|
||||
'clear_all_completed_downloads',
|
||||
}
|
||||
|
||||
|
||||
def _import_plugin_classes():
|
||||
"""Import every download source class lazily inside the test
|
||||
rather than at module load — avoids dragging tidalapi /
|
||||
spotipy / yt-dlp imports into every other test module's
|
||||
collection phase."""
|
||||
from core.soulseek_client import SoulseekClient
|
||||
from core.youtube_client import YouTubeClient
|
||||
from core.tidal_download_client import TidalDownloadClient
|
||||
from core.qobuz_client import QobuzClient
|
||||
from core.hifi_client import HiFiClient
|
||||
from core.deezer_download_client import DeezerDownloadClient
|
||||
from core.lidarr_download_client import LidarrDownloadClient
|
||||
from core.soundcloud_client import SoundcloudClient
|
||||
|
||||
return {
|
||||
'soulseek': SoulseekClient,
|
||||
'youtube': YouTubeClient,
|
||||
'tidal': TidalDownloadClient,
|
||||
'qobuz': QobuzClient,
|
||||
'hifi': HiFiClient,
|
||||
'deezer': DeezerDownloadClient,
|
||||
'lidarr': LidarrDownloadClient,
|
||||
'soundcloud': SoundcloudClient,
|
||||
}
|
||||
|
||||
|
||||
def test_default_registry_registers_all_eight_sources():
|
||||
"""Smoke check that the foundation registry knows about every
|
||||
source the orchestrator historically dispatched to. If someone
|
||||
drops a registration here, every other test in this module would
|
||||
silently miss the missing source."""
|
||||
from core.download_plugins.registry import build_default_registry
|
||||
|
||||
registry = build_default_registry()
|
||||
expected = {
|
||||
'soulseek', 'youtube', 'tidal', 'qobuz',
|
||||
'hifi', 'deezer', 'lidarr', 'soundcloud',
|
||||
}
|
||||
assert set(registry.names()) == expected
|
||||
|
||||
|
||||
def test_deezer_dl_alias_is_registered_against_deezer_spec():
|
||||
"""Legacy ``deezer_dl`` source-name string used in config + per-
|
||||
source dispatch must keep resolving — frontend, settings,
|
||||
download_orchestrator's username dispatch all depend on it."""
|
||||
from core.download_plugins.registry import build_default_registry
|
||||
|
||||
registry = build_default_registry()
|
||||
spec = registry.get_spec('deezer_dl')
|
||||
assert spec is not None
|
||||
assert spec.name == 'deezer'
|
||||
assert 'deezer_dl' in spec.aliases
|
||||
|
||||
|
||||
@pytest.mark.parametrize('plugin_name', [
|
||||
'soulseek', 'youtube', 'tidal', 'qobuz',
|
||||
'hifi', 'deezer', 'lidarr', 'soundcloud',
|
||||
])
|
||||
def test_plugin_class_has_all_required_methods(plugin_name):
|
||||
"""Every registered plugin class exposes every protocol method
|
||||
by name. Diagnostic-friendly: tells you WHICH method is missing
|
||||
when a new source is added without all the required methods."""
|
||||
classes = _import_plugin_classes()
|
||||
cls = classes[plugin_name]
|
||||
|
||||
missing = []
|
||||
for method_name in REQUIRED_SYNC_METHODS | REQUIRED_ASYNC_METHODS:
|
||||
if not hasattr(cls, method_name):
|
||||
missing.append(method_name)
|
||||
assert not missing, (
|
||||
f"{plugin_name} ({cls.__name__}) missing methods: {missing}"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize('plugin_name', [
|
||||
'soulseek', 'youtube', 'tidal', 'qobuz',
|
||||
'hifi', 'deezer', 'lidarr', 'soundcloud',
|
||||
])
|
||||
def test_plugin_class_async_methods_are_coroutines(plugin_name):
|
||||
"""Methods declared async in the protocol must be async on every
|
||||
plugin class. A sync `download()` would silently skip the
|
||||
orchestrator's `await` and return a coroutine object instead of
|
||||
a download_id — the kind of bug that only surfaces at runtime
|
||||
against a live user."""
|
||||
classes = _import_plugin_classes()
|
||||
cls = classes[plugin_name]
|
||||
|
||||
not_async = []
|
||||
for method_name in REQUIRED_ASYNC_METHODS:
|
||||
method = getattr(cls, method_name, None)
|
||||
if method is None:
|
||||
continue
|
||||
if not inspect.iscoroutinefunction(method):
|
||||
not_async.append(method_name)
|
||||
assert not not_async, (
|
||||
f"{plugin_name} ({cls.__name__}) declared these methods as "
|
||||
f"sync but the protocol requires async: {not_async}"
|
||||
)
|
||||
|
||||
|
||||
def test_orchestrator_uses_registry_for_dispatch():
|
||||
"""The orchestrator must hold a registry reference and the generic
|
||||
``client(name)`` accessor must return the same instances the
|
||||
registry holds. Per-source attribute aliases (``orchestrator.soulseek``
|
||||
etc.) were removed in favor of ``orchestrator.client('soulseek')``;
|
||||
the legacy alias name (``deezer_dl``) still resolves to the canonical
|
||||
deezer plugin via the registry's alias map."""
|
||||
from core.download_orchestrator import DownloadOrchestrator
|
||||
|
||||
orchestrator = DownloadOrchestrator()
|
||||
assert hasattr(orchestrator, 'registry')
|
||||
assert orchestrator.client('soulseek') is orchestrator.registry.get('soulseek')
|
||||
assert orchestrator.client('youtube') is orchestrator.registry.get('youtube')
|
||||
assert orchestrator.client('deezer_dl') is orchestrator.registry.get('deezer')
|
||||
assert orchestrator.client('lidarr') is orchestrator.registry.get('lidarr')
|
||||
|
|
@ -6,7 +6,7 @@ Settings showed "Connected: <username> (Active)" but underneath an error
|
|||
yellow even after a successful login.
|
||||
|
||||
Root cause: SoulSync runs two QobuzClient instances side by side — one
|
||||
through ``soulseek_client.qobuz`` for the auth-flow endpoints, and a
|
||||
through ``download_orchestrator.client('qobuz')`` for the auth-flow endpoints, and a
|
||||
second owned by the enrichment worker thread for thread safety. Login
|
||||
only updated the first instance's in-memory state. The dashboard's
|
||||
"configured" check (and the connection-test step) read the worker
|
||||
|
|
|
|||
|
|
@ -30,7 +30,7 @@ import pytest
|
|||
|
||||
from core import soundcloud_client
|
||||
from core.soundcloud_client import SoundcloudClient, _sanitize_filename
|
||||
from core.soulseek_client import AlbumResult, DownloadStatus, TrackResult
|
||||
from core.download_plugins.types import AlbumResult, DownloadStatus, TrackResult
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
|
|
@ -91,6 +91,19 @@ def tmp_dl(tmp_path: Path) -> Path:
|
|||
return p
|
||||
|
||||
|
||||
def _wire_engine(client: SoundcloudClient) -> 'DownloadEngine':
|
||||
"""Phase C7: SoundCloud no longer owns its own active_downloads
|
||||
dict — state lives on engine.DownloadEngine. Tests that
|
||||
construct a bare client must wire an engine so download() can
|
||||
dispatch and the client's query/cancel methods read from
|
||||
somewhere. Returns the engine for tests that want to inspect
|
||||
state directly."""
|
||||
from core.download_engine import DownloadEngine
|
||||
engine = DownloadEngine()
|
||||
client.set_engine(engine)
|
||||
return engine
|
||||
|
||||
|
||||
def test_is_available_when_yt_dlp_installed(tmp_dl: Path) -> None:
|
||||
client = SoundcloudClient(download_path=str(tmp_dl))
|
||||
# In our test env yt-dlp is installed (it's a hard dep)
|
||||
|
|
@ -261,9 +274,10 @@ def test_download_rejects_invalid_filename_format(tmp_dl: Path) -> None:
|
|||
|
||||
|
||||
def test_download_starts_thread_and_returns_id(tmp_dl: Path) -> None:
|
||||
"""Verify the contract: returns a download_id, populates active_downloads,
|
||||
spawns a thread that ultimately drives state to terminal."""
|
||||
"""Verify the contract: returns a download_id, engine record is
|
||||
populated, the worker drives state to terminal."""
|
||||
client = SoundcloudClient(download_path=str(tmp_dl))
|
||||
engine = _wire_engine(client)
|
||||
completed_path = tmp_dl / "track.mp3"
|
||||
completed_path.write_bytes(b"x" * (200 * 1024)) # > MIN_AUDIO_SIZE
|
||||
|
||||
|
|
@ -275,16 +289,14 @@ def test_download_starts_thread_and_returns_id(tmp_dl: Path) -> None:
|
|||
))
|
||||
|
||||
assert download_id is not None
|
||||
# Thread runs async; wait briefly for terminal state
|
||||
deadline = time.time() + 2
|
||||
while time.time() < deadline:
|
||||
with client._download_lock:
|
||||
state = client.active_downloads[download_id]['state']
|
||||
if state == 'Completed, Succeeded':
|
||||
record = engine.get_record('soundcloud', download_id)
|
||||
if record and record['state'] == 'Completed, Succeeded':
|
||||
break
|
||||
time.sleep(0.05)
|
||||
|
||||
info = client.active_downloads[download_id]
|
||||
info = engine.get_record('soundcloud', download_id)
|
||||
assert info['state'] == 'Completed, Succeeded'
|
||||
assert info['progress'] == 100.0
|
||||
assert info['file_path'] == str(completed_path)
|
||||
|
|
@ -293,6 +305,7 @@ def test_download_starts_thread_and_returns_id(tmp_dl: Path) -> None:
|
|||
|
||||
def test_download_thread_marks_failed_when_sync_returns_none(tmp_dl: Path) -> None:
|
||||
client = SoundcloudClient(download_path=str(tmp_dl))
|
||||
engine = _wire_engine(client)
|
||||
with patch.object(client, '_download_sync', return_value=None):
|
||||
download_id = _run(client.download(
|
||||
'soundcloud',
|
||||
|
|
@ -300,25 +313,25 @@ def test_download_thread_marks_failed_when_sync_returns_none(tmp_dl: Path) -> No
|
|||
))
|
||||
deadline = time.time() + 2
|
||||
while time.time() < deadline:
|
||||
with client._download_lock:
|
||||
state = client.active_downloads[download_id]['state']
|
||||
if state == 'Errored':
|
||||
record = engine.get_record('soundcloud', download_id)
|
||||
if record and record['state'] == 'Errored':
|
||||
break
|
||||
time.sleep(0.05)
|
||||
assert client.active_downloads[download_id]['state'] == 'Errored'
|
||||
assert engine.get_record('soundcloud', download_id)['state'] == 'Errored'
|
||||
|
||||
|
||||
def test_download_thread_does_not_clobber_cancelled_state(tmp_dl: Path) -> None:
|
||||
"""If a user cancels mid-download and the sync function then returns
|
||||
None, the thread should NOT overwrite the explicit Cancelled state
|
||||
with a generic Errored state."""
|
||||
None, the worker must NOT overwrite the explicit Cancelled state
|
||||
with a generic Errored state. The legacy per-client thread had
|
||||
this guard; engine.worker._mark_terminal preserves it for every
|
||||
source via a single check."""
|
||||
client = SoundcloudClient(download_path=str(tmp_dl))
|
||||
engine = _wire_engine(client)
|
||||
|
||||
def _slow_sync(download_id, *_):
|
||||
# Simulate cancellation racing a None return
|
||||
time.sleep(0.05)
|
||||
with client._download_lock:
|
||||
client.active_downloads[download_id]['state'] = 'Cancelled'
|
||||
engine.update_record('soundcloud', download_id, {'state': 'Cancelled'})
|
||||
return None
|
||||
|
||||
with patch.object(client, '_download_sync', side_effect=_slow_sync):
|
||||
|
|
@ -326,12 +339,11 @@ def test_download_thread_does_not_clobber_cancelled_state(tmp_dl: Path) -> None:
|
|||
|
||||
deadline = time.time() + 2
|
||||
while time.time() < deadline:
|
||||
with client._download_lock:
|
||||
state = client.active_downloads[download_id]['state']
|
||||
if state == 'Cancelled':
|
||||
record = engine.get_record('soundcloud', download_id)
|
||||
if record and record['state'] in ('Cancelled', 'Errored'):
|
||||
break
|
||||
time.sleep(0.05)
|
||||
assert client.active_downloads[download_id]['state'] == 'Cancelled'
|
||||
assert engine.get_record('soundcloud', download_id)['state'] == 'Cancelled'
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
|
|
@ -375,15 +387,15 @@ def test_download_sync_writes_file_and_returns_path(tmp_dl: Path, monkeypatch) -
|
|||
)
|
||||
monkeypatch.setattr(soundcloud_client, "yt_dlp", fake_yt_dlp)
|
||||
client = SoundcloudClient(download_path=str(tmp_dl))
|
||||
engine = _wire_engine(client)
|
||||
|
||||
with client._download_lock:
|
||||
client.active_downloads['dl1'] = {
|
||||
'id': 'dl1', 'filename': '', 'username': 'soundcloud',
|
||||
'state': 'Initializing', 'progress': 0.0, 'size': 0,
|
||||
'transferred': 0, 'speed': 0, 'time_remaining': None,
|
||||
'track_id': 'abc', 'permalink_url': 'u', 'display_name': 'My Track',
|
||||
'file_path': None,
|
||||
}
|
||||
engine.add_record('soundcloud', 'dl1', {
|
||||
'id': 'dl1', 'filename': '', 'username': 'soundcloud',
|
||||
'state': 'Initializing', 'progress': 0.0, 'size': 0,
|
||||
'transferred': 0, 'speed': 0, 'time_remaining': None,
|
||||
'track_id': 'abc', 'permalink_url': 'u', 'display_name': 'My Track',
|
||||
'file_path': None,
|
||||
})
|
||||
|
||||
result = client._download_sync('dl1', 'https://soundcloud.com/x/y', 'My Track')
|
||||
assert result is not None
|
||||
|
|
@ -414,15 +426,15 @@ def test_download_sync_rejects_too_small_file(tmp_dl: Path, monkeypatch) -> None
|
|||
)
|
||||
monkeypatch.setattr(soundcloud_client, "yt_dlp", fake_yt_dlp)
|
||||
client = SoundcloudClient(download_path=str(tmp_dl))
|
||||
engine = _wire_engine(client)
|
||||
|
||||
with client._download_lock:
|
||||
client.active_downloads['dl2'] = {
|
||||
'id': 'dl2', 'filename': '', 'username': 'soundcloud',
|
||||
'state': 'Initializing', 'progress': 0.0, 'size': 0,
|
||||
'transferred': 0, 'speed': 0, 'time_remaining': None,
|
||||
'track_id': 'tiny', 'permalink_url': 'u', 'display_name': 'Tiny',
|
||||
'file_path': None,
|
||||
}
|
||||
engine.add_record('soundcloud', 'dl2', {
|
||||
'id': 'dl2', 'filename': '', 'username': 'soundcloud',
|
||||
'state': 'Initializing', 'progress': 0.0, 'size': 0,
|
||||
'transferred': 0, 'speed': 0, 'time_remaining': None,
|
||||
'track_id': 'tiny', 'permalink_url': 'u', 'display_name': 'Tiny',
|
||||
'file_path': None,
|
||||
})
|
||||
result = client._download_sync('dl2', 'https://soundcloud.com/x/y', 'Tiny')
|
||||
assert result is None
|
||||
# File got cleaned up after rejection
|
||||
|
|
@ -451,13 +463,13 @@ def test_download_sync_handles_yt_dlp_raising(tmp_dl: Path, monkeypatch) -> None
|
|||
)
|
||||
monkeypatch.setattr(soundcloud_client, "yt_dlp", fake_yt_dlp)
|
||||
client = SoundcloudClient(download_path=str(tmp_dl))
|
||||
engine = _wire_engine(client)
|
||||
|
||||
with client._download_lock:
|
||||
client.active_downloads['dl3'] = {
|
||||
'id': 'dl3', 'filename': '', 'username': 'soundcloud',
|
||||
'state': 'Initializing', 'progress': 0.0, 'size': 0,
|
||||
'transferred': 0, 'speed': 0, 'time_remaining': None,
|
||||
}
|
||||
engine.add_record('soundcloud', 'dl3', {
|
||||
'id': 'dl3', 'filename': '', 'username': 'soundcloud',
|
||||
'state': 'Initializing', 'progress': 0.0, 'size': 0,
|
||||
'transferred': 0, 'speed': 0, 'time_remaining': None,
|
||||
})
|
||||
assert client._download_sync('dl3', 'https://soundcloud.com/x/y', 'Boom') is None
|
||||
|
||||
|
||||
|
|
@ -474,104 +486,96 @@ def test_download_sync_returns_none_when_yt_dlp_unavailable(tmp_dl: Path, monkey
|
|||
|
||||
def test_update_download_progress_populates_ledger(tmp_dl: Path) -> None:
|
||||
client = SoundcloudClient(download_path=str(tmp_dl))
|
||||
with client._download_lock:
|
||||
client.active_downloads['p1'] = {
|
||||
'id': 'p1', 'filename': '', 'username': 'soundcloud',
|
||||
'state': 'InProgress, Downloading', 'progress': 0.0, 'size': 0,
|
||||
'transferred': 0, 'speed': 0, 'time_remaining': None,
|
||||
}
|
||||
engine = _wire_engine(client)
|
||||
engine.add_record('soundcloud', 'p1', {
|
||||
'id': 'p1', 'filename': '', 'username': 'soundcloud',
|
||||
'state': 'InProgress, Downloading', 'progress': 0.0, 'size': 0,
|
||||
'transferred': 0, 'speed': 0, 'time_remaining': None,
|
||||
})
|
||||
speed_start = time.time() - 1.0 # 1 second ago
|
||||
client._update_download_progress('p1', downloaded=512_000, total=1_024_000,
|
||||
speed_start=speed_start)
|
||||
info = client.active_downloads['p1']
|
||||
info = engine.get_record('soundcloud', 'p1')
|
||||
assert info['transferred'] == 512_000
|
||||
assert info['size'] == 1_024_000
|
||||
# 50% complete, capped below 100
|
||||
assert 49.0 <= info['progress'] <= 51.0
|
||||
# Speed roughly 512KB/s
|
||||
assert info['speed'] > 0
|
||||
# Time remaining should be roughly 1 second
|
||||
assert info['time_remaining'] is not None
|
||||
assert 0 < info['time_remaining'] < 5
|
||||
|
||||
|
||||
def test_update_download_progress_caps_at_99_9(tmp_dl: Path) -> None:
|
||||
client = SoundcloudClient(download_path=str(tmp_dl))
|
||||
with client._download_lock:
|
||||
client.active_downloads['p2'] = {
|
||||
'id': 'p2', 'filename': '', 'username': 'soundcloud',
|
||||
'state': 'InProgress, Downloading', 'progress': 0.0, 'size': 0,
|
||||
'transferred': 0, 'speed': 0, 'time_remaining': None,
|
||||
}
|
||||
engine = _wire_engine(client)
|
||||
engine.add_record('soundcloud', 'p2', {
|
||||
'id': 'p2', 'filename': '', 'username': 'soundcloud',
|
||||
'state': 'InProgress, Downloading', 'progress': 0.0, 'size': 0,
|
||||
'transferred': 0, 'speed': 0, 'time_remaining': None,
|
||||
})
|
||||
client._update_download_progress('p2', downloaded=1_000_000,
|
||||
total=1_000_000, speed_start=time.time() - 1)
|
||||
assert client.active_downloads['p2']['progress'] == 99.9
|
||||
assert engine.get_record('soundcloud', 'p2')['progress'] == 99.9
|
||||
|
||||
|
||||
def test_update_download_progress_silently_skips_unknown_id(tmp_dl: Path) -> None:
|
||||
"""No-op if the download id isn't tracked — defensive against late hooks."""
|
||||
client = SoundcloudClient(download_path=str(tmp_dl))
|
||||
_wire_engine(client)
|
||||
# Should not raise
|
||||
client._update_download_progress('does_not_exist', 100, 1000, time.time())
|
||||
|
||||
|
||||
def test_update_download_progress_fragmented_uses_fragment_count(tmp_dl: Path) -> None:
|
||||
"""HLS-aware progress: fragment 5 of 10 → ~50%, regardless of byte
|
||||
estimate. Pins the SoundCloud HLS UX fix where byte-based progress
|
||||
stayed stuck near 0."""
|
||||
estimate."""
|
||||
client = SoundcloudClient(download_path=str(tmp_dl))
|
||||
with client._download_lock:
|
||||
client.active_downloads['hls1'] = {
|
||||
'id': 'hls1', 'filename': '', 'username': 'soundcloud',
|
||||
'state': 'InProgress, Downloading', 'progress': 0.0, 'size': 0,
|
||||
'transferred': 0, 'speed': 0, 'time_remaining': None,
|
||||
}
|
||||
engine = _wire_engine(client)
|
||||
engine.add_record('soundcloud', 'hls1', {
|
||||
'id': 'hls1', 'filename': '', 'username': 'soundcloud',
|
||||
'state': 'InProgress, Downloading', 'progress': 0.0, 'size': 0,
|
||||
'transferred': 0, 'speed': 0, 'time_remaining': None,
|
||||
})
|
||||
client._update_download_progress_fragmented(
|
||||
'hls1', downloaded=512_000, fragment_index=5, fragment_count=10,
|
||||
speed_start=time.time() - 1.0,
|
||||
)
|
||||
info = client.active_downloads['hls1']
|
||||
info = engine.get_record('soundcloud', 'hls1')
|
||||
assert 49.0 <= info['progress'] <= 51.0
|
||||
# Total size estimate extrapolates from per-fragment average
|
||||
assert info['size'] == 1_024_000 # 512000 * (10/5)
|
||||
# Time remaining computed
|
||||
assert info['time_remaining'] is not None and info['time_remaining'] > 0
|
||||
|
||||
|
||||
def test_update_download_progress_fragmented_caps_at_99_9(tmp_dl: Path) -> None:
|
||||
"""Final fragment shouldn't push percentage to 100 — outer thread
|
||||
owns the final flip. Mirrors byte-based behavior."""
|
||||
"""Final fragment shouldn't push percentage to 100."""
|
||||
client = SoundcloudClient(download_path=str(tmp_dl))
|
||||
with client._download_lock:
|
||||
client.active_downloads['hls2'] = {
|
||||
'id': 'hls2', 'filename': '', 'username': 'soundcloud',
|
||||
'state': 'InProgress, Downloading', 'progress': 0.0, 'size': 0,
|
||||
'transferred': 0, 'speed': 0, 'time_remaining': None,
|
||||
}
|
||||
engine = _wire_engine(client)
|
||||
engine.add_record('soundcloud', 'hls2', {
|
||||
'id': 'hls2', 'filename': '', 'username': 'soundcloud',
|
||||
'state': 'InProgress, Downloading', 'progress': 0.0, 'size': 0,
|
||||
'transferred': 0, 'speed': 0, 'time_remaining': None,
|
||||
})
|
||||
client._update_download_progress_fragmented(
|
||||
'hls2', downloaded=10_000_000, fragment_index=10, fragment_count=10,
|
||||
speed_start=time.time() - 5.0,
|
||||
)
|
||||
assert client.active_downloads['hls2']['progress'] == 99.9
|
||||
assert engine.get_record('soundcloud', 'hls2')['progress'] == 99.9
|
||||
|
||||
|
||||
def test_update_download_progress_fragmented_handles_zero_index(tmp_dl: Path) -> None:
|
||||
"""Defensive: yt-dlp can call the progress hook with fragment_index=0
|
||||
on the very first callback (HLS init segment). Progress is 0% and
|
||||
nothing crashes."""
|
||||
"""Defensive: fragment_index=0 on first callback → progress 0,
|
||||
no crash."""
|
||||
client = SoundcloudClient(download_path=str(tmp_dl))
|
||||
with client._download_lock:
|
||||
client.active_downloads['hls3'] = {
|
||||
'id': 'hls3', 'filename': '', 'username': 'soundcloud',
|
||||
'state': 'InProgress, Downloading', 'progress': 0.0, 'size': 0,
|
||||
'transferred': 0, 'speed': 0, 'time_remaining': None,
|
||||
}
|
||||
engine = _wire_engine(client)
|
||||
engine.add_record('soundcloud', 'hls3', {
|
||||
'id': 'hls3', 'filename': '', 'username': 'soundcloud',
|
||||
'state': 'InProgress, Downloading', 'progress': 0.0, 'size': 0,
|
||||
'transferred': 0, 'speed': 0, 'time_remaining': None,
|
||||
})
|
||||
client._update_download_progress_fragmented(
|
||||
'hls3', downloaded=0, fragment_index=0, fragment_count=10,
|
||||
speed_start=time.time(),
|
||||
)
|
||||
# No exception, progress stays 0
|
||||
assert client.active_downloads['hls3']['progress'] == 0.0
|
||||
assert engine.get_record('soundcloud', 'hls3')['progress'] == 0.0
|
||||
|
||||
|
||||
def test_update_download_progress_fragmented_silently_skips_unknown_id(tmp_dl: Path) -> None:
|
||||
|
|
@ -590,13 +594,13 @@ def test_update_download_progress_fragmented_silently_skips_unknown_id(tmp_dl: P
|
|||
|
||||
def test_get_all_downloads_returns_status_objects(tmp_dl: Path) -> None:
|
||||
client = SoundcloudClient(download_path=str(tmp_dl))
|
||||
with client._download_lock:
|
||||
client.active_downloads['s1'] = {
|
||||
'id': 's1', 'filename': 'f', 'username': 'soundcloud',
|
||||
'state': 'InProgress, Downloading', 'progress': 33.3, 'size': 1000,
|
||||
'transferred': 333, 'speed': 100, 'time_remaining': 7,
|
||||
'file_path': None,
|
||||
}
|
||||
engine = _wire_engine(client)
|
||||
engine.add_record('soundcloud', 's1', {
|
||||
'id': 's1', 'filename': 'f', 'username': 'soundcloud',
|
||||
'state': 'InProgress, Downloading', 'progress': 33.3, 'size': 1000,
|
||||
'transferred': 333, 'speed': 100, 'time_remaining': 7,
|
||||
'file_path': None,
|
||||
})
|
||||
out = _run(client.get_all_downloads())
|
||||
assert len(out) == 1
|
||||
assert isinstance(out[0], DownloadStatus)
|
||||
|
|
@ -606,54 +610,56 @@ def test_get_all_downloads_returns_status_objects(tmp_dl: Path) -> None:
|
|||
|
||||
def test_get_download_status_returns_none_for_unknown(tmp_dl: Path) -> None:
|
||||
client = SoundcloudClient(download_path=str(tmp_dl))
|
||||
_wire_engine(client)
|
||||
assert _run(client.get_download_status('nope')) is None
|
||||
|
||||
|
||||
def test_cancel_download_marks_state(tmp_dl: Path) -> None:
|
||||
client = SoundcloudClient(download_path=str(tmp_dl))
|
||||
with client._download_lock:
|
||||
client.active_downloads['c1'] = {
|
||||
'id': 'c1', 'filename': '', 'username': 'soundcloud',
|
||||
'state': 'InProgress, Downloading', 'progress': 50.0, 'size': 0,
|
||||
'transferred': 0, 'speed': 0, 'time_remaining': None,
|
||||
}
|
||||
engine = _wire_engine(client)
|
||||
engine.add_record('soundcloud', 'c1', {
|
||||
'id': 'c1', 'filename': '', 'username': 'soundcloud',
|
||||
'state': 'InProgress, Downloading', 'progress': 50.0, 'size': 0,
|
||||
'transferred': 0, 'speed': 0, 'time_remaining': None,
|
||||
})
|
||||
assert _run(client.cancel_download('c1')) is True
|
||||
assert client.active_downloads['c1']['state'] == 'Cancelled'
|
||||
assert engine.get_record('soundcloud', 'c1')['state'] == 'Cancelled'
|
||||
|
||||
|
||||
def test_cancel_download_with_remove_drops_entry(tmp_dl: Path) -> None:
|
||||
client = SoundcloudClient(download_path=str(tmp_dl))
|
||||
with client._download_lock:
|
||||
client.active_downloads['c2'] = {
|
||||
'id': 'c2', 'filename': '', 'username': 'soundcloud',
|
||||
'state': 'InProgress, Downloading', 'progress': 0.0, 'size': 0,
|
||||
'transferred': 0, 'speed': 0, 'time_remaining': None,
|
||||
}
|
||||
engine = _wire_engine(client)
|
||||
engine.add_record('soundcloud', 'c2', {
|
||||
'id': 'c2', 'filename': '', 'username': 'soundcloud',
|
||||
'state': 'InProgress, Downloading', 'progress': 0.0, 'size': 0,
|
||||
'transferred': 0, 'speed': 0, 'time_remaining': None,
|
||||
})
|
||||
assert _run(client.cancel_download('c2', remove=True)) is True
|
||||
assert 'c2' not in client.active_downloads
|
||||
assert engine.get_record('soundcloud', 'c2') is None
|
||||
|
||||
|
||||
def test_cancel_download_returns_false_for_unknown(tmp_dl: Path) -> None:
|
||||
client = SoundcloudClient(download_path=str(tmp_dl))
|
||||
_wire_engine(client)
|
||||
assert _run(client.cancel_download('not_real')) is False
|
||||
|
||||
|
||||
def test_clear_completed_drops_terminal_entries_only(tmp_dl: Path) -> None:
|
||||
"""Terminal states get cleared; in-flight downloads survive."""
|
||||
client = SoundcloudClient(download_path=str(tmp_dl))
|
||||
engine = _wire_engine(client)
|
||||
base = {'filename': '', 'username': 'soundcloud', 'progress': 0.0,
|
||||
'size': 0, 'transferred': 0, 'speed': 0, 'time_remaining': None}
|
||||
with client._download_lock:
|
||||
client.active_downloads['done'] = {**base, 'id': 'done', 'state': 'Completed, Succeeded'}
|
||||
client.active_downloads['err'] = {**base, 'id': 'err', 'state': 'Errored'}
|
||||
client.active_downloads['cnc'] = {**base, 'id': 'cnc', 'state': 'Cancelled'}
|
||||
client.active_downloads['live'] = {**base, 'id': 'live', 'state': 'InProgress, Downloading'}
|
||||
engine.add_record('soundcloud', 'done', {**base, 'id': 'done', 'state': 'Completed, Succeeded'})
|
||||
engine.add_record('soundcloud', 'err', {**base, 'id': 'err', 'state': 'Errored'})
|
||||
engine.add_record('soundcloud', 'cnc', {**base, 'id': 'cnc', 'state': 'Cancelled'})
|
||||
engine.add_record('soundcloud', 'live', {**base, 'id': 'live', 'state': 'InProgress, Downloading'})
|
||||
|
||||
assert _run(client.clear_all_completed_downloads()) is True
|
||||
assert 'done' not in client.active_downloads
|
||||
assert 'err' not in client.active_downloads
|
||||
assert 'cnc' not in client.active_downloads
|
||||
assert 'live' in client.active_downloads
|
||||
assert engine.get_record('soundcloud', 'done') is None
|
||||
assert engine.get_record('soundcloud', 'err') is None
|
||||
assert engine.get_record('soundcloud', 'cnc') is None
|
||||
assert engine.get_record('soundcloud', 'live') is not None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
|
|
@ -723,6 +729,7 @@ def test_live_download_a_known_public_track(tmp_dl: Path) -> None:
|
|||
another reliably-public free track.
|
||||
"""
|
||||
client = SoundcloudClient(download_path=str(tmp_dl))
|
||||
engine = _wire_engine(client)
|
||||
# Search-then-download flow: pick the first hit for a popular query
|
||||
tracks, _ = _run(client.search("creative commons electronic music"))
|
||||
assert tracks, "Live search returned no results"
|
||||
|
|
@ -735,7 +742,7 @@ def test_live_download_a_known_public_track(tmp_dl: Path) -> None:
|
|||
final_state = None
|
||||
final_path = None
|
||||
while time.time() < deadline:
|
||||
info = client.active_downloads.get(download_id, {})
|
||||
info = engine.get_record('soundcloud', download_id) or {}
|
||||
final_state = info.get('state')
|
||||
final_path = info.get('file_path')
|
||||
if final_state in {'Completed, Succeeded', 'Errored', 'Cancelled'}:
|
||||
|
|
|
|||
318
web_server.py
318
web_server.py
|
|
@ -88,7 +88,7 @@ from plexapi.myplex import MyPlexAccount, MyPlexPinLogin
|
|||
from core.jellyfin_client import JellyfinClient
|
||||
from core.navidrome_client import NavidromeClient
|
||||
from core.soulseek_client import SoulseekClient
|
||||
from core.download_orchestrator import DownloadOrchestrator
|
||||
from core.download_orchestrator import DownloadOrchestrator, set_download_orchestrator
|
||||
from core.tidal_client import TidalClient # Added import for Tidal
|
||||
from core.matching_engine import MusicMatchingEngine
|
||||
from core.database_update_worker import DatabaseUpdateWorker
|
||||
|
|
@ -570,7 +570,7 @@ IS_SHUTTING_DOWN = False
|
|||
# Each client is initialized independently so one failure doesn't take down everything.
|
||||
# Previously, a single exception set ALL clients to None, breaking the entire app.
|
||||
logger.info("Initializing SoulSync services for Web UI...")
|
||||
spotify_client = plex_client = jellyfin_client = navidrome_client = soulsync_library_client = soulseek_client = tidal_client = matching_engine = sync_service = web_scan_manager = None
|
||||
spotify_client = plex_client = jellyfin_client = navidrome_client = soulsync_library_client = download_orchestrator = tidal_client = matching_engine = sync_service = web_scan_manager = None
|
||||
|
||||
try:
|
||||
spotify_client = get_spotify_client()
|
||||
|
|
@ -604,7 +604,11 @@ except Exception as e:
|
|||
logger.error(f" SoulSync library client failed to initialize: {e}")
|
||||
|
||||
try:
|
||||
soulseek_client = DownloadOrchestrator()
|
||||
download_orchestrator = DownloadOrchestrator()
|
||||
# Install as the process-wide singleton so callers reaching for
|
||||
# get_download_orchestrator() see the same instance web_server.py
|
||||
# constructs at boot. Matches Cin's metadata engine pattern.
|
||||
set_download_orchestrator(download_orchestrator)
|
||||
logger.info(" Download orchestrator initialized")
|
||||
except Exception as e:
|
||||
logger.error(f" Download orchestrator failed to initialize: {e}")
|
||||
|
|
@ -622,28 +626,22 @@ except Exception as e:
|
|||
logger.error(f" Matching engine failed to initialize: {e}")
|
||||
|
||||
try:
|
||||
sync_service = PlaylistSyncService(spotify_client, plex_client, soulseek_client, jellyfin_client, navidrome_client)
|
||||
sync_service = PlaylistSyncService(spotify_client, plex_client, download_orchestrator, jellyfin_client, navidrome_client)
|
||||
logger.info(" Playlist sync service initialized")
|
||||
except Exception as e:
|
||||
logger.error(f" Playlist sync service failed to initialize: {e}")
|
||||
|
||||
# Inject shutdown check callback into YouTube and Tidal clients (avoids circular imports)
|
||||
if soulseek_client:
|
||||
if hasattr(soulseek_client, 'youtube'):
|
||||
soulseek_client.youtube.set_shutdown_check(lambda: IS_SHUTTING_DOWN)
|
||||
logger.info(" Configured YouTube client shutdown callback")
|
||||
if hasattr(soulseek_client, 'tidal'):
|
||||
soulseek_client.tidal.set_shutdown_check(lambda: IS_SHUTTING_DOWN)
|
||||
logger.info(" Configured Tidal download client shutdown callback")
|
||||
if hasattr(soulseek_client, 'qobuz'):
|
||||
soulseek_client.qobuz.set_shutdown_check(lambda: IS_SHUTTING_DOWN)
|
||||
logger.info(" Configured Qobuz client shutdown callback")
|
||||
if hasattr(soulseek_client, 'hifi'):
|
||||
soulseek_client.hifi.set_shutdown_check(lambda: IS_SHUTTING_DOWN)
|
||||
logger.info(" Configured HiFi client shutdown callback")
|
||||
if hasattr(soulseek_client, 'soundcloud') and soulseek_client.soundcloud:
|
||||
soulseek_client.soundcloud.set_shutdown_check(lambda: IS_SHUTTING_DOWN)
|
||||
logger.info(" Configured SoundCloud client shutdown callback")
|
||||
# Inject shutdown check callback into every download source that
|
||||
# accepts one. Generic dispatch via the registry — no per-source
|
||||
# attribute reaches needed.
|
||||
if download_orchestrator and hasattr(download_orchestrator, 'registry'):
|
||||
for _src_name, _src_client in download_orchestrator.registry.all_plugins():
|
||||
if _src_client is not None and hasattr(_src_client, 'set_shutdown_check'):
|
||||
try:
|
||||
_src_client.set_shutdown_check(lambda: IS_SHUTTING_DOWN)
|
||||
logger.info(" Configured %s client shutdown callback", _src_name)
|
||||
except Exception as _exc:
|
||||
logger.warning(" %s set_shutdown_check failed: %s", _src_name, _exc)
|
||||
|
||||
# Initialize web scan manager for automatic post-download scanning
|
||||
try:
|
||||
|
|
@ -1999,9 +1997,9 @@ def _register_automation_handlers():
|
|||
hybrid_order = config_manager.get('download_source.hybrid_order', ['hifi', 'youtube', 'soulseek'])
|
||||
soulseek_active = (dl_mode == 'soulseek' or
|
||||
(dl_mode == 'hybrid' and 'soulseek' in hybrid_order))
|
||||
# soulseek_client is a DownloadOrchestrator; the real client lives on
|
||||
# .soulseek. Match the getattr pattern used at the other call sites.
|
||||
slskd = getattr(soulseek_client, 'soulseek', None) if soulseek_client else None
|
||||
# Reach the underlying SoulseekClient via the orchestrator's
|
||||
# generic accessor.
|
||||
slskd = download_orchestrator.client('soulseek') if download_orchestrator else None
|
||||
if not soulseek_active or not slskd or not slskd.base_url:
|
||||
_update_automation_progress(automation_id,
|
||||
log_line='Soulseek not active — skipped', log_type='skip')
|
||||
|
|
@ -2011,7 +2009,7 @@ def _register_automation_handlers():
|
|||
log_line='Auto-clear disabled in settings', log_type='skip')
|
||||
return {'status': 'skipped'}
|
||||
try:
|
||||
success = run_async(soulseek_client.maintain_search_history_with_buffer(
|
||||
success = run_async(download_orchestrator.maintain_search_history_with_buffer(
|
||||
keep_searches=50, trigger_threshold=200
|
||||
))
|
||||
if success:
|
||||
|
|
@ -2047,7 +2045,7 @@ def _register_automation_handlers():
|
|||
log_line='Skipped — downloads active', log_type='skip')
|
||||
return {'status': 'completed'}
|
||||
|
||||
run_async(soulseek_client.clear_all_completed_downloads())
|
||||
run_async(download_orchestrator.clear_all_completed_downloads())
|
||||
if not has_post_processing:
|
||||
_sweep_empty_download_directories()
|
||||
_update_automation_progress(automation_id,
|
||||
|
|
@ -2102,7 +2100,7 @@ def _register_automation_handlers():
|
|||
log_line='Download queue: skipped (active batches)', log_type='skip')
|
||||
else:
|
||||
try:
|
||||
run_async(soulseek_client.clear_all_completed_downloads())
|
||||
run_async(download_orchestrator.clear_all_completed_downloads())
|
||||
steps.append('Download queue: cleared')
|
||||
_update_automation_progress(automation_id,
|
||||
log_line='Download queue: cleared', log_type='success')
|
||||
|
|
@ -2160,7 +2158,7 @@ def _register_automation_handlers():
|
|||
_update_automation_progress(automation_id,
|
||||
log_line='Search cleanup: disabled in settings', log_type='skip')
|
||||
else:
|
||||
run_async(soulseek_client.maintain_search_history_with_buffer(
|
||||
run_async(download_orchestrator.maintain_search_history_with_buffer(
|
||||
keep_searches=50, trigger_threshold=200
|
||||
))
|
||||
steps.append('Search history: cleaned')
|
||||
|
|
@ -2292,7 +2290,7 @@ def _register_automation_handlers():
|
|||
if automation_id:
|
||||
_update_automation_progress(automation_id,
|
||||
phase='Searching', log_line=f'Searching: {query}', log_type='info')
|
||||
result = run_async(soulseek_client.search_and_download_best(query))
|
||||
result = run_async(download_orchestrator.search_and_download_best(query))
|
||||
if result:
|
||||
if automation_id:
|
||||
_update_automation_progress(automation_id,
|
||||
|
|
@ -2354,7 +2352,7 @@ try:
|
|||
app.register_blueprint(api_bp, url_prefix='/api/v1')
|
||||
app.soulsync = {
|
||||
'spotify_client': spotify_client,
|
||||
'soulseek_client': soulseek_client,
|
||||
'download_orchestrator': download_orchestrator,
|
||||
'tidal_client': tidal_client,
|
||||
'matching_engine': matching_engine,
|
||||
'config_manager': config_manager,
|
||||
|
|
@ -2459,8 +2457,9 @@ def get_cached_transfer_data():
|
|||
|
||||
# First, get Soulseek downloads from API
|
||||
transfers_data = None
|
||||
if not soulseek_known_down and soulseek_client and getattr(soulseek_client, 'soulseek', None) and soulseek_client.soulseek.base_url:
|
||||
transfers_data = run_async(soulseek_client._make_request('GET', 'transfers/downloads'))
|
||||
_slsk = download_orchestrator.client("soulseek") if download_orchestrator else None
|
||||
if not soulseek_known_down and _slsk and _slsk.base_url:
|
||||
transfers_data = run_async(download_orchestrator._make_request('GET', 'transfers/downloads'))
|
||||
if transfers_data:
|
||||
all_transfers = []
|
||||
for user_data in transfers_data:
|
||||
|
|
@ -2475,33 +2474,35 @@ def get_cached_transfer_data():
|
|||
key = _make_context_key(transfer.get('username'), transfer.get('filename', ''))
|
||||
live_transfers_lookup[key] = transfer
|
||||
|
||||
# Also add non-Soulseek downloads (avoid redundant slskd call through orchestrator).
|
||||
# Every streaming source must appear here — task progress for in-flight
|
||||
# downloads comes from this lookup. Missing a source = task.progress
|
||||
# stays at 0 even when the underlying client knows the real percent.
|
||||
# Also add non-Soulseek downloads. Soulseek is excluded
|
||||
# because slskd's transfers endpoint was already pulled
|
||||
# above — without the exclude both fetch paths run.
|
||||
# Every streaming source must appear here — task progress
|
||||
# for in-flight downloads comes from this lookup. Missing a
|
||||
# source = task.progress stays at 0 even when the
|
||||
# underlying client knows the real percent.
|
||||
try:
|
||||
all_downloads = []
|
||||
for _dl_client in [soulseek_client.youtube, soulseek_client.tidal, soulseek_client.qobuz,
|
||||
soulseek_client.hifi, soulseek_client.deezer_dl, soulseek_client.lidarr,
|
||||
getattr(soulseek_client, 'soundcloud', None)]:
|
||||
if _dl_client:
|
||||
try:
|
||||
all_downloads.extend(run_async(_dl_client.get_all_downloads()))
|
||||
except Exception:
|
||||
pass
|
||||
if download_orchestrator and hasattr(download_orchestrator, 'engine'):
|
||||
try:
|
||||
all_downloads = run_async(
|
||||
download_orchestrator.engine.get_all_downloads(exclude=('soulseek',))
|
||||
)
|
||||
except Exception:
|
||||
pass
|
||||
for download in all_downloads:
|
||||
key = _make_context_key(download.username, download.filename)
|
||||
# Convert DownloadStatus to transfer dict format
|
||||
live_transfers_lookup[key] = {
|
||||
'id': download.id,
|
||||
'filename': download.filename,
|
||||
'username': download.username,
|
||||
'state': download.state,
|
||||
'percentComplete': download.progress,
|
||||
'size': download.size,
|
||||
'bytesTransferred': download.transferred,
|
||||
'averageSpeed': download.speed,
|
||||
}
|
||||
key = _make_context_key(download.username, download.filename)
|
||||
# Convert DownloadStatus to transfer dict format
|
||||
live_transfers_lookup[key] = {
|
||||
'id': download.id,
|
||||
'filename': download.filename,
|
||||
'username': download.username,
|
||||
'state': download.state,
|
||||
'percentComplete': download.progress,
|
||||
'size': download.size,
|
||||
'bytesTransferred': download.transferred,
|
||||
'averageSpeed': download.speed,
|
||||
}
|
||||
except Exception as e:
|
||||
logger.error(f"Could not fetch streaming source downloads: {e}")
|
||||
|
||||
|
|
@ -3059,7 +3060,7 @@ def _build_prepare_stream_deps():
|
|||
|
||||
return _streaming_prepare.PrepareStreamDeps(
|
||||
config_manager=config_manager,
|
||||
soulseek_client=soulseek_client,
|
||||
download_orchestrator=download_orchestrator,
|
||||
stream_lock=stream_lock,
|
||||
project_root=os.path.dirname(os.path.abspath(__file__)),
|
||||
docker_resolve_path=docker_resolve_path,
|
||||
|
|
@ -3506,10 +3507,10 @@ def get_status():
|
|||
if is_serverless:
|
||||
soulseek_status = True
|
||||
soulseek_response_time = 0
|
||||
elif soulseek_relevant and soulseek_client:
|
||||
elif soulseek_relevant and download_orchestrator:
|
||||
soulseek_start = time.time()
|
||||
try:
|
||||
soulseek_status = run_async(soulseek_client.check_connection())
|
||||
soulseek_status = run_async(download_orchestrator.check_connection())
|
||||
except Exception:
|
||||
soulseek_status = False
|
||||
soulseek_response_time = (time.time() - soulseek_start) * 1000
|
||||
|
|
@ -3922,7 +3923,7 @@ def _build_system_stats():
|
|||
|
||||
if soulseek_active and not soulseek_known_down:
|
||||
try:
|
||||
transfers_data = run_async(soulseek_client._make_request('GET', 'transfers/downloads'))
|
||||
transfers_data = run_async(download_orchestrator._make_request('GET', 'transfers/downloads'))
|
||||
if transfers_data:
|
||||
for user_data in transfers_data:
|
||||
if 'directories' in user_data:
|
||||
|
|
@ -4148,11 +4149,12 @@ def handle_settings():
|
|||
if navidrome_client:
|
||||
navidrome_client.reload_config()
|
||||
# Reload orchestrator settings (download source mode, hybrid_primary, etc.)
|
||||
if soulseek_client:
|
||||
soulseek_client.reload_settings()
|
||||
if download_orchestrator:
|
||||
download_orchestrator.reload_settings()
|
||||
# Reload YouTube client settings (rate limiting, cookies)
|
||||
if hasattr(soulseek_client, 'youtube'):
|
||||
soulseek_client.youtube.reload_settings()
|
||||
_yt = download_orchestrator.client("youtube")
|
||||
if _yt:
|
||||
_yt.reload_settings()
|
||||
# FIX: Re-instantiate the global tidal_client to pick up new settings
|
||||
try:
|
||||
tidal_client = TidalClient()
|
||||
|
|
@ -4184,7 +4186,7 @@ def handle_settings():
|
|||
data = dict(config_manager.config_data)
|
||||
# Include which download sources are configured so the UI can auto-disable unconfigured ones
|
||||
try:
|
||||
data['_source_status'] = soulseek_client.get_source_status()
|
||||
data['_source_status'] = download_orchestrator.get_source_status()
|
||||
except Exception:
|
||||
pass
|
||||
return jsonify(data)
|
||||
|
|
@ -6735,7 +6737,7 @@ def _build_search_deps():
|
|||
spotify_client=spotify_client,
|
||||
hydrabase_client=hydrabase_client,
|
||||
hydrabase_worker=hydrabase_worker,
|
||||
soulseek_client=soulseek_client,
|
||||
download_orchestrator=download_orchestrator,
|
||||
fix_artist_image_url=fix_artist_image_url,
|
||||
is_hydrabase_active=_is_hydrabase_active,
|
||||
get_metadata_fallback_source=_get_metadata_fallback_source,
|
||||
|
|
@ -6761,7 +6763,7 @@ def search_music():
|
|||
add_activity_item("", "Search Started", f"'{query}'", "Now")
|
||||
|
||||
try:
|
||||
results = _search_basic.run_basic_soulseek_search(query, soulseek_client, run_async)
|
||||
results = _search_basic.run_basic_soulseek_search(query, download_orchestrator, run_async)
|
||||
add_activity_item("", "Search Complete", f"'{query}' - {len(results)} results", "Now")
|
||||
return jsonify({"results": results})
|
||||
except Exception as e:
|
||||
|
|
@ -6816,7 +6818,7 @@ def enhanced_search_source(source_name):
|
|||
|
||||
When the requested source's client isn't available (Spotify unauthed,
|
||||
Discogs missing token, Hydrabase disconnected, MusicBrainz import
|
||||
failure, soulseek_client.youtube missing), returns plain JSON
|
||||
failure, download_orchestrator.client("youtube") missing), returns plain JSON
|
||||
`{"artists":[],"albums":[],"tracks":[],"available":false}` to match
|
||||
the original endpoint contract.
|
||||
"""
|
||||
|
|
@ -6901,7 +6903,7 @@ def stream_enhanced_search_track():
|
|||
album_name=album_name,
|
||||
duration_ms=duration_ms,
|
||||
config_manager=config_manager,
|
||||
soulseek_client=soulseek_client,
|
||||
download_orchestrator=download_orchestrator,
|
||||
matching_engine=matching_engine,
|
||||
run_async=run_async,
|
||||
)
|
||||
|
|
@ -7040,7 +7042,7 @@ def download_music_video():
|
|||
def _progress(pct):
|
||||
_music_video_downloads[video_id]['progress'] = round(pct, 1)
|
||||
|
||||
final_path = soulseek_client.youtube.download_music_video(video_url, output_path, progress_callback=_progress)
|
||||
final_path = download_orchestrator.client("youtube").download_music_video(video_url, output_path, progress_callback=_progress)
|
||||
|
||||
if final_path and os.path.exists(final_path):
|
||||
_music_video_downloads[video_id]['status'] = 'completed'
|
||||
|
|
@ -7099,7 +7101,7 @@ def start_download():
|
|||
filename = track_data.get('filename')
|
||||
file_size = track_data.get('size', 0)
|
||||
|
||||
download_id = run_async(soulseek_client.download(
|
||||
download_id = run_async(download_orchestrator.download(
|
||||
username,
|
||||
filename,
|
||||
file_size
|
||||
|
|
@ -7148,7 +7150,7 @@ def start_download():
|
|||
if not username or not filename:
|
||||
return jsonify({"error": "Missing username or filename."}), 400
|
||||
|
||||
download_id = run_async(soulseek_client.download(username, filename, file_size))
|
||||
download_id = run_async(download_orchestrator.download(username, filename, file_size))
|
||||
logger.info(f"Download ID returned: {download_id}")
|
||||
|
||||
if download_id:
|
||||
|
|
@ -7344,7 +7346,7 @@ def get_download_status():
|
|||
A robust status checker that correctly finds completed files by searching
|
||||
the entire download directory with fuzzy matching, mirroring the logic from downloads.py.
|
||||
"""
|
||||
if not soulseek_client:
|
||||
if not download_orchestrator:
|
||||
return jsonify({"transfers": []})
|
||||
|
||||
try:
|
||||
|
|
@ -7353,7 +7355,7 @@ def get_download_status():
|
|||
soulseek_known_down = not _status_cache.get('soulseek', {}).get('connected', True)
|
||||
transfers_data = None
|
||||
if not soulseek_known_down:
|
||||
transfers_data = run_async(soulseek_client._make_request('GET', 'transfers/downloads'))
|
||||
transfers_data = run_async(download_orchestrator._make_request('GET', 'transfers/downloads'))
|
||||
|
||||
# Don't return early if no Soulseek transfers - YouTube/Tidal downloads need to be checked too!
|
||||
all_transfers = []
|
||||
|
|
@ -7421,7 +7423,7 @@ def get_download_status():
|
|||
transfer_id = file_info.get('id')
|
||||
if transfer_id:
|
||||
try:
|
||||
run_async(soulseek_client.cancel_download(str(transfer_id), username, remove=True))
|
||||
run_async(download_orchestrator.cancel_download(str(transfer_id), username, remove=True))
|
||||
except Exception:
|
||||
pass
|
||||
_orphaned_download_keys.discard(context_key)
|
||||
|
|
@ -7533,7 +7535,7 @@ def get_download_status():
|
|||
|
||||
# Also include YouTube/Tidal downloads in the response
|
||||
try:
|
||||
all_streaming_downloads = run_async(soulseek_client.get_all_downloads())
|
||||
all_streaming_downloads = run_async(download_orchestrator.get_all_downloads())
|
||||
|
||||
for download in all_streaming_downloads:
|
||||
if download.username in ('youtube', 'tidal', 'qobuz', 'hifi', 'deezer_dl', 'lidarr', 'soundcloud'):
|
||||
|
|
@ -7665,7 +7667,7 @@ def cancel_download():
|
|||
return jsonify({"success": False, "error": "Missing download_id or username."}), 400
|
||||
|
||||
try:
|
||||
success = _downloads_cancel.cancel_single_download(soulseek_client, run_async, download_id, username)
|
||||
success = _downloads_cancel.cancel_single_download(download_orchestrator, run_async, download_id, username)
|
||||
if success:
|
||||
return jsonify({"success": True, "message": "Download cancelled."})
|
||||
return jsonify({"success": False, "error": "Failed to cancel download via slskd."}), 500
|
||||
|
|
@ -7679,7 +7681,7 @@ def cancel_all_downloads():
|
|||
"""Cancel all active downloads from slskd, then clear completed ones."""
|
||||
try:
|
||||
success, msg = _downloads_cancel.cancel_all_active(
|
||||
soulseek_client, run_async, _sweep_empty_download_directories,
|
||||
download_orchestrator, run_async, _sweep_empty_download_directories,
|
||||
)
|
||||
if success:
|
||||
return jsonify({"success": True, "message": msg})
|
||||
|
|
@ -7694,7 +7696,7 @@ def clear_finished_downloads():
|
|||
"""Clear all terminal (completed, cancelled, failed) downloads from slskd."""
|
||||
try:
|
||||
success = _downloads_cancel.clear_finished_active(
|
||||
soulseek_client, run_async, _sweep_empty_download_directories,
|
||||
download_orchestrator, run_async, _sweep_empty_download_directories,
|
||||
)
|
||||
if success:
|
||||
return jsonify({"success": True, "message": "Finished downloads cleared."})
|
||||
|
|
@ -7803,7 +7805,7 @@ def download_selected_candidate(task_id):
|
|||
batch['active_count'] = batch.get('active_count', 0) + 1
|
||||
|
||||
# Build a TrackResult-like candidate object
|
||||
from core.soulseek_client import TrackResult
|
||||
from core.download_plugins.types import TrackResult
|
||||
candidate = TrackResult(
|
||||
username=username,
|
||||
filename=filename,
|
||||
|
|
@ -8081,7 +8083,7 @@ def clear_all_searches():
|
|||
Clear all searches from slskd search history.
|
||||
"""
|
||||
try:
|
||||
success = run_async(soulseek_client.clear_all_searches())
|
||||
success = run_async(download_orchestrator.clear_all_searches())
|
||||
if success:
|
||||
add_activity_item("", "Search Cleanup", "All search history cleared manually", "Now")
|
||||
return jsonify({"success": True, "message": "All searches cleared."})
|
||||
|
|
@ -8101,7 +8103,7 @@ def maintain_search_history():
|
|||
keep_searches = data.get('keep_searches', 50)
|
||||
trigger_threshold = data.get('trigger_threshold', 200)
|
||||
|
||||
success = run_async(soulseek_client.maintain_search_history_with_buffer(
|
||||
success = run_async(download_orchestrator.maintain_search_history_with_buffer(
|
||||
keep_searches=keep_searches, trigger_threshold=trigger_threshold
|
||||
))
|
||||
if success:
|
||||
|
|
@ -11678,34 +11680,19 @@ def redownload_search_sources(track_id):
|
|||
candidates = []
|
||||
database = get_database()
|
||||
|
||||
# Get all available download source clients
|
||||
# Get all available download source clients via the orchestrator's
|
||||
# generic accessor — replaces the old per-source if/hasattr chain
|
||||
# that Cin called out as defeating the purpose of the registry refactor.
|
||||
download_clients = {}
|
||||
try:
|
||||
orch = soulseek_client # The download orchestrator
|
||||
if hasattr(orch, 'soulseek') and orch.soulseek:
|
||||
if not (hasattr(orch.soulseek, 'is_configured') and not orch.soulseek.is_configured()):
|
||||
download_clients['soulseek'] = orch.soulseek
|
||||
if hasattr(orch, 'youtube') and orch.youtube:
|
||||
if not (hasattr(orch.youtube, 'is_configured') and not orch.youtube.is_configured()):
|
||||
download_clients['youtube'] = orch.youtube
|
||||
if hasattr(orch, 'tidal') and orch.tidal:
|
||||
if not (hasattr(orch.tidal, 'is_configured') and not orch.tidal.is_configured()):
|
||||
download_clients['tidal'] = orch.tidal
|
||||
if hasattr(orch, 'qobuz') and orch.qobuz:
|
||||
if not (hasattr(orch.qobuz, 'is_configured') and not orch.qobuz.is_configured()):
|
||||
download_clients['qobuz'] = orch.qobuz
|
||||
if hasattr(orch, 'hifi') and orch.hifi:
|
||||
if not (hasattr(orch.hifi, 'is_configured') and not orch.hifi.is_configured()):
|
||||
download_clients['hifi'] = orch.hifi
|
||||
if hasattr(orch, 'deezer_dl') and orch.deezer_dl:
|
||||
if not (hasattr(orch.deezer_dl, 'is_configured') and not orch.deezer_dl.is_configured()):
|
||||
download_clients['deezer_dl'] = orch.deezer_dl
|
||||
if download_orchestrator and hasattr(download_orchestrator, 'configured_clients'):
|
||||
download_clients = dict(download_orchestrator.configured_clients())
|
||||
except Exception as e:
|
||||
logger.warning(f"[Redownload] Error getting download clients: {e}")
|
||||
|
||||
if not download_clients:
|
||||
# Fallback: use orchestrator directly
|
||||
download_clients = {'default': soulseek_client}
|
||||
download_clients = {'default': download_orchestrator}
|
||||
|
||||
logger.info(f"[Redownload] Streaming search across {len(download_clients)} sources: {list(download_clients.keys())}")
|
||||
|
||||
|
|
@ -12667,7 +12654,7 @@ def _start_enhanced_album_download(enhanced_tracks, unmatched_tracks, spotify_ar
|
|||
continue
|
||||
|
||||
# Start download
|
||||
download_id = run_async(soulseek_client.download(username, filename, size))
|
||||
download_id = run_async(download_orchestrator.download(username, filename, size))
|
||||
|
||||
if download_id:
|
||||
context_key = _make_context_key(username, filename)
|
||||
|
|
@ -12715,7 +12702,7 @@ def _start_enhanced_album_download(enhanced_tracks, unmatched_tracks, spotify_ar
|
|||
if not username or not filename:
|
||||
continue
|
||||
|
||||
download_id = run_async(soulseek_client.download(username, filename, size))
|
||||
download_id = run_async(download_orchestrator.download(username, filename, size))
|
||||
|
||||
if download_id:
|
||||
context_key = _make_context_key(username, filename)
|
||||
|
|
@ -12823,7 +12810,7 @@ def _start_album_download_tasks(album_result, spotify_artist, spotify_album):
|
|||
'disc_number': corrected_meta.get('disc_number', 1)
|
||||
}
|
||||
|
||||
download_id = run_async(soulseek_client.download(username, filename, size))
|
||||
download_id = run_async(download_orchestrator.download(username, filename, size))
|
||||
|
||||
if download_id:
|
||||
context_key = _make_context_key(username, filename)
|
||||
|
|
@ -12890,7 +12877,7 @@ def start_matched_download():
|
|||
if not username or not filename:
|
||||
return jsonify({"success": False, "error": "Missing username or filename"}), 400
|
||||
|
||||
download_id = run_async(soulseek_client.download(username, filename, size))
|
||||
download_id = run_async(download_orchestrator.download(username, filename, size))
|
||||
|
||||
if download_id:
|
||||
context_key = _make_context_key(username, filename)
|
||||
|
|
@ -12950,7 +12937,7 @@ def start_matched_download():
|
|||
download_payload['title'] = parsed_meta.get('title') or download_payload.get('title')
|
||||
download_payload['artist'] = parsed_meta.get('artist') or download_payload.get('artist')
|
||||
|
||||
download_id = run_async(soulseek_client.download(username, filename, size))
|
||||
download_id = run_async(download_orchestrator.download(username, filename, size))
|
||||
|
||||
if download_id:
|
||||
context_key = _make_context_key(username, filename)
|
||||
|
|
@ -12979,7 +12966,7 @@ def start_matched_download():
|
|||
|
||||
def _parse_filename_metadata(filename: str) -> dict:
|
||||
"""
|
||||
A direct port of the metadata parsing logic from the GUI's soulseek_client.py.
|
||||
A direct port of the metadata parsing logic from the GUI's download_orchestrator.py.
|
||||
This is the crucial missing step that cleans filenames BEFORE Spotify matching.
|
||||
"""
|
||||
return parse_filename_metadata(filename)
|
||||
|
|
@ -14155,7 +14142,7 @@ def _enhance_file_metadata(file_path: str, context: dict, artist: dict, album_in
|
|||
genius_worker=genius_worker,
|
||||
spotify_enrichment_worker=spotify_enrichment_worker,
|
||||
itunes_enrichment_worker=itunes_enrichment_worker,
|
||||
hifi_client=soulseek_client.hifi if soulseek_client else None,
|
||||
hifi_client=download_orchestrator.client("hifi") if download_orchestrator else None,
|
||||
),
|
||||
)
|
||||
|
||||
|
|
@ -14260,7 +14247,7 @@ def _post_process_matched_download_with_verification(context_key, context, file_
|
|||
genius_worker=genius_worker,
|
||||
spotify_enrichment_worker=spotify_enrichment_worker,
|
||||
itunes_enrichment_worker=itunes_enrichment_worker,
|
||||
hifi_client=soulseek_client.hifi if soulseek_client else None,
|
||||
hifi_client=download_orchestrator.client("hifi") if download_orchestrator else None,
|
||||
),
|
||||
)
|
||||
|
||||
|
|
@ -14384,7 +14371,7 @@ def _post_process_matched_download(context_key, context, file_path):
|
|||
genius_worker=genius_worker,
|
||||
spotify_enrichment_worker=spotify_enrichment_worker,
|
||||
itunes_enrichment_worker=itunes_enrichment_worker,
|
||||
hifi_client=soulseek_client.hifi if soulseek_client else None,
|
||||
hifi_client=download_orchestrator.client("hifi") if download_orchestrator else None,
|
||||
),
|
||||
)
|
||||
|
||||
|
|
@ -16828,7 +16815,7 @@ def _build_master_deps():
|
|||
|
||||
return _downloads_master.MasterDeps(
|
||||
config_manager=config_manager,
|
||||
soulseek_client=soulseek_client,
|
||||
download_orchestrator=download_orchestrator,
|
||||
run_async=run_async,
|
||||
mb_worker=mb_worker,
|
||||
mb_release_cache=mb_release_cache,
|
||||
|
|
@ -16866,7 +16853,7 @@ def _build_post_processing_deps():
|
|||
"""Build the PostProcessDeps bundle from web_server.py globals on each call."""
|
||||
return _downloads_post_processing.PostProcessDeps(
|
||||
config_manager=config_manager,
|
||||
soulseek_client=soulseek_client,
|
||||
download_orchestrator=download_orchestrator,
|
||||
run_async=run_async,
|
||||
docker_resolve_path=docker_resolve_path,
|
||||
extract_filename=extract_filename,
|
||||
|
|
@ -16893,7 +16880,7 @@ from core.downloads import task_worker as _downloads_task_worker
|
|||
def _build_task_worker_deps():
|
||||
"""Build TaskWorkerDeps bundle from web_server.py globals on each call."""
|
||||
return _downloads_task_worker.TaskWorkerDeps(
|
||||
soulseek_client=soulseek_client,
|
||||
download_orchestrator=download_orchestrator,
|
||||
matching_engine=matching_engine,
|
||||
run_async=run_async,
|
||||
try_source_reuse=_try_source_reuse,
|
||||
|
|
@ -16919,7 +16906,7 @@ from core.downloads import candidates as _downloads_candidates
|
|||
def _build_candidates_deps():
|
||||
"""Build the CandidatesDeps bundle from web_server.py globals on each call."""
|
||||
return _downloads_candidates.CandidatesDeps(
|
||||
soulseek_client=soulseek_client,
|
||||
download_orchestrator=download_orchestrator,
|
||||
spotify_client=spotify_client,
|
||||
run_async=run_async,
|
||||
get_database=get_database,
|
||||
|
|
@ -17084,7 +17071,7 @@ def _try_source_reuse(task_id, batch_id, track):
|
|||
# Sort by confidence, filter by quality preference
|
||||
candidates.sort(key=lambda c: c.confidence, reverse=True)
|
||||
_sr.info(f"Found {len(candidates)} candidates above 0.70, best={candidates[0].confidence:.3f} ({candidates[0].filename})")
|
||||
slsk = soulseek_client.soulseek if hasattr(soulseek_client, 'soulseek') else soulseek_client
|
||||
slsk = download_orchestrator.client("soulseek") if hasattr(download_orchestrator, 'client') else download_orchestrator
|
||||
filtered = slsk.filter_results_by_quality_preference(candidates)
|
||||
if not filtered:
|
||||
_sr.info(f"Quality filter rejected all candidates for task {task_id}")
|
||||
|
|
@ -17163,8 +17150,8 @@ def _store_batch_source(batch_id, username, filename):
|
|||
return
|
||||
|
||||
try:
|
||||
# Access SoulseekClient directly (soulseek_client is DownloadOrchestrator)
|
||||
slsk = soulseek_client.soulseek if hasattr(soulseek_client, 'soulseek') else soulseek_client
|
||||
# Access SoulseekClient directly (download_orchestrator is DownloadOrchestrator)
|
||||
slsk = download_orchestrator.client("soulseek") if hasattr(download_orchestrator, 'client') else download_orchestrator
|
||||
_sr.info(f"Browsing {username}:{folder_path}...")
|
||||
files = run_async(slsk.browse_user_directory(username, folder_path))
|
||||
if not files:
|
||||
|
|
@ -17495,7 +17482,7 @@ def cancel_download_task():
|
|||
if download_id and username:
|
||||
try:
|
||||
# This is an async call, so we run it and wait
|
||||
run_async(soulseek_client.cancel_download(download_id, username, remove=True))
|
||||
run_async(download_orchestrator.cancel_download(download_id, username, remove=True))
|
||||
logger.warning(f"Successfully cancelled Soulseek download {download_id} for task {task_id}")
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to cancel download on slskd, but worker already moved on: {e}")
|
||||
|
|
@ -17740,14 +17727,14 @@ def cancel_task_v2():
|
|||
# username: youtube/tidal/qobuz/hifi/deezer_dl/lidarr go to
|
||||
# their streaming clients, anything else goes to Soulseek.
|
||||
#
|
||||
# Replaces an older block that assumed soulseek_client was a
|
||||
# Replaces an older block that assumed download_orchestrator was a
|
||||
# raw SoulseekClient and accessed .base_url / ._make_request
|
||||
# directly — crashed with AttributeError on the orchestrator
|
||||
# and silently left streaming downloads running in background.
|
||||
try:
|
||||
logger.info(f"[Atomic Cancel] Dispatching cancel to orchestrator: username={username} download_id={download_id}")
|
||||
cancel_success = run_async(
|
||||
soulseek_client.cancel_download(download_id, username, remove=True)
|
||||
download_orchestrator.cancel_download(download_id, username, remove=True)
|
||||
)
|
||||
if cancel_success:
|
||||
logger.info(f"[Atomic Cancel] Orchestrator cancelled download: {download_id}")
|
||||
|
|
@ -19732,7 +19719,7 @@ def get_discover_album(source, album_id):
|
|||
def hifi_status():
|
||||
"""Check if HiFi API instances are reachable."""
|
||||
try:
|
||||
hifi = soulseek_client.hifi
|
||||
hifi = download_orchestrator.client("hifi")
|
||||
available = hifi.is_available()
|
||||
version = hifi.get_version() if available else None
|
||||
return jsonify({
|
||||
|
|
@ -19755,9 +19742,7 @@ def soundcloud_status():
|
|||
of just verifying the import succeeded.
|
||||
"""
|
||||
try:
|
||||
sc = None
|
||||
if soulseek_client and hasattr(soulseek_client, 'soundcloud'):
|
||||
sc = soulseek_client.soundcloud
|
||||
sc = download_orchestrator.client("soundcloud") if download_orchestrator and hasattr(download_orchestrator, 'client') else None
|
||||
if not sc:
|
||||
return jsonify({
|
||||
"available": False,
|
||||
|
|
@ -19785,7 +19770,7 @@ def hifi_instances():
|
|||
"""Check availability of all HiFi API instances."""
|
||||
import requests as req
|
||||
try:
|
||||
hifi = soulseek_client.hifi
|
||||
hifi = download_orchestrator.client("hifi")
|
||||
instances = list(hifi._instances)
|
||||
results = []
|
||||
for url in instances:
|
||||
|
|
@ -19840,9 +19825,9 @@ def hifi_add_instance():
|
|||
added = db.add_hifi_instance(url, priority)
|
||||
if not added:
|
||||
return jsonify({'success': False, 'error': 'Instance already exists'}), 400
|
||||
# Reload the client
|
||||
if soulseek_client and hasattr(soulseek_client, 'hifi') and soulseek_client.hifi:
|
||||
soulseek_client.hifi.reload_instances()
|
||||
# Reload the HiFi client
|
||||
if download_orchestrator:
|
||||
download_orchestrator.reload_instances('hifi')
|
||||
return jsonify({'success': True, 'url': url})
|
||||
except Exception as e:
|
||||
logger.error(f"Error adding HiFi instance: {e}")
|
||||
|
|
@ -19862,9 +19847,9 @@ def hifi_remove_instance():
|
|||
removed = db.remove_hifi_instance(url)
|
||||
if not removed:
|
||||
return jsonify({'success': False, 'error': 'Instance not found'}), 404
|
||||
# Reload the client
|
||||
if soulseek_client and hasattr(soulseek_client, 'hifi') and soulseek_client.hifi:
|
||||
soulseek_client.hifi.reload_instances()
|
||||
# Reload the HiFi client
|
||||
if download_orchestrator:
|
||||
download_orchestrator.reload_instances('hifi')
|
||||
return jsonify({'success': True, 'url': url})
|
||||
except Exception as e:
|
||||
logger.error(f"Error removing HiFi instance: {e}")
|
||||
|
|
@ -19884,8 +19869,8 @@ def hifi_toggle_instance():
|
|||
from database.music_database import get_database
|
||||
db = get_database()
|
||||
db.toggle_hifi_instance(url, enabled)
|
||||
if soulseek_client and hasattr(soulseek_client, 'hifi') and soulseek_client.hifi:
|
||||
soulseek_client.hifi.reload_instances()
|
||||
if download_orchestrator:
|
||||
download_orchestrator.reload_instances('hifi')
|
||||
return jsonify({'success': True})
|
||||
except Exception as e:
|
||||
logger.error(f"Error toggling HiFi instance: {e}")
|
||||
|
|
@ -19905,9 +19890,9 @@ def hifi_reorder_instances():
|
|||
db = get_database()
|
||||
if not db.reorder_hifi_instances(urls):
|
||||
return jsonify({'success': False, 'error': 'One or more URLs not found'}), 400
|
||||
# Reload the client
|
||||
if soulseek_client and hasattr(soulseek_client, 'hifi') and soulseek_client.hifi:
|
||||
soulseek_client.hifi.reload_instances()
|
||||
# Reload the HiFi client
|
||||
if download_orchestrator:
|
||||
download_orchestrator.reload_instances('hifi')
|
||||
return jsonify({'success': True})
|
||||
except Exception as e:
|
||||
logger.error(f"Error reordering HiFi instances: {e}")
|
||||
|
|
@ -20059,11 +20044,12 @@ def deezer_download_test_download():
|
|||
|
||||
def _get_tidal_download_client():
|
||||
"""Get Tidal download client from the orchestrator, with helpful error if unavailable."""
|
||||
if not soulseek_client:
|
||||
if not download_orchestrator:
|
||||
raise RuntimeError("Download orchestrator not initialized — check startup logs for errors")
|
||||
if not hasattr(soulseek_client, 'tidal') or not soulseek_client.tidal:
|
||||
tidal = download_orchestrator.client("tidal") if hasattr(download_orchestrator, 'client') else None
|
||||
if not tidal:
|
||||
raise RuntimeError("Tidal download client not available — ensure tidalapi is installed")
|
||||
return soulseek_client.tidal
|
||||
return tidal
|
||||
|
||||
@app.route('/api/tidal/download/auth/start', methods=['POST'])
|
||||
def tidal_download_auth_start():
|
||||
|
|
@ -20132,7 +20118,7 @@ def qobuz_auth_login():
|
|||
if not email or not password:
|
||||
return jsonify({"success": False, "error": "Email and password required"}), 400
|
||||
|
||||
qobuz = soulseek_client.qobuz
|
||||
qobuz = download_orchestrator.client("qobuz")
|
||||
result = qobuz.login(email, password)
|
||||
|
||||
if result['status'] == 'success':
|
||||
|
|
@ -20155,7 +20141,7 @@ def qobuz_auth_token():
|
|||
if not token:
|
||||
return jsonify({"success": False, "error": "Auth token required"}), 400
|
||||
|
||||
qobuz = soulseek_client.qobuz
|
||||
qobuz = download_orchestrator.client("qobuz")
|
||||
result = qobuz.login_with_token(token)
|
||||
|
||||
if result['status'] == 'success':
|
||||
|
|
@ -20172,7 +20158,7 @@ def qobuz_auth_token():
|
|||
def qobuz_auth_status():
|
||||
"""Check if Qobuz client is authenticated."""
|
||||
try:
|
||||
qobuz = soulseek_client.qobuz
|
||||
qobuz = download_orchestrator.client("qobuz")
|
||||
authenticated = qobuz.is_authenticated()
|
||||
user_info = {}
|
||||
if authenticated and qobuz.user_info:
|
||||
|
|
@ -20189,7 +20175,7 @@ def qobuz_auth_status():
|
|||
def qobuz_auth_logout():
|
||||
"""Logout from Qobuz."""
|
||||
try:
|
||||
soulseek_client.qobuz.logout()
|
||||
download_orchestrator.client("qobuz").logout()
|
||||
_sync_qobuz_credentials_to_worker()
|
||||
return jsonify({"success": True})
|
||||
except Exception as e:
|
||||
|
|
@ -21131,9 +21117,7 @@ def _get_metadata_fallback_client():
|
|||
def get_deezer_arl_status():
|
||||
"""Check if Deezer ARL is configured and authenticated."""
|
||||
try:
|
||||
deezer_dl = None
|
||||
if soulseek_client and hasattr(soulseek_client, 'deezer_dl') and soulseek_client.deezer_dl:
|
||||
deezer_dl = soulseek_client.deezer_dl
|
||||
deezer_dl = download_orchestrator.client("deezer_dl") if download_orchestrator and hasattr(download_orchestrator, 'client') else None
|
||||
if deezer_dl and deezer_dl.is_authenticated():
|
||||
user_data = deezer_dl._user_data or {}
|
||||
return jsonify({
|
||||
|
|
@ -21150,9 +21134,7 @@ def get_deezer_arl_status():
|
|||
def get_deezer_arl_playlists():
|
||||
"""Fetch user playlists via Deezer ARL authentication (like /api/spotify/playlists)."""
|
||||
try:
|
||||
deezer_dl = None
|
||||
if soulseek_client and hasattr(soulseek_client, 'deezer_dl') and soulseek_client.deezer_dl:
|
||||
deezer_dl = soulseek_client.deezer_dl
|
||||
deezer_dl = download_orchestrator.client("deezer_dl") if download_orchestrator and hasattr(download_orchestrator, 'client') else None
|
||||
if not deezer_dl or not deezer_dl.is_authenticated():
|
||||
return jsonify({'error': 'Deezer ARL not authenticated. Configure your ARL token in Settings > Downloads.'}), 401
|
||||
|
||||
|
|
@ -21180,9 +21162,7 @@ def get_deezer_arl_playlists():
|
|||
def get_deezer_arl_playlist_tracks(playlist_id):
|
||||
"""Fetch full playlist with tracks via ARL (like /api/spotify/playlist/<id>)."""
|
||||
try:
|
||||
deezer_dl = None
|
||||
if soulseek_client and hasattr(soulseek_client, 'deezer_dl') and soulseek_client.deezer_dl:
|
||||
deezer_dl = soulseek_client.deezer_dl
|
||||
deezer_dl = download_orchestrator.client("deezer_dl") if download_orchestrator and hasattr(download_orchestrator, 'client') else None
|
||||
if not deezer_dl or not deezer_dl.is_authenticated():
|
||||
return jsonify({'error': 'Deezer ARL not authenticated.'}), 401
|
||||
|
||||
|
|
@ -27374,8 +27354,8 @@ def get_your_artists_sources():
|
|||
try:
|
||||
deezer_cl = _get_deezer_client()
|
||||
deezer_oauth = deezer_cl and hasattr(deezer_cl, 'is_user_authenticated') and deezer_cl.is_user_authenticated()
|
||||
deezer_arl = (hasattr(soulseek_client, 'deezer_dl') and soulseek_client.deezer_dl
|
||||
and soulseek_client.deezer_dl.is_authenticated())
|
||||
deezer_arl = (hasattr(download_orchestrator, 'client') and download_orchestrator.client("deezer_dl")
|
||||
and download_orchestrator.client("deezer_dl").is_authenticated())
|
||||
if deezer_oauth or deezer_arl:
|
||||
connected.append('deezer')
|
||||
except Exception:
|
||||
|
|
@ -27496,10 +27476,10 @@ def _fetch_and_match_liked_artists(profile_id: int):
|
|||
if deezer_cl and hasattr(deezer_cl, 'is_user_authenticated') and deezer_cl.is_user_authenticated():
|
||||
logger.info("[Your Artists] Fetching favorite artists from Deezer (OAuth)...")
|
||||
artists = deezer_cl.get_user_favorite_artists(limit=200)
|
||||
elif (hasattr(soulseek_client, 'deezer_dl') and soulseek_client.deezer_dl
|
||||
and soulseek_client.deezer_dl.is_authenticated()):
|
||||
elif (hasattr(download_orchestrator, 'client') and download_orchestrator.client("deezer_dl")
|
||||
and download_orchestrator.client("deezer_dl").is_authenticated()):
|
||||
logger.info("[Your Artists] Fetching favorite artists from Deezer (ARL)...")
|
||||
artists = soulseek_client.deezer_dl.get_user_favorite_artists(limit=200)
|
||||
artists = download_orchestrator.client("deezer_dl").get_user_favorite_artists(limit=200)
|
||||
for a in artists:
|
||||
database.upsert_liked_artist(
|
||||
artist_name=a['name'], source_service='deezer',
|
||||
|
|
@ -27646,8 +27626,8 @@ def get_your_albums_sources():
|
|||
try:
|
||||
deezer_cl = _get_deezer_client()
|
||||
deezer_oauth = deezer_cl and hasattr(deezer_cl, 'is_user_authenticated') and deezer_cl.is_user_authenticated()
|
||||
deezer_arl = (hasattr(soulseek_client, 'deezer_dl') and soulseek_client.deezer_dl
|
||||
and soulseek_client.deezer_dl.is_authenticated())
|
||||
deezer_arl = (hasattr(download_orchestrator, 'client') and download_orchestrator.client("deezer_dl")
|
||||
and download_orchestrator.client("deezer_dl").is_authenticated())
|
||||
if deezer_oauth or deezer_arl:
|
||||
connected.append('deezer')
|
||||
except Exception:
|
||||
|
|
@ -27754,10 +27734,10 @@ def _fetch_liked_albums(profile_id: int):
|
|||
if deezer_cl and hasattr(deezer_cl, 'is_user_authenticated') and deezer_cl.is_user_authenticated():
|
||||
logger.info("[Your Albums] Fetching favorite albums from Deezer (OAuth)...")
|
||||
albums = deezer_cl.get_user_favorite_albums(limit=500)
|
||||
elif (hasattr(soulseek_client, 'deezer_dl') and soulseek_client.deezer_dl
|
||||
and soulseek_client.deezer_dl.is_authenticated()):
|
||||
elif (hasattr(download_orchestrator, 'client') and download_orchestrator.client("deezer_dl")
|
||||
and download_orchestrator.client("deezer_dl").is_authenticated()):
|
||||
logger.info("[Your Albums] Fetching favorite albums from Deezer (ARL)...")
|
||||
albums = soulseek_client.deezer_dl.get_user_favorite_albums(limit=500)
|
||||
albums = download_orchestrator.client("deezer_dl").get_user_favorite_albums(limit=500)
|
||||
for a in albums:
|
||||
database.upsert_liked_album(
|
||||
album_name=a['album_name'], artist_name=a['artist_name'],
|
||||
|
|
@ -32678,7 +32658,7 @@ register_runtime_clients(
|
|||
)
|
||||
|
||||
_init_connection_test(
|
||||
soulseek_client_obj=soulseek_client,
|
||||
download_orchestrator_obj=download_orchestrator,
|
||||
qobuz_worker=qobuz_enrichment_worker,
|
||||
hydrabase_client_obj=hydrabase_client,
|
||||
docker_resolve_url_fn=docker_resolve_url,
|
||||
|
|
@ -32691,12 +32671,12 @@ _init_discover_hero(get_metadata_fallback_client_fn=_get_metadata_fallback_clien
|
|||
|
||||
_init_download_validation(
|
||||
matching_engine_obj=matching_engine,
|
||||
soulseek_client_obj=soulseek_client,
|
||||
download_orchestrator_obj=download_orchestrator,
|
||||
)
|
||||
|
||||
_init_wishlist_failed(
|
||||
engine=automation_engine,
|
||||
soulseek_client_obj=soulseek_client,
|
||||
download_orchestrator_obj=download_orchestrator,
|
||||
sweep_fn=_sweep_empty_download_directories,
|
||||
)
|
||||
|
||||
|
|
@ -32715,7 +32695,7 @@ _init_debug_info(
|
|||
sync_states_dict=sync_states,
|
||||
youtube_playlist_states_dict=youtube_playlist_states,
|
||||
tidal_discovery_states_dict=tidal_discovery_states,
|
||||
soulseek_client_obj=soulseek_client,
|
||||
download_orchestrator_obj=download_orchestrator,
|
||||
log_path=_log_path,
|
||||
log_dir=_log_dir,
|
||||
flask_app=app,
|
||||
|
|
@ -32734,7 +32714,7 @@ _init_download_monitor(
|
|||
start_next_batch_of_downloads=_start_next_batch_of_downloads,
|
||||
orphaned_download_keys=_orphaned_download_keys,
|
||||
missing_download_executor_obj=missing_download_executor,
|
||||
soulseek_client_obj=soulseek_client,
|
||||
download_orchestrator_obj=download_orchestrator,
|
||||
)
|
||||
|
||||
# --- Hydrabase Auto-Reconnect ---
|
||||
|
|
|
|||
|
|
@ -3432,10 +3432,17 @@ const WHATS_NEW = {
|
|||
'2.4.2': [
|
||||
// --- post-2.4.1 dev work — entries hidden by _getLatestWhatsNewVersion until the build version bumps ---
|
||||
{ date: 'Unreleased — 2.4.2 dev cycle' },
|
||||
{ title: 'Drop SoundCloud Preview Snippets at Search Time', desc: 'soundcloud serves a ~30s preview clip for tracks that are gated behind go+ / login (very common for major-label uploads — official content basically doesn\'t exist on soundcloud, so what shows up is bootlegs, fan uploads, type beats, and these 30s previews). yt-dlp accepts the preview as the download payload, the post-download integrity check catches the duration mismatch and quarantines the file, but the user just sees "all candidates failed" with no explanation. previews also showed up in the candidate-review modal where clicking one bypassed validation and downloaded the same broken file. now `filter_soundcloud_previews` drops these candidates at every entry point — auto-search scoring, modal-cache fallback, AND the not-found raw-results path — so previews never reach the matcher OR the user. drops candidates < 35s or below half the expected duration, gated on expected being non-trivially long (>60s) so genuine short tracks still pass. also fixed a silent regression in the hybrid-fallback path where the per-source attribute removal left `getattr(orch, \'youtube\', None)` returning None for every source — fallback never fired. now resolves through the orchestrator\'s `client(name)` accessor.', page: 'downloads' },
|
||||
{ title: 'Internal: Move Shared Download Dataclasses + Singleton Boot Path', desc: 'internal — two architectural cleanups on top of the download engine refactor. (1) `TrackResult`, `AlbumResult`, `DownloadStatus`, `SearchResult` lived in `core/soulseek_client.py` for historical reasons (they grew up there as the soulseek-only types and got exported when other download sources were added). every plugin imported these from the soulseek module just to satisfy the contract — coupling 8 clients to a sibling source for type imports only. moved them to `core/download_plugins/types.py` (the neutral plugin package) and updated all 14 import sites across deezer/hifi/lidarr/qobuz/soundcloud/tidal/youtube clients + the engine + matching engine + redownload + tests. clean break, no backward-compat re-export. (2) `web_server.py` now boots the orchestrator via `set_download_orchestrator(DownloadOrchestrator())` so the singleton factory + boot path share state — `get_download_orchestrator()` returns the same instance the global handle points at instead of lazily building a separate one. matches cin\'s `get_metadata_engine()` pattern.' },
|
||||
{ title: 'Internal: Rename `soulseek_client` Global → `download_orchestrator`', desc: 'internal — followup cleanup. the global handle in web_server.py was named `soulseek_client` for historical reasons (the orchestrator was originally just the soulseek client and grew downstream sources around it), but the type has long been `DownloadOrchestrator` not `SoulseekClient`. renamed the global + every parameter/attribute that carried the legacy name across web_server.py, api/, core/downloads/*, core/search/*, core/streaming/*, services/sync_service.py, and the test fixtures (`MasterDeps.soulseek_client` → `download_orchestrator`, `init(soulseek_client_obj)` → `init(download_orchestrator_obj)`, etc). module path `core.soulseek_client` and class `SoulseekClient` (the actual soulseek-only client) are unchanged — only the orchestrator handle renamed. ~250 references touched, suite green.' },
|
||||
{ title: 'Internal: Drop Backward-Compat Per-Source Attrs', desc: 'internal — followup to cin\'s download engine review. removed the `orchestrator.soulseek` / `.youtube` / `.tidal` / `.qobuz` / `.hifi` / `.deezer_dl` / `.lidarr` / `.soundcloud` attribute aliases that were preserved for backward compat. external callers (core/downloads/, core/search/, web_server.py) all migrated to the generic `orchestrator.client(\'<source>\')` accessor — alias-aware (legacy `deezer_dl` resolves to canonical `deezer`), single source of truth via the registry. the orchestrator\'s own internal `self.soulseek` / `self.deezer_dl` reaches also routed through `client()` so the only place that knows about per-source identity is the registry. test fakes updated to expose `client(name)` instead of stuffing attributes; conformance test pinned to the new accessor contract. zero behavior change — just cleaner shape.' },
|
||||
{ title: 'Internal: Download Engine Review Followup', desc: 'internal — three correctness fixes on top of the download engine refactor, all flagged in cin\'s pr review. (1) `engine.cancel_download(source_hint=\'deezer_dl\')` was silently routing deezer cancels to soulseek because the legacy alias never made it to the engine\'s plugin map — only the registry knew about it. fix: aliases now flow through `register_plugin` and `get_plugin` / `cancel_download` resolve them to the canonical name. (2) `_resolve_source_chain` filtered hybrid_order against canonical registry names only, so any user with `deezer_dl` in their config quietly dropped deezer from hybrid mode. fix: orchestrator normalizes through `registry.get_spec()` first. (3) the worker\'s terminal write was a read-then-write split — a cancel landing between the snapshot and the update could be overwritten back to errored / completed. fix: new atomic `update_record_unless_state` on the engine holds `state_lock` across the check + write; both `_mark_terminal` AND the success path use it now. also added generic `client(name)` / `configured_clients()` / `reload_instances(name?)` accessors on the orchestrator + a `get/set_download_orchestrator()` singleton matching cin\'s `get_metadata_engine()` shape, and migrated 30 external `soulseek_client.<source>` reaches in web_server.py to `client("<source>")`. 18 new tests pin every fix.' },
|
||||
{ title: 'Internal: Typed Metadata Foundation', desc: 'internal — first step of a multi-pr migration to give the metadata pipeline a real contract. the codebase historically grew duck-typed extractors (`_extract_lookup_value(album_data, "id", "album_id", "collectionId", "release_id", default=...)`) at every consumer site because each provider returns its own response shape. ~150 of those across the codebase. new `core/metadata/types.py` defines canonical typed `Album` / `Track` / `Artist` dataclasses with strict required fields. per-source classmethod converters (from_spotify_dict, from_itunes_dict, from_deezer_dict, from_discogs_dict, from_musicbrainz_dict, from_hydrabase_dict) are the SINGLE place that knows each provider\'s wire shape. zero behavior changes in this pr — pure additive foundation. follow-up prs migrate consumers one at a time. full migration plan documented at docs/metadata-types-migration.md.', page: 'library' },
|
||||
{ title: 'Internal: Migrate Album-Info Builders to Typed Path', desc: 'internal — steps 2+3 of the typed metadata migration in one pr. two album-info builders now route through `Album.from_<source>_dict()` when the caller passes a known source: `_build_album_info` (used by every album-tracks lookup) and the embedded album section of `_build_single_import_context_payload` (used by single-track import context resolution). legacy duck-typed extraction stays as the fallback when source is empty/unknown, raw input isn\'t a dict, or the typed converter raises — so a converter bug can\'t break album resolution or import context. caller-provided album_id / album_name / artist_name fallbacks apply on the typed path the same way they did on legacy. zero behavior change for existing callers since they don\'t pass a source yet — opt-in only. 22 new tests pin the typed path, the legacy fallback, and parametrized coverage across registered providers.' },
|
||||
{ title: 'Internal: Migrate Discography + Quality Scanner to Typed Path', desc: 'internal — next round of the typed metadata migration. three more album-shape consumers now route through `Album.from_<source>_dict()` when the caller passes a known source: `_build_discography_release_dict` (artist discography release cards), `_build_artist_detail_release_card` (artist detail page release cards), and `_normalize_track_album` (quality scanner result normalization). legacy duck-typed extraction stays as the fallback when source is empty/unknown, raw input isn\'t a dict, or the typed converter raises — same safety contract as the prior migration steps. 20 new tests pin the typed path + legacy fallback + parametrized coverage across registered providers.' },
|
||||
{ title: 'Fix: Maintenance Findings Badge Showed Inflated Count With Empty Findings Tab', desc: 'discord report (husoyo): duplicate detector and cover art filler badges showed "364 findings" / "31 findings" after a scan, but clicking into the findings tab showed nothing. cause: `_create_finding` silently dedup-skipped re-discovered issues (when an equivalent row already existed with status pending/resolved/dismissed) but the caller incremented `result.findings_created` regardless of whether a row was actually inserted. so on a re-scan that found the same problems as a prior scan, the badge snapshot recorded 364 even though zero NEW pending rows hit the db. fix: `_create_finding` now returns a bool (True on insert, False on dedup-skip / db error). all 16 repair jobs updated to only increment `findings_created` on True. new `findings_skipped_dedup` counter added to job results and surfaced in the scan log: "Done: 2791 scanned, 0 fixed, 0 findings (363 already existed), 0 errors" — so re-scans show a real count, and you can see at a glance how many findings were carried over from prior scans. also fixed a missing `job_id` kwarg in the album tag consistency job that was silently breaking finding creation for that scan. companion ux improvement: findings tab now auto-switches its status filter from "pending" to "all status" when 0 pending rows exist but resolved/dismissed/auto-fixed rows do — with a small notice so you can see what carried over instead of staring at an "all clear" empty state.', page: 'library' },
|
||||
{ title: 'Internal: Download Source Plugin Contract', desc: 'internal — first step of a multi-step refactor on the multi-source download dispatcher. the orchestrator historically had 8 download sources (soulseek/youtube/tidal/qobuz/hifi/deezer/lidarr/soundcloud) hardcoded into 6+ different dispatch sites — `if username == "youtube" elif username == "tidal" elif ...` chains in `__init__`, search, download, get_all_downloads, cancel_download, etc. adding usenet (planned) would have meant 700+ lines of copy-paste across the same files. new `core/download_plugins/` package defines `DownloadSourcePlugin` (Protocol) — the canonical contract every source must satisfy: `is_configured`, `check_connection`, `search`, `download`, `get_all_downloads`, `get_download_status`, `cancel_download`, `clear_all_completed_downloads`. plus `DownloadPluginRegistry` — single source of truth for which sources exist, with name/alias resolution (legacy `deezer_dl` alias preserved). orchestrator now dispatches through the registry instead of hardcoded `[self.soulseek, self.youtube, ...]` lists; backward-compat `self.<source>` attributes still work so external callers reaching for source-specific internals (e.g. `orchestrator.soulseek._make_request`) keep working unchanged. zero behavior change for users — pure additive foundation that lets future PRs extract shared logic (background thread workers, search query normalization, post-processing context) into the contract instead of copy-pasted across all 8 sources. 19 new tests pin every plugin class\'s structural conformance to the contract — drift in any source will fail at the test boundary instead of at runtime against a live download.' },
|
||||
{ title: 'Internal: Download Engine — Background Worker, State, Fallback', desc: 'internal — followup to the download source plugin contract. lifts the duplicated thread-spawn boilerplate, per-source active_downloads dicts, and hybrid-fallback dispatch into a central `core/download_engine/` package. each streaming source (youtube, tidal, qobuz, hifi, deezer, soundcloud) used to hand-roll the same ~70 LOC of background thread management — semaphore-gated serialization, rate-limit sleep between downloads, state-dict updates for InProgress/Completed/Errored transitions, exception capture. ~490 LOC of copy-paste across 7 files. all of it gone now — `engine.worker.dispatch(source, target_id, impl_callable, ...)` owns thread spawning + semaphore + delay + state lifecycle. plugins provide only `_download_sync(download_id, target_id, display_name) → file_path`, the source-specific atomic download. per-source rate-limit policy declared via `RateLimitPolicy` (concurrency, delay) — engine reads at register time. cross-source state queries (`get_all_downloads`, `get_download_status`, `cancel_download`, `clear_all_completed_downloads`) read engine state directly instead of iterating per-source dicts. hybrid-mode search now goes through `engine.search_with_fallback(chain)` — same ordering / skip-unconfigured / swallow-per-source-exceptions semantics as before. every per-source migration commit gated by phase A pinning tests (54 tests across all 8 sources) so contract drift fails fast at the test boundary instead of at runtime against a live download. net: ~700 LOC removed across 6 client files, ~85 new engine + worker + rate-limit tests, suite green at every commit. zero behavior change for users — same downloads, same lifecycle states, same hybrid mode. backward-compat preserved for everything that reaches into `orchestrator.soulseek._make_request` etc. adding usenet now = one new client class + one registry entry, no orchestrator changes. follow-up: cin\'s metadata engine work may shape further refactors (e.g. extracting search retry / quality filter — left per-source for now since search code is genuinely 90% source-specific).' },
|
||||
{ title: 'Discogs Collection in "Your Albums"', desc: 'discord request: pull your discogs collection into the your albums section on discover, similar to spotify liked albums. set your discogs personal access token on settings → connections (already there from prior work) and add discogs as one of the configured sources via the gear button on your albums. background fetcher pulls your full collection (all folders, all pages — capped at 5000 releases), normalizes artist names (strips discogs `(N)` disambiguation suffix), dedupes against any spotify/tidal/deezer-saved versions of the same album. clicking a discogs-only album opens with discogs context — full release detail (year, format, label, country, tracklist) from the /releases endpoint. clicking an album that exists in both your spotify saved AND discogs collection prefers spotify (download flow is more direct). discogs is physical-media-first so many releases won\'t have streaming equivalents — those still show in the grid but the modal flow may need to fall back to a name search to find a downloadable digital version.', page: 'discover' },
|
||||
{ title: 'Drop Redundant "Your Spotify Library" Section on Discover', desc: 'discover page used to show two near-identical sections: "Your Albums" (cross-source aggregator across spotify/deezer/etc) AND "Your Spotify Library" (spotify-only). same UI, same grid, same filter / sort / download-missing controls — the spotify-only one was a strict subset of what your albums already covers. removed it. spotify saved albums still surface via the your albums section with spotify as one of its configured sources (gear button → configure sources). backend collection / storage is unchanged — the watchlist scanner still populates the spotify_library_albums cache for your albums to read.', page: 'discover' },
|
||||
{ title: 'Library Disk Usage on Stats Page', desc: 'discord request (samuel [KC]): show how much disk space the library takes. new card on stats → system statistics shows total bytes + per-format breakdown (FLAC vs MP3 vs M4A bars). data comes from `tracks.file_size` populated during deep scan from whatever the media server already returns (plex MediaPart.size, jellyfin MediaSources[].Size, navidrome song.size, soulsync standalone os.path.getsize) — zero filesystem walk overhead. existing libraries see "Run a Deep Scan to populate" until the next deep scan fills in sizes; partial coverage shown as "X tracks measured (+Y pending)". migration is additive (NULL on legacy rows) so upgrading users have nothing to do.', page: 'stats' },
|
||||
|
|
|
|||
Loading…
Reference in a new issue