Harden download orchestrator and surface init failures in debug info

- Each of the 6 download clients initializes independently via
  _safe_init() — one failing client no longer kills the orchestrator
- All methods guarded against None clients with appropriate fallbacks
- Init failures logged at startup and tracked in _init_failures list
- Copy Debug Info shows "Download Client Failures" section when any
  client failed to initialize, or "ALL" if orchestrator itself is dead
This commit is contained in:
Broque Thomas 2026-04-02 11:40:11 -07:00
parent d5bb0dfaa1
commit 22552032e2
3 changed files with 146 additions and 190 deletions

View file

@ -37,13 +37,19 @@ class DownloadOrchestrator:
""" """
def __init__(self): def __init__(self):
"""Initialize orchestrator with all clients""" """Initialize orchestrator with all clients.
self.soulseek = SoulseekClient() Each client is initialized independently one failing client doesn't prevent others from working."""
self.youtube = YouTubeClient() self._init_failures = []
self.tidal = TidalDownloadClient()
self.qobuz = QobuzClient() self.soulseek = self._safe_init('Soulseek', SoulseekClient)
self.hifi = HiFiClient() self.youtube = self._safe_init('YouTube', YouTubeClient)
self.deezer_dl = DeezerDownloadClient() self.tidal = self._safe_init('Tidal', TidalDownloadClient)
self.qobuz = self._safe_init('Qobuz', QobuzClient)
self.hifi = self._safe_init('HiFi', HiFiClient)
self.deezer_dl = self._safe_init('Deezer', DeezerDownloadClient)
if self._init_failures:
logger.warning(f"⚠️ Download clients failed to initialize: {', '.join(self._init_failures)}")
# Load mode from config # Load mode from config
self.mode = config_manager.get('download_source.mode', 'soulseek') self.mode = config_manager.get('download_source.mode', 'soulseek')
@ -58,6 +64,15 @@ class DownloadOrchestrator:
else: else:
logger.info(f" Primary: {self.hybrid_primary}, Fallback: {self.hybrid_secondary}") logger.info(f" Primary: {self.hybrid_primary}, Fallback: {self.hybrid_secondary}")
def _safe_init(self, name, cls):
"""Initialize a download client, returning None on failure instead of crashing."""
try:
return cls()
except Exception as e:
logger.error(f"{name} download client failed to initialize: {e}")
self._init_failures.append(name)
return None
def reload_settings(self): def reload_settings(self):
"""Reload settings from config (call after settings change)""" """Reload settings from config (call after settings change)"""
self.mode = config_manager.get('download_source.mode', 'soulseek') self.mode = config_manager.get('download_source.mode', 'soulseek')
@ -66,53 +81,43 @@ class DownloadOrchestrator:
self.hybrid_order = config_manager.get('download_source.hybrid_order', ['hifi', 'youtube', 'soulseek']) self.hybrid_order = config_manager.get('download_source.hybrid_order', ['hifi', 'youtube', 'soulseek'])
# Reload underlying client configs (SLSKD URL, API key, etc.) # Reload underlying client configs (SLSKD URL, API key, etc.)
self.soulseek._setup_client() if self.soulseek:
logger.info(f"🔄 Soulseek client config reloaded") self.soulseek._setup_client()
logger.info(f"🔄 Soulseek client config reloaded")
# Reconnect Deezer if ARL changed # Reconnect Deezer if ARL changed
deezer_arl = config_manager.get('deezer_download.arl', '') deezer_arl = config_manager.get('deezer_download.arl', '')
if deezer_arl: if deezer_arl and self.deezer_dl:
self.deezer_dl.reconnect(deezer_arl) self.deezer_dl.reconnect(deezer_arl)
self.deezer_dl._quality = config_manager.get('deezer_download.quality', 'flac') self.deezer_dl._quality = config_manager.get('deezer_download.quality', 'flac')
logger.info(f"🔄 Download Orchestrator settings reloaded - Mode: {self.mode}") logger.info(f"🔄 Download Orchestrator settings reloaded - Mode: {self.mode}")
def _client(self, name):
"""Get a client by name, returning None if not initialized."""
return {'soulseek': self.soulseek, 'youtube': self.youtube, 'tidal': self.tidal,
'qobuz': self.qobuz, 'hifi': self.hifi, 'deezer_dl': self.deezer_dl}.get(name)
def is_configured(self) -> bool: def is_configured(self) -> bool:
""" """
Check if orchestrator is configured and ready to use. Check if orchestrator is configured and ready to use.
Returns True if at least one download source is configured. Returns True if at least one download source is configured.
""" """
if self.mode == 'soulseek': client = self._client(self.mode)
return self.soulseek.is_configured() if client:
elif self.mode == 'youtube': return client.is_configured()
return self.youtube.is_configured()
elif self.mode == 'tidal':
return self.tidal.is_configured()
elif self.mode == 'qobuz':
return self.qobuz.is_configured()
elif self.mode == 'hifi':
return self.hifi.is_configured()
elif self.mode == 'deezer_dl':
return self.deezer_dl.is_configured()
elif self.mode == 'hybrid': elif self.mode == 'hybrid':
clients = {'soulseek': self.soulseek, 'youtube': self.youtube, 'tidal': self.tidal, 'qobuz': self.qobuz, 'hifi': self.hifi, 'deezer_dl': self.deezer_dl}
sources = self.hybrid_order if self.hybrid_order else [self.hybrid_primary, self.hybrid_secondary] sources = self.hybrid_order if self.hybrid_order else [self.hybrid_primary, self.hybrid_secondary]
return any(clients[s].is_configured() for s in sources if s in clients) return any(c.is_configured() for s in sources if (c := self._client(s)))
return False return False
def get_source_status(self) -> dict: def get_source_status(self) -> dict:
"""Return configured status for each download source.""" """Return configured status for each download source."""
clients = { return {name: (c.is_configured() if c else False)
'soulseek': self.soulseek, for name, c in [('soulseek', self.soulseek), ('youtube', self.youtube),
'youtube': self.youtube, ('tidal', self.tidal), ('qobuz', self.qobuz),
'tidal': self.tidal, ('hifi', self.hifi), ('deezer_dl', self.deezer_dl)]}
'qobuz': self.qobuz,
'hifi': self.hifi,
'deezer_dl': self.deezer_dl,
}
return {name: client.is_configured() for name, client in clients.items()}
async def check_connection(self) -> bool: async def check_connection(self) -> bool:
""" """
@ -120,29 +125,14 @@ class DownloadOrchestrator:
Returns True if the configured source(s) are reachable. Returns True if the configured source(s) are reachable.
""" """
if self.mode == 'soulseek': client = self._client(self.mode)
return await self.soulseek.check_connection() if client and self.mode != 'hybrid':
elif self.mode == 'youtube': return await client.check_connection()
return await self.youtube.check_connection()
elif self.mode == 'tidal':
return await self.tidal.check_connection()
elif self.mode == 'qobuz':
return await self.qobuz.check_connection()
elif self.mode == 'hifi':
return await self.hifi.check_connection()
elif self.mode == 'deezer_dl':
return await self.deezer_dl.check_connection()
elif self.mode == 'hybrid': elif self.mode == 'hybrid':
# Only check sources that are in the configured hybrid order sources_to_check = self.hybrid_order if self.hybrid_order else ['soulseek', 'youtube', 'tidal', 'qobuz', 'hifi', 'deezer_dl']
clients = {
'soulseek': self.soulseek, 'youtube': self.youtube,
'tidal': self.tidal, 'qobuz': self.qobuz,
'hifi': self.hifi, 'deezer_dl': self.deezer_dl,
}
sources_to_check = self.hybrid_order if self.hybrid_order else list(clients.keys())
results = {} results = {}
for source in sources_to_check: for source in sources_to_check:
client = clients.get(source) client = self._client(source)
if client: if client:
try: try:
results[source] = await client.check_connection() results[source] = await client.check_connection()
@ -168,39 +158,19 @@ class DownloadOrchestrator:
Returns: Returns:
Tuple of (track_results, album_results) Tuple of (track_results, album_results)
""" """
if self.mode == 'soulseek': source_names = {'soulseek': 'Soulseek', 'youtube': 'YouTube', 'tidal': 'Tidal',
logger.info(f"🔍 Searching Soulseek: {query}") 'qobuz': 'Qobuz', 'hifi': 'HiFi', 'deezer_dl': 'Deezer'}
return await self.soulseek.search(query, timeout, progress_callback)
elif self.mode == 'youtube': if self.mode != 'hybrid':
logger.info(f"🔍 Searching YouTube: {query}") client = self._client(self.mode)
return await self.youtube.search(query, timeout, progress_callback) if not client:
logger.error(f"{source_names.get(self.mode, self.mode)} client not available (failed to initialize)")
elif self.mode == 'tidal': return [], []
logger.info(f"🔍 Searching Tidal: {query}") logger.info(f"🔍 Searching {source_names.get(self.mode, self.mode)}: {query}")
return await self.tidal.search(query, timeout, progress_callback) return await client.search(query, timeout, progress_callback)
elif self.mode == 'qobuz':
logger.info(f"🔍 Searching Qobuz: {query}")
return await self.qobuz.search(query, timeout, progress_callback)
elif self.mode == 'hifi':
logger.info(f"🔍 Searching HiFi: {query}")
return await self.hifi.search(query, timeout, progress_callback)
elif self.mode == 'deezer_dl':
logger.info(f"🔍 Searching Deezer: {query}")
return await self.deezer_dl.search(query, timeout, progress_callback)
elif self.mode == 'hybrid': elif self.mode == 'hybrid':
clients = { clients = {name: self._client(name) for name in source_names}
'soulseek': self.soulseek,
'youtube': self.youtube,
'tidal': self.tidal,
'qobuz': self.qobuz,
'hifi': self.hifi,
'deezer_dl': self.deezer_dl,
}
# Build ordered source list: prefer hybrid_order, fall back to legacy primary/secondary # Build ordered source list: prefer hybrid_order, fall back to legacy primary/secondary
if self.hybrid_order: if self.hybrid_order:
@ -217,9 +187,12 @@ class DownloadOrchestrator:
logger.info(f"🔍 Hybrid search ({''.join(source_order)}): {query}") logger.info(f"🔍 Hybrid search ({''.join(source_order)}): {query}")
# Try each source in priority order (skip unconfigured ones) # Try each source in priority order (skip unconfigured/unavailable ones)
for i, source_name in enumerate(source_order): for i, source_name in enumerate(source_order):
client = clients[source_name] client = clients.get(source_name)
if not client:
logger.info(f"⏭️ Skipping {source_name} (not available)")
continue
if hasattr(client, 'is_configured') and not client.is_configured(): if hasattr(client, 'is_configured') and not client.is_configured():
logger.info(f"⏭️ Skipping {source_name} (not configured)") logger.info(f"⏭️ Skipping {source_name} (not configured)")
continue continue
@ -274,7 +247,7 @@ class DownloadOrchestrator:
if is_streaming: if is_streaming:
filtered_results = tracks filtered_results = tracks
else: else:
filtered_results = self.soulseek.filter_results_by_quality_preference(tracks) filtered_results = self.soulseek.filter_results_by_quality_preference(tracks) if self.soulseek else tracks
if not filtered_results: if not filtered_results:
logger.warning(f"No suitable quality results found for: {query}") logger.warning(f"No suitable quality results found for: {query}")
@ -305,22 +278,20 @@ class DownloadOrchestrator:
download_id: Unique download ID for tracking download_id: Unique download ID for tracking
""" """
# Detect which client to use based on username # Detect which client to use based on username
if username == 'youtube': source_map = {'youtube': self.youtube, 'tidal': self.tidal, 'qobuz': self.qobuz,
logger.info(f"📥 Downloading from YouTube: {filename}") 'hifi': self.hifi, 'deezer_dl': self.deezer_dl}
return await self.youtube.download(username, filename, file_size) source_names = {'youtube': 'YouTube', 'tidal': 'Tidal', 'qobuz': 'Qobuz',
elif username == 'tidal': 'hifi': 'HiFi', 'deezer_dl': 'Deezer'}
logger.info(f"📥 Downloading from Tidal: {filename}")
return await self.tidal.download(username, filename, file_size) if username in source_map:
elif username == 'qobuz': client = source_map[username]
logger.info(f"📥 Downloading from Qobuz: {filename}") if not client:
return await self.qobuz.download(username, filename, file_size) raise RuntimeError(f"{source_names[username]} download client not available (failed to initialize)")
elif username == 'hifi': logger.info(f"📥 Downloading from {source_names[username]}: {filename}")
logger.info(f"📥 Downloading from HiFi: {filename}") return await client.download(username, filename, file_size)
return await self.hifi.download(username, filename, file_size)
elif username == 'deezer_dl':
logger.info(f"📥 Downloading from Deezer: {filename}")
return await self.deezer_dl.download(username, filename, file_size)
else: else:
if not self.soulseek:
raise RuntimeError("Soulseek client not available (failed to initialize)")
logger.info(f"📥 Downloading from Soulseek: {filename}") logger.info(f"📥 Downloading from Soulseek: {filename}")
return await self.soulseek.download(username, filename, file_size) return await self.soulseek.download(username, filename, file_size)
@ -331,15 +302,15 @@ class DownloadOrchestrator:
Returns: Returns:
List of DownloadStatus objects List of DownloadStatus objects
""" """
# Get downloads from all sources # Get downloads from all available sources
soulseek_downloads = await self.soulseek.get_all_downloads() all_downloads = []
youtube_downloads = await self.youtube.get_all_downloads() for client in [self.soulseek, self.youtube, self.tidal, self.qobuz, self.hifi, self.deezer_dl]:
tidal_downloads = await self.tidal.get_all_downloads() if client:
qobuz_downloads = await self.qobuz.get_all_downloads() try:
hifi_downloads = await self.hifi.get_all_downloads() all_downloads.extend(await client.get_all_downloads())
deezer_downloads = await self.deezer_dl.get_all_downloads() except Exception:
pass
return soulseek_downloads + youtube_downloads + tidal_downloads + qobuz_downloads + hifi_downloads + deezer_downloads return all_downloads
async def get_download_status(self, download_id: str) -> Optional[DownloadStatus]: async def get_download_status(self, download_id: str) -> Optional[DownloadStatus]:
""" """
@ -351,35 +322,16 @@ class DownloadOrchestrator:
Returns: Returns:
DownloadStatus object or None if not found DownloadStatus object or None if not found
""" """
# Try Soulseek first # Try each source until we find the download
status = await self.soulseek.get_download_status(download_id) for client in [self.soulseek, self.youtube, self.tidal, self.qobuz, self.hifi, self.deezer_dl]:
if status: if not client:
return status continue
try:
# Try YouTube status = await client.get_download_status(download_id)
status = await self.youtube.get_download_status(download_id) if status:
if status: return status
return status except Exception:
pass
# Try Tidal
status = await self.tidal.get_download_status(download_id)
if status:
return status
# Try Qobuz
status = await self.qobuz.get_download_status(download_id)
if status:
return status
# Try HiFi
status = await self.hifi.get_download_status(download_id)
if status:
return status
# Try Deezer
status = await self.deezer_dl.get_download_status(download_id)
if status:
return status
return None return None
@ -395,43 +347,25 @@ class DownloadOrchestrator:
Returns: Returns:
True if cancelled successfully True if cancelled successfully
""" """
# If username is provided, route directly # If username is provided, route directly to that source
if username == 'youtube': source_map = {'youtube': self.youtube, 'tidal': self.tidal, 'qobuz': self.qobuz,
return await self.youtube.cancel_download(download_id, username, remove) 'hifi': self.hifi, 'deezer_dl': self.deezer_dl}
elif username == 'tidal': if username in source_map:
return await self.tidal.cancel_download(download_id, username, remove) client = source_map[username]
elif username == 'qobuz': return await client.cancel_download(download_id, username, remove) if client else False
return await self.qobuz.cancel_download(download_id, username, remove)
elif username == 'hifi':
return await self.hifi.cancel_download(download_id, username, remove)
elif username == 'deezer_dl':
return await self.deezer_dl.cancel_download(download_id, username, remove)
elif username: elif username:
return await self.soulseek.cancel_download(download_id, username, remove) return await self.soulseek.cancel_download(download_id, username, remove) if self.soulseek else False
# Otherwise, try all sources # Otherwise, try all available sources
soulseek_cancelled = await self.soulseek.cancel_download(download_id, username, remove) for client in [self.soulseek, self.youtube, self.tidal, self.qobuz, self.hifi, self.deezer_dl]:
if soulseek_cancelled: if not client:
return True continue
try:
youtube_cancelled = await self.youtube.cancel_download(download_id, username, remove) if await client.cancel_download(download_id, username, remove):
if youtube_cancelled: return True
return True except Exception:
pass
tidal_cancelled = await self.tidal.cancel_download(download_id, username, remove) return False
if tidal_cancelled:
return True
qobuz_cancelled = await self.qobuz.cancel_download(download_id, username, remove)
if qobuz_cancelled:
return True
hifi_cancelled = await self.hifi.cancel_download(download_id, username, remove)
if hifi_cancelled:
return True
deezer_cancelled = await self.deezer_dl.cancel_download(download_id, username, remove)
return deezer_cancelled
async def signal_download_completion(self, download_id: str, username: str, remove: bool = True) -> bool: async def signal_download_completion(self, download_id: str, username: str, remove: bool = True) -> bool:
""" """
@ -446,6 +380,8 @@ class DownloadOrchestrator:
True if successful True if successful
""" """
# This is Soulseek-specific, so only call on Soulseek client # This is Soulseek-specific, so only call on Soulseek client
if not self.soulseek:
return False
return await self.soulseek.signal_download_completion(download_id, username, remove) return await self.soulseek.signal_download_completion(download_id, username, remove)
async def clear_all_completed_downloads(self) -> bool: async def clear_all_completed_downloads(self) -> bool:
@ -455,14 +391,15 @@ class DownloadOrchestrator:
Returns: Returns:
True if successful True if successful
""" """
soulseek_cleared = await self.soulseek.clear_all_completed_downloads() results = []
youtube_cleared = await self.youtube.clear_all_completed_downloads() for client in [self.soulseek, self.youtube, self.tidal, self.qobuz, self.hifi, self.deezer_dl]:
tidal_cleared = await self.tidal.clear_all_completed_downloads() if client:
qobuz_cleared = await self.qobuz.clear_all_completed_downloads() try:
hifi_cleared = await self.hifi.clear_all_completed_downloads() results.append(await client.clear_all_completed_downloads())
deezer_cleared = await self.deezer_dl.clear_all_completed_downloads() except Exception:
pass
return soulseek_cleared and youtube_cleared and tidal_cleared and qobuz_cleared and hifi_cleared and deezer_cleared return all(results) if results else True
# ===== Soulseek-specific methods (for backwards compatibility) ===== # ===== Soulseek-specific methods (for backwards compatibility) =====
# These are internal methods that some parts of the codebase use directly # These are internal methods that some parts of the codebase use directly
@ -480,6 +417,8 @@ class DownloadOrchestrator:
Returns: Returns:
API response API response
""" """
if not self.soulseek:
raise RuntimeError("Soulseek client not available (failed to initialize)")
return await self.soulseek._make_request(method, endpoint, **kwargs) return await self.soulseek._make_request(method, endpoint, **kwargs)
async def _make_direct_request(self, method: str, endpoint: str, **kwargs): async def _make_direct_request(self, method: str, endpoint: str, **kwargs):
@ -495,6 +434,8 @@ class DownloadOrchestrator:
Returns: Returns:
API response API response
""" """
if not self.soulseek:
raise RuntimeError("Soulseek client not available (failed to initialize)")
return await self.soulseek._make_direct_request(method, endpoint, **kwargs) return await self.soulseek._make_direct_request(method, endpoint, **kwargs)
async def clear_all_searches(self) -> bool: async def clear_all_searches(self) -> bool:
@ -504,7 +445,7 @@ class DownloadOrchestrator:
Returns: Returns:
True if successful True if successful
""" """
return await self.soulseek.clear_all_searches() return await self.soulseek.clear_all_searches() if self.soulseek else True
async def maintain_search_history_with_buffer(self, keep_searches: int = 50, trigger_threshold: int = 200) -> bool: async def maintain_search_history_with_buffer(self, keep_searches: int = 50, trigger_threshold: int = 200) -> bool:
""" """
@ -517,13 +458,15 @@ class DownloadOrchestrator:
Returns: Returns:
True if successful True if successful
""" """
return await self.soulseek.maintain_search_history_with_buffer(keep_searches, trigger_threshold) return await self.soulseek.maintain_search_history_with_buffer(keep_searches, trigger_threshold) if self.soulseek else True
async def cancel_all_downloads(self) -> bool: async def cancel_all_downloads(self) -> bool:
"""Cancel and remove all downloads from all sources.""" """Cancel and remove all downloads from all sources."""
soulseek_ok = await self.soulseek.cancel_all_downloads() ok = True
await self.tidal.clear_all_completed_downloads() for client in [self.soulseek, self.tidal, self.qobuz, self.hifi, self.deezer_dl]:
await self.qobuz.clear_all_completed_downloads() if client:
await self.hifi.clear_all_completed_downloads() try:
await self.deezer_dl.clear_all_completed_downloads() await client.cancel_all_downloads() if hasattr(client, 'cancel_all_downloads') else await client.clear_all_completed_downloads()
return soulseek_ok except Exception:
ok = False
return ok

View file

@ -4536,6 +4536,13 @@ def get_debug_info():
'm3u_export_enabled': config_manager.get('m3u.enabled', False), 'm3u_export_enabled': config_manager.get('m3u.enabled', False),
} }
# Download client init failures
info['download_client_failures'] = []
if soulseek_client and hasattr(soulseek_client, '_init_failures'):
info['download_client_failures'] = soulseek_client._init_failures
elif not soulseek_client:
info['download_client_failures'] = ['ALL (orchestrator failed to initialize)']
# API rate monitor — current calls/min per service + Spotify rate limit state # API rate monitor — current calls/min per service + Spotify rate limit state
try: try:
from core.api_call_tracker import api_call_tracker from core.api_call_tracker import api_call_tracker

View file

@ -2291,6 +2291,12 @@ function initializeDocsPage() {
} }
text += '\n'; text += '\n';
if (data.download_client_failures?.length > 0) {
text += '── Download Client Failures ──\n';
data.download_client_failures.forEach(f => { text += `${f}\n`; });
text += '\n';
}
text += '── API Rates (calls/min) ──\n'; text += '── API Rates (calls/min) ──\n';
if (data.api_rates) { if (data.api_rates) {
Object.entries(data.api_rates).forEach(([svc, info]) => { Object.entries(data.api_rates).forEach(([svc, info]) => {