From 0fd6be3239f9f15c609383a9a1faea59803715ff Mon Sep 17 00:00:00 2001 From: Broque Thomas <26755000+Nezreka@users.noreply.github.com> Date: Wed, 1 Apr 2026 13:53:57 -0700 Subject: [PATCH] Stream download source results progressively via NDJSON MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Step 2 of the redownload modal now streams results as each download source responds instead of waiting for all sources to finish. Tidal/ YouTube/Qobuz columns appear instantly while Soulseek searches. Backend: search-sources endpoint uses ThreadPoolExecutor + NDJSON streaming โ€” one JSON line per source as it completes. Frontend: reads the NDJSON stream, appends columns with fade-in animation as each source responds. Download button enables as soon as any results arrive. Each source gets its own column with results grouped and sorted by confidence. Visual confidence bars, format badges, and source-specific metadata (Soulseek username/slots). Best overall match auto-selected. --- web_server.py | 161 +++++++++++++++--------- webui/static/script.js | 279 ++++++++++++++++++++++++++++++++++------- webui/static/style.css | 121 ++++++++++++++---- 3 files changed, 435 insertions(+), 126 deletions(-) diff --git a/web_server.py b/web_server.py index 6ef35e4a..ff867aa8 100644 --- a/web_server.py +++ b/web_server.py @@ -13068,13 +13068,20 @@ def redownload_search_metadata(track_id): except Exception as e: logger.debug(f"Deezer client not available for redownload search: {e}") + # Build source-optimized queries + deezer_query = f'artist:"{artist_name}" track:"{clean_title}"' + def _search_source(source_name, client): try: - logger.info(f"[Redownload] Searching {source_name} for: {query}") - track_objs = client.search_tracks(query, limit=10) - # If no results with full query, try title only - if not track_objs and clean_title: - logger.info(f"[Redownload] {source_name} got 0 results, trying title only: {clean_title}") + # Deezer works best with structured artist:/track: queries + search_q = deezer_query if source_name == 'deezer' else query + logger.info(f"[Redownload] Searching {source_name} for: {search_q}") + track_objs = client.search_tracks(search_q, limit=10) + # If no results, try plain query as fallback + if not track_objs and search_q != query: + track_objs = client.search_tracks(query, limit=10) + # Last resort: title only + if not track_objs and clean_title != query: track_objs = client.search_tracks(clean_title, limit=10) logger.info(f"[Redownload] {source_name} returned {len(track_objs)} results") results = [] @@ -13151,69 +13158,109 @@ def redownload_search_sources(track_id): if not search_queries: search_queries = [f"{metadata.get('artist', '')} {metadata['name']}".strip()] - # Limit to first 3 queries for speed - search_queries = search_queries[:3] + # Use first 2 queries for speed + search_queries = search_queries[:2] - # Search download sources + # Search ALL configured download sources individually (not through hybrid which stops at first hit) candidates = [] database = get_database() - for qi, query in enumerate(search_queries): - try: - tracks_result, _ = run_async(soulseek_client.search(query, timeout=25)) - if not tracks_result: - continue + # Get all available download source clients + download_clients = {} + try: + orch = soulseek_client # The download orchestrator + if hasattr(orch, 'soulseek') and orch.soulseek: + if not (hasattr(orch.soulseek, 'is_configured') and not orch.soulseek.is_configured()): + download_clients['soulseek'] = orch.soulseek + if hasattr(orch, 'youtube') and orch.youtube: + if not (hasattr(orch.youtube, 'is_configured') and not orch.youtube.is_configured()): + download_clients['youtube'] = orch.youtube + if hasattr(orch, 'tidal') and orch.tidal: + if not (hasattr(orch.tidal, 'is_configured') and not orch.tidal.is_configured()): + download_clients['tidal'] = orch.tidal + if hasattr(orch, 'qobuz') and orch.qobuz: + if not (hasattr(orch.qobuz, 'is_configured') and not orch.qobuz.is_configured()): + download_clients['qobuz'] = orch.qobuz + if hasattr(orch, 'hifi') and orch.hifi: + if not (hasattr(orch.hifi, 'is_configured') and not orch.hifi.is_configured()): + download_clients['hifi'] = orch.hifi + if hasattr(orch, 'deezer_dl') and orch.deezer_dl: + if not (hasattr(orch.deezer_dl, 'is_configured') and not orch.deezer_dl.is_configured()): + download_clients['deezer_dl'] = orch.deezer_dl + except Exception as e: + logger.warning(f"[Redownload] Error getting download clients: {e}") - valid = get_valid_candidates(tracks_result, track_obj, query) - for candidate in valid: - # Check blacklist - is_bl = database.is_blacklisted(candidate.username, candidate.filename) + if not download_clients: + # Fallback: use orchestrator directly + download_clients = {'default': soulseek_client} - # Extract display name from filename - display_name = os.path.basename(candidate.filename.replace('\\', '/')) - ext = os.path.splitext(display_name)[1].lstrip('.').upper() - quality = ext if ext in ('FLAC', 'MP3', 'OPUS', 'OGG', 'M4A', 'WAV') else candidate.quality or '' + logger.info(f"[Redownload] Streaming search across {len(download_clients)} sources: {list(download_clients.keys())}") - candidates.append({ - 'username': candidate.username, - 'filename': candidate.filename, - 'display_name': display_name, - 'size': candidate.size or 0, - 'size_display': f"{(candidate.size or 0) / 1048576:.1f} MB", - 'bitrate': candidate.bitrate or 0, - 'quality': quality, - 'duration': candidate.duration or 0, - 'confidence': round(getattr(candidate, 'confidence', 0), 3), - 'source_query': query, - 'blacklisted': is_bl, - 'free_upload_slots': getattr(candidate, 'free_upload_slots', 0), - 'upload_speed': getattr(candidate, 'upload_speed', 0), - 'queue_length': getattr(candidate, 'queue_length', 0), - }) - except Exception as e: - logger.debug(f"Download source search failed for query '{query}': {e}") + def _search_one_source(source_name, client): + """Search a single download source and return formatted candidates.""" + source_candidates = [] + for qi, q in enumerate(search_queries): + try: + tracks_result, _ = run_async(client.search(q, timeout=20)) + if not tracks_result: + continue + valid = get_valid_candidates(tracks_result, track_obj, q) + for candidate in valid: + is_bl = database.is_blacklisted(candidate.username, candidate.filename) + display_name = os.path.basename(candidate.filename.replace('\\', '/')) + ext = os.path.splitext(display_name)[1].lstrip('.').upper() + quality = ext if ext in ('FLAC', 'MP3', 'OPUS', 'OGG', 'M4A', 'WAV') else candidate.quality or '' + svc = source_name if source_name != 'default' else 'hybrid' + uname = candidate.username + if uname in ('youtube', 'tidal', 'qobuz', 'hifi', 'deezer_dl'): + svc = uname + source_candidates.append({ + 'username': uname, + 'filename': candidate.filename, + 'display_name': display_name, + 'size': candidate.size or 0, + 'size_display': f"{(candidate.size or 0) / 1048576:.1f} MB", + 'bitrate': candidate.bitrate or 0, + 'quality': quality, + 'duration': candidate.duration or 0, + 'confidence': round(getattr(candidate, 'confidence', 0), 3), + 'source_service': svc, + 'source_query': q, + 'blacklisted': is_bl, + 'free_upload_slots': getattr(candidate, 'free_upload_slots', 0), + 'upload_speed': getattr(candidate, 'upload_speed', 0), + 'queue_length': getattr(candidate, 'queue_length', 0), + }) + except Exception as e: + logger.debug(f"[Redownload] {source_name} search failed for query '{q}': {e}") + # Deduplicate within source + seen = set() + unique = [] + for c in source_candidates: + key = f"{c['username']}|{c['filename']}" + if key not in seen: + seen.add(key) + unique.append(c) + unique.sort(key=lambda c: (-int(not c['blacklisted']), -c['confidence'])) + return unique - # Deduplicate by username+filename - seen = set() - unique = [] - for c in candidates: - key = f"{c['username']}|{c['filename']}" - if key not in seen: - seen.add(key) - unique.append(c) - candidates = unique + # Stream NDJSON โ€” one line per source as it completes + from concurrent.futures import ThreadPoolExecutor, as_completed - # Sort: non-blacklisted first, then by confidence - candidates.sort(key=lambda c: (-int(not c['blacklisted']), -c['confidence'])) + def generate_stream(): + with ThreadPoolExecutor(max_workers=4) as pool: + futures = {pool.submit(_search_one_source, name, client): name for name, client in download_clients.items()} + for future in as_completed(futures): + source_name = futures[future] + try: + results = future.result() + yield json.dumps({'source': source_name, 'candidates': results}) + '\n' + except Exception as e: + yield json.dumps({'source': source_name, 'candidates': [], 'error': str(e)}) + '\n' + yield json.dumps({'done': True}) + '\n' - best_idx = next((i for i, c in enumerate(candidates) if not c['blacklisted']), 0) if candidates else -1 + return app.response_class(generate_stream(), mimetype='application/x-ndjson', headers={'X-Accel-Buffering': 'no'}) - return jsonify({ - "success": True, - "candidates": candidates, - "best_candidate_index": best_idx, - "queries_used": search_queries, - }) except Exception as e: logger.error(f"Error in redownload source search: {e}", exc_info=True) return jsonify({"success": False, "error": str(e)}), 500 diff --git a/webui/static/script.js b/webui/static/script.js index 70e282e9..78dba4aa 100644 --- a/webui/static/script.js +++ b/webui/static/script.js @@ -42982,71 +42982,260 @@ function _renderRedownloadStep1(overlay, track, data) { overlay.querySelectorAll('.redownload-step').forEach(s => s.classList.remove('active')); overlay.querySelector('.redownload-step[data-step="2"]').classList.add('active'); - body.innerHTML = '
Searching download sources...
'; + // Stream results from all download sources โ€” columns appear as each source responds + body.innerHTML = ` +
+
Searching download sources...
+
+ +
+ + +
+ `; - try { - const res = await fetch(`/api/library/track/${track.id}/redownload/search-sources`, { - method: 'POST', - headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify({ metadata: selectedMeta }) - }); - const srcData = await res.json(); - if (!srcData.success) throw new Error(srcData.error); - _renderRedownloadStep2(overlay, track, selectedMeta, srcData); - } catch (e) { - body.innerHTML = `
Source search failed: ${_esc(e.message)}
`; - } + _streamRedownloadSources(overlay, track, selectedMeta); }); } -function _renderRedownloadStep2(overlay, track, metadata, srcData) { - const body = document.getElementById('redownload-body'); - if (!body) return; +async function _streamRedownloadSources(overlay, track, metadata) { + const columnsEl = document.getElementById('rdl-src-columns'); + const loadingEl = document.getElementById('rdl-src-loading'); + const startBtn = document.getElementById('redownload-start-btn'); + if (!columnsEl) return; - const candidates = srcData.candidates || []; - if (candidates.length === 0) { - body.innerHTML = ` -
No download sources found for this track.
-
- -
`; - return; + const serviceIcons = { soulseek: '๐Ÿ”', youtube: 'โ–ถ๏ธ', tidal: '๐ŸŒŠ', qobuz: '๐ŸŽต', hifi: '๐ŸŽง', deezer_dl: '๐Ÿ’œ', hybrid: 'โšก' }; + const serviceLabels = { soulseek: 'Soulseek', youtube: 'YouTube', tidal: 'Tidal', qobuz: 'Qobuz', hifi: 'HiFi', deezer_dl: 'Deezer', hybrid: 'Auto' }; + + let allCandidates = []; + let firstResult = true; + let bestGlobalIdx = -1; + + try { + const res = await fetch(`/api/library/track/${track.id}/redownload/search-sources`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ metadata }) + }); + + const reader = res.body.getReader(); + const decoder = new TextDecoder(); + let buffer = ''; + + while (true) { + const { done, value } = await reader.read(); + if (done) break; + buffer += decoder.decode(value, { stream: true }); + + const lines = buffer.split('\n'); + buffer = lines.pop(); // keep incomplete line + + for (const line of lines) { + if (!line.trim()) continue; + try { + const data = JSON.parse(line); + if (data.done) continue; + + const svc = data.source; + const candidates = data.candidates || []; + + // Remove loading spinner on first result + if (firstResult && loadingEl) { loadingEl.remove(); firstResult = false; } + + // Assign global indices + const startIdx = allCandidates.length; + candidates.forEach((c, i) => { c._globalIdx = startIdx + i; }); + allCandidates.push(...candidates); + + // Find best overall candidate + bestGlobalIdx = -1; + let bestConf = 0; + allCandidates.forEach((c, i) => { + if (!c.blacklisted && c.confidence > bestConf) { bestConf = c.confidence; bestGlobalIdx = i; } + }); + + // Render column for this source + const icon = serviceIcons[svc] || '๐Ÿ“ฆ'; + const label = serviceLabels[svc] || svc; + + const itemsHtml = candidates.length === 0 + ? '
No results
' + : candidates.slice(0, 10).map(c => { + const confPct = Math.round((c.confidence || 0) * 100); + const confCls = confPct >= 90 ? 'high' : confPct >= 70 ? 'medium' : 'low'; + const isRec = c._globalIdx === bestGlobalIdx; + const blClass = c.blacklisted ? ' blacklisted' : ''; + const dur = c.duration ? `${Math.floor(c.duration / 60000)}:${String(Math.floor((c.duration % 60000) / 1000)).padStart(2, '0')}` : ''; + return ` + `; + }).join(''); + + const colEl = document.createElement('div'); + colEl.className = 'rdl-src-col'; + colEl.style.animation = 'fadeSlideUp 0.3s ease both'; + colEl.innerHTML = ` +
+ ${icon} + ${label} + ${candidates.length} +
+
${itemsHtml}
+ `; + columnsEl.appendChild(colEl); + + // Enable the download button + if (startBtn && allCandidates.some(c => !c.blacklisted)) { + startBtn.disabled = false; + startBtn.textContent = 'Download Selected'; + } + + } catch (e) { /* skip malformed lines */ } + } + } + } catch (e) { + if (loadingEl) loadingEl.innerHTML = `
Error: ${_esc(e.message)}
`; } - const bestIdx = srcData.best_candidate_index >= 0 ? srcData.best_candidate_index : 0; + // If no results at all + if (allCandidates.length === 0 && loadingEl) { + loadingEl.innerHTML = '
No download sources found for this track.
'; + } - const candidatesHtml = candidates.map((c, i) => { - const confPct = Math.round((c.confidence || 0) * 100); - const confCls = confPct >= 90 ? 'high' : confPct >= 70 ? 'medium' : 'low'; - const checked = (i === bestIdx && !c.blacklisted) ? 'checked' : ''; - const blClass = c.blacklisted ? ' blacklisted' : ''; + // Store candidates for the download button + window._redownloadCandidates = allCandidates; + + // Wire up download button + const startBtn2 = document.getElementById('redownload-start-btn'); + if (startBtn2) { + startBtn2.addEventListener('click', async () => { + const checked = document.querySelector('input[name="source-choice"]:checked'); + if (!checked) { showToast('Select a download source', 'error'); return; } + const candidate = window._redownloadCandidates[parseInt(checked.value)]; + const deleteOld = document.getElementById('redownload-delete-old-check')?.checked ?? true; + + overlay.querySelectorAll('.redownload-step').forEach(s => s.classList.remove('active')); + overlay.querySelector('.redownload-step[data-step="3"]').classList.add('active'); + + const body = document.getElementById('redownload-body'); + body.innerHTML = ` +
+
Downloading: ${_esc(candidate.display_name)}
+
from ${_esc(candidate.source_service === 'soulseek' ? candidate.username : (candidate.source_service || 'unknown'))}
+
+
Starting download...
+
+ `; + + try { + const res = await fetch(`/api/library/track/${track.id}/redownload/start`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ metadata, candidate, delete_old_file: deleteOld }) + }); + const startData = await res.json(); + if (!startData.success) throw new Error(startData.error); + _pollRedownloadProgress(startData.task_id, overlay); + } catch (e) { + body.innerHTML = `
Download failed: ${_esc(e.message)}
`; + } + }); + } +} + +/* _renderRedownloadStep2 removed โ€” replaced by _streamRedownloadSources above */ +if (false) { + const serviceIcons = { soulseek: '๐Ÿ”', youtube: 'โ–ถ๏ธ', tidal: '๐ŸŒŠ', qobuz: '๐ŸŽต', hifi: '๐ŸŽง', deezer_dl: '๐Ÿ’œ', hybrid: 'โšก' }; + const serviceLabels = { soulseek: 'Soulseek', youtube: 'YouTube', tidal: 'Tidal', qobuz: 'Qobuz', hifi: 'HiFi', deezer_dl: 'Deezer', hybrid: 'Auto' }; + + // Group candidates by source service + const grouped = {}; + candidates.forEach((c, i) => { + c._origIdx = i; // preserve original index for radio value + const svc = c.source_service || 'unknown'; + if (!grouped[svc]) grouped[svc] = []; + grouped[svc].push(c); + }); + + // Build columns โ€” one per source + const sourceColumnsHtml = Object.entries(grouped).map(([svc, items]) => { + const icon = serviceIcons[svc] || '๐Ÿ“ฆ'; + const label = serviceLabels[svc] || svc; + + const itemsHtml = items.slice(0, 10).map(c => { + const confPct = Math.round((c.confidence || 0) * 100); + const confCls = confPct >= 90 ? 'high' : confPct >= 70 ? 'medium' : 'low'; + const isRecommended = c._origIdx === bestIdx && !c.blacklisted; + const checked = isRecommended ? 'checked' : ''; + const blClass = c.blacklisted ? ' blacklisted' : ''; + const dur = c.duration ? `${Math.floor(c.duration / 60000)}:${String(Math.floor((c.duration % 60000) / 1000)).padStart(2, '0')}` : ''; + + return ` + `; + }).join(''); return ` -