From 1bb59f3c24c220cac8faa5bf7c17403b62808a72 Mon Sep 17 00:00:00 2001 From: Broque Thomas <26755000+Nezreka@users.noreply.github.com> Date: Thu, 2 Apr 2026 08:48:19 -0700 Subject: [PATCH] Stream enhanced search source results as NDJSON for progressive rendering MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Source endpoint now streams artists/albums/tracks as separate NDJSON lines as each search type completes — iTunes users see artists in ~3s instead of waiting 9+ seconds for all 3 rate-limited calls to finish - Enhanced search _fetchAlternateSource reads stream with ReadableStream reader, merges each chunk into source data, re-renders tabs immediately - Global search uses same streaming pattern via _gsFetchSourceStream - No data loss: streamed data merges incrementally, primary response preserves already-received alternate source data --- web_server.py | 58 +++++++++++++++++++++-- webui/static/script.js | 104 ++++++++++++++++++++++++++++++++++------- 2 files changed, 141 insertions(+), 21 deletions(-) diff --git a/web_server.py b/web_server.py index 92ba2eb8..57ed841e 100644 --- a/web_server.py +++ b/web_server.py @@ -7349,8 +7349,9 @@ def _enhanced_search_source(query, client): def enhanced_search_source(source_name): """Fetch search results from a specific alternate metadata source. - Called asynchronously by the frontend after the primary search returns. - Each source tab fires its own request so slow sources don't block fast ones. + Streams NDJSON — one line per search type (artists, albums, tracks) as each completes. + This prevents slow sources (iTunes with 3s rate limit) from blocking the UI. + Falls back to single JSON response if streaming not supported. """ if source_name not in ('spotify', 'itunes', 'deezer', 'hydrabase'): return jsonify({"error": f"Unknown source: {source_name}"}), 400 @@ -7378,8 +7379,57 @@ def enhanced_search_source(source_name): else: return jsonify({"artists": [], "albums": [], "tracks": [], "available": False}) - result = _enhanced_search_source(query, client) - return jsonify(result) + def generate(): + # Stream each search type as it completes + try: + artist_objs = client.search_artists(query, limit=10) + artists = [] + for artist in artist_objs: + artists.append({ + "id": artist.id, "name": artist.name, + "image_url": artist.image_url, + "external_urls": artist.external_urls or {}, + }) + yield json.dumps({"type": "artists", "data": artists}) + "\n" + except Exception as e: + logger.debug(f"Artist search failed for {source_name}: {e}") + yield json.dumps({"type": "artists", "data": []}) + "\n" + + try: + album_objs = client.search_albums(query, limit=10) + albums = [] + for album in album_objs: + artist_name = ', '.join(album.artists) if album.artists else 'Unknown Artist' + albums.append({ + "id": album.id, "name": album.name, "artist": artist_name, + "image_url": album.image_url, "release_date": album.release_date, + "total_tracks": album.total_tracks, "album_type": album.album_type, + "external_urls": album.external_urls or {}, + }) + yield json.dumps({"type": "albums", "data": albums}) + "\n" + except Exception as e: + logger.warning(f"Album search failed for {source_name}: {e}") + yield json.dumps({"type": "albums", "data": []}) + "\n" + + try: + track_objs = client.search_tracks(query, limit=10) + tracks = [] + for track in track_objs: + artist_name = ', '.join(track.artists) if track.artists else 'Unknown Artist' + tracks.append({ + "id": track.id, "name": track.name, "artist": artist_name, + "album": track.album, "duration_ms": track.duration_ms, + "image_url": track.image_url, "release_date": track.release_date, + "external_urls": track.external_urls or {}, + }) + yield json.dumps({"type": "tracks", "data": tracks}) + "\n" + except Exception as e: + logger.warning(f"Track search failed for {source_name}: {e}") + yield json.dumps({"type": "tracks", "data": []}) + "\n" + + yield json.dumps({"type": "done"}) + "\n" + + return app.response_class(generate(), mimetype='application/x-ndjson') except Exception as e: logger.error(f"Enhanced search source ({source_name}) error: {e}") return jsonify({"artists": [], "albums": [], "tracks": [], "available": False}) diff --git a/webui/static/script.js b/webui/static/script.js index 0f7e84c8..bd355eab 100644 --- a/webui/static/script.js +++ b/webui/static/script.js @@ -8295,17 +8295,50 @@ function initializeSearchModeToggle() { signal: _altSourceController?.signal, }); if (!response.ok) return; - const result = await response.json(); - if (!result.available) return; - // Store in multi-source state - if (_enhancedSearchData) { - _enhancedSearchData.sources[sourceName] = result; - // Re-render tabs if primary has loaded (primary_source is set) - if (_enhancedSearchData.primary_source) { - renderSourceTabs(_enhancedSearchData); + // Stream NDJSON — render each search type (artists, albums, tracks) as it arrives + if (!_enhancedSearchData) return; + if (!_enhancedSearchData.sources[sourceName]) { + _enhancedSearchData.sources[sourceName] = { artists: [], albums: [], tracks: [], available: true }; + } + const sourceData = _enhancedSearchData.sources[sourceName]; + + const reader = response.body.getReader(); + const decoder = new TextDecoder(); + let buffer = ''; + + while (true) { + const { done, value } = await reader.read(); + if (done) break; + buffer += decoder.decode(value, { stream: true }); + + let newlineIdx; + while ((newlineIdx = buffer.indexOf('\n')) !== -1) { + const line = buffer.slice(0, newlineIdx).trim(); + buffer = buffer.slice(newlineIdx + 1); + if (!line) continue; + + try { + const chunk = JSON.parse(line); + if (chunk.type === 'artists') sourceData.artists = chunk.data; + else if (chunk.type === 'albums') sourceData.albums = chunk.data; + else if (chunk.type === 'tracks') sourceData.tracks = chunk.data; + else if (chunk.type === 'done') break; + + // Re-render tabs after each chunk + if (_enhancedSearchData.primary_source) { + renderSourceTabs(_enhancedSearchData); + } + } catch (parseErr) { + console.debug(`NDJSON parse error for ${sourceName}:`, parseErr); + } } } + + // Final render + if (_enhancedSearchData.primary_source) { + renderSourceTabs(_enhancedSearchData); + } } catch (e) { if (e.name !== 'AbortError') { console.debug(`Alternate source ${sourceName} failed:`, e); @@ -16973,24 +17006,61 @@ async function _gsPerformSearch(query) { // Async library ownership check — adds badges + swaps play buttons for library tracks setTimeout(() => _gsLibraryCheck(), 200); - // Fetch alternate sources + // Fetch alternate sources — stream NDJSON so slow sources render incrementally const alts = data.alternate_sources || []; for (const src of alts) { if (src === _gsState.activeSource) continue; - fetch(`/api/enhanced-search/source/${src}`, { - method: 'POST', - headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify({ query }), - signal: _gsState.altAbortCtrl.signal, - }).then(r => r.json()).then(altData => { - if (altData.available) { _gsState.sources[src] = altData; _gsRenderTabs(); } - }).catch(() => {}); + _gsFetchSourceStream(src, query); } } catch (e) { if (e.name !== 'AbortError') results.innerHTML = '
Search failed
'; } } +async function _gsFetchSourceStream(src, query) { + try { + const res = await fetch(`/api/enhanced-search/source/${src}`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ query }), + signal: _gsState.altAbortCtrl.signal, + }); + if (!res.ok) return; + + if (!_gsState.sources[src]) { + _gsState.sources[src] = { artists: [], albums: [], tracks: [], available: true }; + } + const sourceData = _gsState.sources[src]; + + const reader = res.body.getReader(); + const decoder = new TextDecoder(); + let buffer = ''; + + while (true) { + const { done, value } = await reader.read(); + if (done) break; + buffer += decoder.decode(value, { stream: true }); + + let idx; + while ((idx = buffer.indexOf('\n')) !== -1) { + const line = buffer.slice(0, idx).trim(); + buffer = buffer.slice(idx + 1); + if (!line) continue; + try { + const chunk = JSON.parse(line); + if (chunk.type === 'artists') sourceData.artists = chunk.data; + else if (chunk.type === 'albums') sourceData.albums = chunk.data; + else if (chunk.type === 'tracks') sourceData.tracks = chunk.data; + _gsRenderTabs(); + } catch (e) {} + } + } + _gsRenderTabs(); + } catch (e) { + if (e.name !== 'AbortError') console.debug(`GS alt source ${src} failed:`, e); + } +} + function _gsRender(data) { const results = document.getElementById('gsearch-results'); if (!results) return;