danvics/soulsync
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 1

Batch discography completion matching against pre-fetched candidates

Browse source 
Artist detail pages ran check_album_exists_with_editions and check_track_exists
per discography item, each firing 5+ title variations times 3 artist variations
of fuzzy LIKE searches plus fallback broad-artist queries. For a 30-album artist
that was ~450 SQL round-trips just to answer "which of these do I own."

Hoist the artist's library albums and tracks into memory once per request via
two new helpers — get_candidate_albums_for_artist and get_candidate_tracks_for_albums —
and thread them through as optional candidate_albums / candidate_tracks params on
check_album_exists_with_editions, check_album_exists_with_completeness,
check_track_exists, check_album_completion, and check_single_completion.

Batched path scores the same _calculate_album_confidence / _calculate_track_confidence
against the in-memory list, preserving Smart Edition Matching and accuracy.
Title-only cross-artist fallback still fires for collaborative-album edge cases.
None on either param preserves legacy per-item SQL behavior for unaffected callers.

Applied to both /api/library/completion-stream (library artist detail page) and
iter_artist_discography_completion_events (Artists search page). Timing logs
added to confirm the pre-fetch cost and loop elapsed time.

On a Kendrick page load, per-album resolution drops from ~8 seconds to under
the 50ms streaming sleep floor. Observed ~100x SQL reduction on the happy path.
This commit is contained in:
Broque Thomas 2026-04-21 10:47:47 -07:00
parent 739d2f67c1
commit cf143f70af
4 changed files with 281 additions and 98 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

57
core/metadata_service.py
View file

@ -1032,8 +1032,15 @@ def check_album_completion(
artist_name: str,
source_override: Optional[str] = None,
source_chain: Optional[List[str]] = None,
candidate_albums: Optional[List[Any]] = None,
) -> Dict[str, Any]:
"""Check completion status for a single album."""
"""Check completion status for a single album.
When `candidate_albums` is provided, the DB matcher skips per-album SQL
searches and scores every pre-fetched candidate in-memory. Intended for
callers iterating a discography that have already loaded the artist's
full library once via `db.get_candidate_albums_for_artist(...)`.
"""
try:
source_chain = source_chain or _get_completion_source_chain(source_override)
album_name = album_data.get('name', '')
@ -1057,7 +1064,8 @@ def check_album_completion(
artist=artist_name,
expected_track_count=total_tracks if total_tracks > 0 else None,
confidence_threshold=0.7,
server_source=active_server
server_source=active_server,
candidate_albums=candidate_albums
)
except Exception as db_error:
print(f"Database error for album '{album_name}': {db_error}")
@ -1123,8 +1131,16 @@ def check_single_completion(
artist_name: str,
source_override: Optional[str] = None,
source_chain: Optional[List[str]] = None,
candidate_albums: Optional[List[Any]] = None,
candidate_tracks: Optional[List[Any]] = None,
) -> Dict[str, Any]:
"""Check completion status for a single/EP."""
"""Check completion status for a single/EP.
`candidate_albums` applies to the EP branch (treated as an album lookup).
`candidate_tracks` applies to the true-single branch (track-level lookup).
Both are optional; None on either preserves the legacy per-item SQL path
for that branch.
"""
try:
source_chain = source_chain or _get_completion_source_chain(source_override)
single_name = single_data.get('name', '')
@ -1148,7 +1164,8 @@ def check_single_completion(
artist=artist_name,
expected_track_count=total_tracks,
confidence_threshold=0.7,
server_source=active_server
server_source=active_server,
candidate_albums=candidate_albums
)
except Exception as db_error:
print(f"Database error for EP '{single_name}': {db_error}")
@ -1189,7 +1206,8 @@ def check_single_completion(
title=single_name,
artist=artist_name,
confidence_threshold=0.7,
server_source=active_server
server_source=active_server,
candidate_tracks=candidate_tracks
)
except Exception as db_error:
print(f"Database error for single '{single_name}': {db_error}")
@ -1258,12 +1276,35 @@ def iter_artist_discography_completion_events(
total_items = len(albums) + len(singles)
processed_count = 0
# Pre-fetch the artist's library albums AND tracks ONCE so per-item matching
# runs in-memory. Same batching trick as the library completion-stream endpoint.
import time as _time_metadata
candidate_albums = None
candidate_tracks = None
try:
from config.settings import config_manager as _cm_metadata
_active_server = _cm_metadata.get_active_media_server()
_t0 = _time_metadata.perf_counter()
candidate_albums = db.get_candidate_albums_for_artist(resolved_artist_name, server_source=_active_server)
_t1 = _time_metadata.perf_counter()
print(f"[artist-completion-stream] Pre-fetched {len(candidate_albums) if candidate_albums is not None else 0} library albums for '{resolved_artist_name}' in {(_t1 - _t0) * 1000:.0f}ms")
if candidate_albums:
_t2 = _time_metadata.perf_counter()
candidate_tracks = db.get_candidate_tracks_for_albums([a.id for a in candidate_albums])
_t3 = _time_metadata.perf_counter()
print(f"[artist-completion-stream] Pre-fetched {len(candidate_tracks) if candidate_tracks is not None else 0} library tracks in {(_t3 - _t2) * 1000:.0f}ms")
except Exception as _pre_err:
print(f"[artist-completion-stream] Failed to pre-fetch candidates for '{resolved_artist_name}': {_pre_err}")
candidate_albums = None
candidate_tracks = None
yield {
'type': 'start',
'total_items': total_items,
'artist_name': resolved_artist_name,
}
_loop_start = _time_metadata.perf_counter()
for album in albums:
try:
completion_data = check_album_completion(
@ -1272,6 +1313,7 @@ def iter_artist_discography_completion_events(
resolved_artist_name,
source_override=source_override,
source_chain=source_chain,
candidate_albums=candidate_albums,
)
completion_data['type'] = 'album_completion'
completion_data['container_type'] = 'albums'
@ -1295,6 +1337,8 @@ def iter_artist_discography_completion_events(
resolved_artist_name,
source_override=source_override,
source_chain=source_chain,
candidate_albums=candidate_albums,
candidate_tracks=candidate_tracks,
)
completion_data['type'] = 'single_completion'
completion_data['container_type'] = 'singles'
@ -1310,6 +1354,9 @@ def iter_artist_discography_completion_events(
'error': str(e),
}
_loop_elapsed = _time_metadata.perf_counter() - _loop_start
print(f"[artist-completion-stream] Processed {total_items} items for '{resolved_artist_name}' in {_loop_elapsed * 1000:.0f}ms")
yield {
'type': 'complete',
'processed_count': processed_count,

286
database/music_database.py
View file

@ -5371,48 +5371,63 @@ class MusicDatabase:
return list(set(variations))
def check_track_exists(self, title: str, artist: str, confidence_threshold: float = 0.8, server_source: str = None, album: str = None) -> Tuple[Optional[DatabaseTrack], float]:
def check_track_exists(self, title: str, artist: str, confidence_threshold: float = 0.8, server_source: str = None, album: str = None, candidate_tracks: Optional[List[DatabaseTrack]] = None) -> Tuple[Optional[DatabaseTrack], float]:
"""
Check if a track exists in the database with enhanced fuzzy matching and confidence scoring.
Args:
album: Optional album name — enables album-aware matching for multi-artist albums
candidate_tracks: Optional pre-fetched list of tracks to match against in-memory,
skipping the per-variation SQL loop. Intended for callers iterating
a discography that already fetched the artist's tracks once via
get_candidate_tracks_for_albums. None preserves original behavior.
Returns (track, confidence) tuple where confidence is 0.0-1.0
"""
try:
# Generate title variations for better matching (similar to album approach)
title_variations = self._generate_track_title_variations(title)
logger.debug(f"Enhanced track matching for '{title}' by '{artist}': trying {len(title_variations)} variations")
for i, var in enumerate(title_variations):
logger.debug(f" {i+1}. '{var}'")
best_match = None
best_confidence = 0.0
# Try each title variation
for title_variation in title_variations:
# Search for potential matches with this variation
potential_matches = []
artist_variations = self._get_artist_variations(artist)
for artist_variation in artist_variations:
potential_matches.extend(self.search_tracks(title=title_variation, artist=artist_variation, limit=20, server_source=server_source))
if not potential_matches:
continue
logger.debug(f"Found {len(potential_matches)} tracks for variation '{title_variation}'")
# Score each potential match
for track in potential_matches:
if candidate_tracks is not None:
# BATCHED PATH — score every pre-fetched track in-memory.
# _calculate_track_confidence already handles title normalization,
# so no need for the per-variation SQL widening.
logger.debug(f"Enhanced track matching for '{title}' by '{artist}': batched against {len(candidate_tracks)} candidates")
for track in candidate_tracks:
confidence = self._calculate_track_confidence(title, artist, track)
logger.debug(f" '{track.title}' confidence: {confidence:.3f}")
if confidence > best_confidence:
best_confidence = confidence
best_match = track
else:
# LEGACY PATH — generate title variations and fire SQL per variation.
title_variations = self._generate_track_title_variations(title)
logger.debug(f"Enhanced track matching for '{title}' by '{artist}': trying {len(title_variations)} variations")
for i, var in enumerate(title_variations):
logger.debug(f" {i+1}. '{var}'")
# Try each title variation
for title_variation in title_variations:
# Search for potential matches with this variation
potential_matches = []
artist_variations = self._get_artist_variations(artist)
for artist_variation in artist_variations:
potential_matches.extend(self.search_tracks(title=title_variation, artist=artist_variation, limit=20, server_source=server_source))
if not potential_matches:
continue
logger.debug(f"Found {len(potential_matches)} tracks for variation '{title_variation}'")
# Score each potential match
for track in potential_matches:
confidence = self._calculate_track_confidence(title, artist, track)
logger.debug(f" '{track.title}' confidence: {confidence:.3f}")
if confidence > best_confidence:
best_confidence = confidence
best_match = track
# Return match only if it meets threshold
if best_match and best_confidence >= confidence_threshold:
logger.debug(f"Enhanced track match found: '{title}' -> '{best_match.title}' (confidence: {best_confidence:.3f})")
@ -5686,15 +5701,84 @@ class MusicDatabase:
logger.error(f"Error getting album formats: {e}")
return []
def check_album_exists_with_completeness(self, title: str, artist: str, expected_track_count: Optional[int] = None, confidence_threshold: float = 0.8, server_source: Optional[str] = None) -> Tuple[Optional[DatabaseAlbum], float, int, int, bool, List[str]]:
def get_candidate_albums_for_artist(self, artist: str, server_source: Optional[str] = None, limit: int = 200) -> List[DatabaseAlbum]:
"""
Fetch every library album for an artist, merged across artist-name variations
and deduplicated by album ID. Intended to be called once per artist page load
so subsequent per-album matching can run in-memory against this list without
re-hitting SQL for each discography item.
"""
candidates: List[DatabaseAlbum] = []
try:
seen_ids = set()
for artist_var in self._get_artist_variations(artist):
found = self.search_albums(title="", artist=artist_var, limit=limit, server_source=server_source)
for album in found:
if album.id not in seen_ids:
candidates.append(album)
seen_ids.add(album.id)
return candidates
except Exception as e:
logger.error(f"Error fetching candidate albums for artist '{artist}': {e}")
return candidates
def get_candidate_tracks_for_albums(self, album_ids: List) -> List[DatabaseTrack]:
"""
Fetch every track belonging to the given set of album IDs in a single query.
Used for batched track-level completion checks (true singles on discography).
Returns DatabaseTrack objects with artist_name/album_title/server_source attrs
attached, matching the shape produced by search_tracks.
"""
if not album_ids:
return []
try:
conn = self._get_connection()
cursor = conn.cursor()
placeholders = ','.join('?' for _ in album_ids)
cursor.execute(f"""
SELECT t.*, a.name as artist_name, al.title as album_title, al.thumb_url as album_thumb_url
FROM tracks t
JOIN artists a ON a.id = t.artist_id
JOIN albums al ON al.id = t.album_id
WHERE t.album_id IN ({placeholders})
""", list(album_ids))
rows = cursor.fetchall()
tracks: List[DatabaseTrack] = []
for row in rows:
track = DatabaseTrack(
id=row['id'],
album_id=row['album_id'],
artist_id=row['artist_id'],
title=row['title'],
track_number=row['track_number'],
duration=row['duration'],
file_path=row['file_path'],
bitrate=row['bitrate'],
)
# Attach joined fields the same way search_tracks does
track.artist_name = row['artist_name']
track.album_title = row['album_title']
track.album_thumb_url = row['album_thumb_url'] if 'album_thumb_url' in row.keys() else ''
track.server_source = row['server_source'] if 'server_source' in row.keys() else ''
tracks.append(track)
return tracks
except Exception as e:
logger.error(f"Error fetching candidate tracks for {len(album_ids)} album IDs: {e}")
return []
def check_album_exists_with_completeness(self, title: str, artist: str, expected_track_count: Optional[int] = None, confidence_threshold: float = 0.8, server_source: Optional[str] = None, candidate_albums: Optional[List[DatabaseAlbum]] = None) -> Tuple[Optional[DatabaseAlbum], float, int, int, bool, List[str]]:
"""
Check if an album exists in the database with completeness information.
Enhanced to handle edition matching (standard <-> deluxe variants).
Returns (album, confidence, owned_tracks, expected_tracks, is_complete, formats)
When `candidate_albums` is provided (via get_candidate_albums_for_artist),
the matcher runs in-memory against that list instead of firing per-album
SQL searches. `None` preserves the original search-every-time behavior.
"""
try:
# Try enhanced edition-aware matching first with expected track count for Smart Edition Matching
album, confidence = self.check_album_exists_with_editions(title, artist, confidence_threshold, expected_track_count, server_source)
album, confidence = self.check_album_exists_with_editions(title, artist, confidence_threshold, expected_track_count, server_source, candidate_albums=candidate_albums)
if not album:
return None, 0.0, 0, 0, False, []
@ -5708,88 +5792,108 @@ class MusicDatabase:
logger.error(f"Error checking album existence with completeness for '{title}' by '{artist}': {e}")
return None, 0.0, 0, 0, False, []
def check_album_exists_with_editions(self, title: str, artist: str, confidence_threshold: float = 0.8, expected_track_count: Optional[int] = None, server_source: Optional[str] = None) -> Tuple[Optional[DatabaseAlbum], float]:
def check_album_exists_with_editions(self, title: str, artist: str, confidence_threshold: float = 0.8, expected_track_count: Optional[int] = None, server_source: Optional[str] = None, candidate_albums: Optional[List[DatabaseAlbum]] = None) -> Tuple[Optional[DatabaseAlbum], float]:
"""
Enhanced album existence check that handles edition variants.
Matches standard albums with deluxe/platinum/special editions and vice versa.
When `candidate_albums` is provided, the artist-level SQL searches are
skipped and matching runs in-memory against that list — used by callers
that already fetched the artist's full library via
get_candidate_albums_for_artist, so a discography of N items doesn't
trigger N*K SQL queries. The title-only cross-artist fallback for
collaborative albums is preserved in both paths.
"""
try:
# Generate album title variations for edition matching
title_variations = self._generate_album_title_variations(title)
logger.debug(f"Edition matching for '{title}' by '{artist}': trying {len(title_variations)} variations")
for i, var in enumerate(title_variations):
logger.debug(f" {i+1}. '{var}'")
best_match = None
best_confidence = 0.0
for variation in title_variations:
# Search for this variation
albums = []
artist_variations = self._get_artist_variations(artist)
for artist_variation in artist_variations:
found = self.search_albums(title=variation, artist=artist_variation, limit=10, server_source=server_source)
# Deduplicate by ID
existing_ids = {a.id for a in albums}
for album in found:
if album.id not in existing_ids:
albums.append(album)
existing_ids.add(album.id)
if albums:
logger.debug(f"Found {len(albums)} albums for variation '{variation}'")
if not albums:
continue
# Score each potential match with Smart Edition Matching
for album in albums:
if candidate_albums is not None:
# BATCHED PATH — score every pre-fetched candidate in-memory.
# _calculate_album_confidence handles title normalization and
# expected-track-count edition matching, so we don't need the
# per-variation SQL widening that the legacy path does.
logger.debug(f"Edition matching for '{title}' by '{artist}': batched against {len(candidate_albums)} candidates")
for album in candidate_albums:
confidence = self._calculate_album_confidence(title, artist, album, expected_track_count)
logger.debug(f" '{album.title}' confidence: {confidence:.3f}")
if confidence > best_confidence:
best_confidence = confidence
best_match = album
# Return match only if it meets threshold
if best_match and best_confidence >= confidence_threshold:
logger.debug(f"Edition match found: '{title}' -> '{best_match.title}' (confidence: {best_confidence:.3f})")
return best_match, best_confidence
# Fallback: Check ALL albums by this artist (resolves SQL accent sensitivity issues #101)
# If we haven't found a match yet, fetch broader list from artist and double check
if best_confidence < confidence_threshold:
logger.debug(f"specific title search failed, trying broad artist search fallback for '{artist}'")
try:
# Get ALL albums by this artist (limit 100 to be safe)
# This bypasses SQL 'LIKE' limitations for diacritics (e.g. 'ă' vs 'a')
# And relies on Python-side normalization in _calculate_album_confidence
artist_albums = []
else:
# LEGACY PATH — generate title variations and fire SQL per variation.
title_variations = self._generate_album_title_variations(title)
logger.debug(f"Edition matching for '{title}' by '{artist}': trying {len(title_variations)} variations")
for i, var in enumerate(title_variations):
logger.debug(f" {i+1}. '{var}'")
for variation in title_variations:
# Search for this variation
albums = []
artist_variations = self._get_artist_variations(artist)
for artist_var in artist_variations:
found_albums = self.search_albums(title="", artist=artist_var, limit=100, server_source=server_source)
# Deduplicate
existing_ids = {a.id for a in artist_albums}
for album in found_albums:
for artist_variation in artist_variations:
found = self.search_albums(title=variation, artist=artist_variation, limit=10, server_source=server_source)
# Deduplicate by ID
existing_ids = {a.id for a in albums}
for album in found:
if album.id not in existing_ids:
artist_albums.append(album)
albums.append(album)
existing_ids.add(album.id)
if artist_albums:
logger.debug(f" Found {len(artist_albums)} total albums for artist fallback")
for album in artist_albums:
if albums:
logger.debug(f"Found {len(albums)} albums for variation '{variation}'")
if not albums:
continue
# Score each potential match with Smart Edition Matching
for album in albums:
confidence = self._calculate_album_confidence(title, artist, album, expected_track_count)
logger.debug(f" '{album.title}' confidence: {confidence:.3f}")
if confidence > best_confidence:
best_confidence = confidence
best_match = album
logger.debug(f" Fallback match: '{album.title}' confidence: {confidence:.3f}")
except Exception as fallback_error:
logger.warning(f"Fallback artist search failed: {fallback_error}")
# Return match only if it meets threshold
if best_match and best_confidence >= confidence_threshold:
logger.debug(f"Edition match found: '{title}' -> '{best_match.title}' (confidence: {best_confidence:.3f})")
return best_match, best_confidence
# Fallback: Check ALL albums by this artist (resolves SQL accent sensitivity issues #101)
# Only runs in the legacy path — batched callers have already
# fetched this broader list via get_candidate_albums_for_artist.
if best_confidence < confidence_threshold:
logger.debug(f"specific title search failed, trying broad artist search fallback for '{artist}'")
try:
# Get ALL albums by this artist (limit 100 to be safe)
# This bypasses SQL 'LIKE' limitations for diacritics (e.g. 'ă' vs 'a')
# And relies on Python-side normalization in _calculate_album_confidence
artist_albums = []
artist_variations = self._get_artist_variations(artist)
for artist_var in artist_variations:
found_albums = self.search_albums(title="", artist=artist_var, limit=100, server_source=server_source)
# Deduplicate
existing_ids = {a.id for a in artist_albums}
for album in found_albums:
if album.id not in existing_ids:
artist_albums.append(album)
existing_ids.add(album.id)
if artist_albums:
logger.debug(f" Found {len(artist_albums)} total albums for artist fallback")
for album in artist_albums:
confidence = self._calculate_album_confidence(title, artist, album, expected_track_count)
if confidence > best_confidence:
best_confidence = confidence
best_match = album
logger.debug(f" Fallback match: '{album.title}' confidence: {confidence:.3f}")
except Exception as fallback_error:
logger.warning(f"Fallback artist search failed: {fallback_error}")
if best_match and best_confidence >= confidence_threshold:
logger.debug(f"Fallback match succeeded: '{title}' -> '{best_match.title}' (confidence: {best_confidence:.3f})")
logger.debug(f"Match succeeded: '{title}' -> '{best_match.title}' (confidence: {best_confidence:.3f})")
return best_match, best_confidence
# Multi-artist fallback: search by title only (any artist)

35
web_server.py
View file

@ -11803,8 +11803,35 @@ def library_completion_stream():
for item in data.get(cat, []):
all_items.append((cat, item))
# Pre-fetch the artist's library albums AND tracks ONCE so per-item
# matching runs in-memory instead of firing per-item SQL searches.
# Turns N*K queries into 2 broad fetches + N track-count lookups.
from config.settings import config_manager as _cm_cs
_active_server = _cm_cs.get_active_media_server()
candidate_albums = None
candidate_tracks = None
_t0 = time.perf_counter()
try:
candidate_albums = db.get_candidate_albums_for_artist(artist_name, server_source=_active_server)
except Exception as _cand_err:
print(f"[completion-stream] Failed to pre-fetch album candidates for '{artist_name}': {_cand_err}")
candidate_albums = None
_t1 = time.perf_counter()
print(f"[completion-stream] Pre-fetched {len(candidate_albums) if candidate_albums is not None else 0} library albums for '{artist_name}' in {(_t1 - _t0) * 1000:.0f}ms")
if candidate_albums:
_t2 = time.perf_counter()
try:
candidate_tracks = db.get_candidate_tracks_for_albums([a.id for a in candidate_albums])
except Exception as _tr_err:
print(f"[completion-stream] Failed to pre-fetch track candidates for '{artist_name}': {_tr_err}")
candidate_tracks = None
_t3 = time.perf_counter()
print(f"[completion-stream] Pre-fetched {len(candidate_tracks) if candidate_tracks is not None else 0} library tracks in {(_t3 - _t2) * 1000:.0f}ms")
yield f"data: {json.dumps({'type': 'start', 'total_items': len(all_items)})}\n\n"
_loop_start = time.perf_counter()
for i, (category, item) in enumerate(all_items):
try:
# Map Library field names to helper field names
@ -11816,9 +11843,9 @@ def library_completion_stream():
}
if category == 'singles':
result = check_single_completion(db, mapped, artist_name, source_override=source_override)
result = check_single_completion(db, mapped, artist_name, source_override=source_override, candidate_albums=candidate_albums, candidate_tracks=candidate_tracks)
else:
result = check_album_completion(db, mapped, artist_name, source_override=source_override)
result = check_album_completion(db, mapped, artist_name, source_override=source_override, candidate_albums=candidate_albums)
result['id'] = item['id']
result['category'] = category
@ -11829,6 +11856,10 @@ def library_completion_stream():
time.sleep(0.05) # 50ms between items for visible streaming
_loop_elapsed = time.perf_counter() - _loop_start
_sleep_floor = 0.05 * len(all_items)
print(f"[completion-stream] Processed {len(all_items)} items for '{artist_name}' in {_loop_elapsed * 1000:.0f}ms (sleep floor: {_sleep_floor * 1000:.0f}ms)")
yield f"data: {json.dumps({'type': 'complete', 'processed_count': len(all_items)})}\n\n"
except Exception as e:

1
webui/static/helper.js
View file

@ -3602,6 +3602,7 @@ const WHATS_NEW = {
'2.35': [
// --- April 21, 2026 ---
{ date: 'April 21, 2026' },
{ title: 'Massively Faster Artist Detail Page Loads', desc: 'Artist discography completion checks used to fire hundreds of SQL queries per page load — 15+ fuzzy title/artist searches per album times 30 albums per artist. Now pre-fetches the artist\'s library albums and tracks ONCE upfront, then matches everything in-memory. Same matching logic and accuracy, roughly 100x fewer SQL round-trips. Applies to both the Library artist page and the Artists search page', page: 'library' },
{ title: 'Fix Reorganize All Ignoring Album Type', desc: 'Reorganize All was sending every album — EPs, singles, and compilations — into the "Albums" folder because the $albumtype template variable silently defaulted to "Album". The variable is now resolved from the album\'s record_type (with track-count fallback) so ${albumtype}s produces the expected Albums/Singles/EPs/Compilations split', page: 'library' },
// --- April 20, 2026 ---