From 871feb3997bf3f0a1bfa84371f07b0d0b3436a0b Mon Sep 17 00:00:00 2001 From: Broque Thomas <26755000+Nezreka@users.noreply.github.com> Date: Sun, 24 May 2026 22:31:34 -0700 Subject: [PATCH] Speed up playlist sync with a lazy per-artist track pool MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Syncing a playlist where most tracks weren't in the library was burning ~30 SQL queries per missed track. `services/sync_service.py` walked each Spotify track through `check_track_exists` with no `candidate_tracks`, hitting the legacy title-variation × artist-variation grid in `database/music_database.py:6041-6069` for every miss. The `sync_match_cache` only covered matches, so misses re-paid the full lookup cost every sync. A 30-track playlist with a 30% match rate (Discover Weekly was 9/30 in the test run) was taking ~4m14s, almost entirely in the matching phase. `check_track_exists` already accepts a `candidate_tracks` kwarg that skips the SQL widening and scores against an in-memory list (the batched path at `music_database.py:6031`, originally added for artist discography iteration). The sync service just wasn't using it. This commit wires that path in via a lazy per-artist pool: - `sync_playlist` creates an empty `candidate_pool` dict and passes it to each `_find_track_in_media_server` call. - `_get_or_fetch_artist_candidates` runs SQL for an artist only on the first track that needs them — playlists where every track is already in `sync_match_cache` pay zero pool cost (no upfront delay). - Subsequent misses for the same artist hit the memoized list and skip the per-variation SQL grid. - Artists with no library tracks still get a cached empty list, which triggers the batched path's instant short-circuit instead of falling into the SQL widening. - Any pool fetch failure returns None so the caller falls through to the original per-track SQL loop, so the worst case is the old behavior, never a regression. On a 30-track / 25-unique-artist playlist with a cold cache the SQL fan-out drops from ~900 queries to ~25; with a warm cache it drops to zero (no pool fetches at all). Applies to every entry point that goes through `sync_playlist`: manual sync, auto-sync schedules, the `playlist_pipeline` automation action, and the Sync All button. --- services/sync_service.py | 60 ++++++++++++++++++++++++++++++++++------ webui/static/helper.js | 1 + 2 files changed, 53 insertions(+), 8 deletions(-) diff --git a/services/sync_service.py b/services/sync_service.py index 5177c59d..7ded8400 100644 --- a/services/sync_service.py +++ b/services/sync_service.py @@ -213,13 +213,21 @@ class PlaylistSyncService: media_client, server_type = self._get_active_media_client() self._update_progress(playlist.name, f"Matching tracks against {server_type.title()} library", "", 20, 5, 2, total_tracks=total_tracks) - + + # Per-artist track pool, populated lazily inside _find_track_in_media_server. + # Only tracks that miss the sync_match_cache fast-path trigger a pool fetch + # for their artist — so warm-cache playlists pay zero pool cost. Misses + # for the same artist later in the playlist reuse the cached list and skip + # the per-variation SQL grid in check_track_exists. Empty dict (not None) + # to signal that pooling is enabled for this sync. + candidate_pool: Dict[str, list] = {} + # Use the same robust matching approach as "Download Missing Tracks" match_results = [] for i, track in enumerate(playlist.tracks): if self._cancelled: return self._create_error_result(playlist.name, ["Sync cancelled"]) - + # Update progress for each track progress_percent = 20 + (40 * (i + 1) / total_tracks) # 20-60% for matching # Extract artist name from both string and dict formats @@ -229,13 +237,13 @@ class PlaylistSyncService: current_track_name = f"{artist_name} - {track.name}" else: current_track_name = track.name - self._update_progress(playlist.name, "Matching tracks", current_track_name, progress_percent, 5, 2, + self._update_progress(playlist.name, "Matching tracks", current_track_name, progress_percent, 5, 2, total_tracks=total_tracks, matched_tracks=len([r for r in match_results if r.is_match]), failed_tracks=len([r for r in match_results if not r.is_match])) - + # Use the robust search approach - plex_match, confidence = await self._find_track_in_media_server(track) + plex_match, confidence = await self._find_track_in_media_server(track, candidate_pool=candidate_pool) match_result = MatchResult( spotify_track=track, @@ -469,7 +477,36 @@ class PlaylistSyncService: self.clear_progress_callback(playlist.name) self._cancelled = False - async def _find_track_in_media_server(self, spotify_track: SpotifyTrack) -> Tuple[Optional[TrackInfo], float]: + def _get_or_fetch_artist_candidates(self, candidate_pool: Optional[Dict[str, list]], db, artist_name: str, active_server) -> Optional[list]: + """Lazy per-artist pool fetch. Only fires SQL when a track for this artist + actually missed the sync_match_cache fast-path — playlists where every + track is cached pay zero pool cost. Returns the candidate list (possibly + empty) on success; returns None when pooling is disabled so the caller + falls back to the legacy per-track SQL loop. + """ + if candidate_pool is None: + return None + key = db._normalize_for_comparison(artist_name) + if key in candidate_pool: + return candidate_pool[key] + try: + candidates = db.search_tracks( + artist=artist_name, + limit=10000, + server_source=active_server, + ) + # Cache the empty result too — same key is asked once per artist this + # sync, then never again. Empty list still triggers the batched path + # in check_track_exists, which short-circuits without firing SQL. + candidate_pool[key] = candidates or [] + return candidate_pool[key] + except Exception as fetch_err: + logger.debug(f"Candidate pool fetch failed for '{artist_name}': {fetch_err}") + # Don't cache the failure — let a later artist for the same key retry, + # and let this call's check_track_exists fall through to legacy SQL. + return None + + async def _find_track_in_media_server(self, spotify_track: SpotifyTrack, candidate_pool: Optional[Dict[str, list]] = None) -> Tuple[Optional[TrackInfo], float]: """Find a track using the same improved database matching as Download Missing Tracks modal""" try: # Check active media server connection @@ -477,7 +514,7 @@ class PlaylistSyncService: if not media_client or not media_client.is_connected(): logger.warning(f"{server_type.upper()} client not connected") return None, 0.0 - + # Use the SAME improved database matching as PlaylistTrackAnalysisWorker from database.music_database import MusicDatabase from config.settings import config_manager @@ -542,7 +579,14 @@ class PlaylistSyncService: # Use the improved database check_track_exists method with server awareness try: db = MusicDatabase() - db_track, confidence = db.check_track_exists(original_title, artist_name, confidence_threshold=0.7, server_source=active_server) + artist_candidates = self._get_or_fetch_artist_candidates( + candidate_pool, db, artist_name, active_server, + ) + db_track, confidence = db.check_track_exists( + original_title, artist_name, + confidence_threshold=0.7, server_source=active_server, + candidate_tracks=artist_candidates, + ) if db_track and confidence >= 0.7: logger.debug(f"Database match found for '{original_title}' by '{artist_name}': '{db_track.title}' with confidence {confidence:.2f}") diff --git a/webui/static/helper.js b/webui/static/helper.js index 1e304a8b..c6859090 100644 --- a/webui/static/helper.js +++ b/webui/static/helper.js @@ -3415,6 +3415,7 @@ function closeHelperSearch() { const WHATS_NEW = { '2.6.2': [ { date: 'May 24, 2026 — 2.6.2 release' }, + { title: 'Playlist sync is way faster on partial-match playlists', desc: 'syncing a playlist where most tracks weren\'t in the library used to take forever — a 30-track playlist with only 9 matches was burning 4+ minutes because every unmatched track ran the full title-variation × artist-variation SQL grid against the tracks table (~30 SQL queries per missed track). cache only covered matches, so misses re-paid the cost every sync. sync now uses a per-artist track pool that fills in lazily — only tracks that miss the sync match cache trigger a one-time fetch of their artist\'s library tracks, and later misses for the same artist reuse the in-memory list. playlists where every track is already cached pay zero pool cost (no upfront delay). benefits every sync entry point — manual, auto-sync, the playlist_pipeline automation action, and the Sync All button.' }, { title: 'Fix: Auto-Sync "next in 8h" timezone bug', desc: 'scheduled Auto-Sync playlists were all showing "next in 8h" regardless of the interval — Every 1 hour, Every 1 day, anything. backend stores next-run as a naive UTC string and the frontend was parsing it as local time, which on Pacific time offset the displayed countdown by ~8 hours. Auto-Sync now uses the existing UTC-aware parser the rest of the Automations page already uses. as a separate correctness fix, the automation update endpoint also now blanks the stored next-run whenever the trigger type or trigger config changes, so the engine recomputes from scratch instead of preserving a leftover timestamp from the previous schedule.' }, { title: 'Auto-Sync modal restyled to your accent color', desc: 'the Playlist Auto-Sync manager now picks up your chosen accent (the same one used everywhere else in the app) instead of the hardcoded sky-blue palette it shipped with. tabs, drop zones, scheduled-playlist chips, scrollbars, and the modal glow all follow the accent theme. drop targets also light up clearly while dragging.' }, ],