Persist mirrored playlist discovery results & retry failed

web_server.py

@@ -21531,7 +21531,12 @@ def start_youtube_discovery(url_hash):
         state['status'] = 'discovering'
         state['discovery_progress'] = 0
         state['spotify_matches'] = 0
-        
+        state['discovery_results'] = []
+
+        # Clear skip_discovery flags on all tracks (in case of prior retry)
+        for track in state['playlist']['tracks']:
+            track.pop('skip_discovery', None)
+
         # Add activity for discovery start
         playlist_name = state['playlist']['name']
         track_count = len(state['playlist']['tracks'])
@@ -21635,6 +21640,40 @@ def update_youtube_discovery_match():
         print(f"✅ Manual match updated: youtube - {identifier} - track {track_index}")
         print(f"   → {result['spotify_artist']} - {result['spotify_track']}")
 
+        # Persist manual fix to DB for mirrored playlists
+        if identifier.startswith('mirrored_'):
+            try:
+                tracks = state['playlist']['tracks']
+                if track_index < len(tracks):
+                    db_track_id = tracks[track_index].get('db_track_id')
+                    if db_track_id:
+                        db = get_database()
+                        artists_list = spotify_track['artists']
+                        if isinstance(artists_list, list):
+                            artists_list = [{'name': a} if isinstance(a, str) else a for a in artists_list]
+                        album_raw = spotify_track.get('album', '')
+                        album_obj = album_raw if isinstance(album_raw, dict) else {'name': album_raw or ''}
+                        matched_data = {
+                            'id': spotify_track['id'],
+                            'name': spotify_track['name'],
+                            'artists': artists_list,
+                            'album': album_obj,
+                            'duration_ms': spotify_track.get('duration_ms', 0),
+                            'source': 'spotify',
+                        }
+                        extra_data = {
+                            'discovered': True,
+                            'provider': 'spotify',
+                            'confidence': 1.0,
+                            'matched_data': matched_data,
+                            'manual_match': True,
+                        }
+                        db.update_mirrored_track_extra_data(db_track_id, extra_data)
+                        result['matched_data'] = matched_data
+                        print(f"💾 Persisted manual fix to DB for track {db_track_id}")
+            except Exception as wb_err:
+                print(f"⚠️ Error persisting manual fix to DB: {wb_err}")
+
         return jsonify({'success': True, 'result': result})
 
     except Exception as e:
@@ -21668,6 +21707,10 @@ def _run_youtube_discovery_worker(url_hash):
                 # Update progress
                 state['discovery_progress'] = int((i / len(tracks)) * 100)
 
+                # Skip tracks flagged by retry (already found)
+                if track.get('skip_discovery'):
+                    continue
+
                 # Search for track using active provider
                 cleaned_title = track['name']
                 cleaned_artist = track['artists'][0] if track['artists'] else 'Unknown Artist'
@@ -21892,6 +21935,43 @@ def _run_youtube_discovery_worker(url_hash):
         state['status'] = 'complete'
         state['discovery_progress'] = 100
 
+        # Sort results by index so array position matches result['index'].
+        # Critical after retry where found results are kept at the front
+        # and newly-discovered results are appended out of order.
+        state['discovery_results'].sort(key=lambda r: r.get('index', 0))
+
+        # Write back discovery results to DB for mirrored playlists
+        if url_hash.startswith('mirrored_'):
+            try:
+                db = get_database()
+                for result in state['discovery_results']:
+                    idx = result.get('index', -1)
+                    if idx < 0 or idx >= len(tracks):
+                        continue
+                    db_track_id = tracks[idx].get('db_track_id')
+                    if not db_track_id:
+                        continue
+                    if result.get('status_class') == 'found' and result.get('matched_data'):
+                        extra_data = {
+                            'discovered': True,
+                            'provider': result.get('discovery_source', discovery_source),
+                            'confidence': result.get('confidence', 0),
+                            'matched_data': result['matched_data'],
+                        }
+                        if result.get('manual_match'):
+                            extra_data['manual_match'] = True
+                        db.update_mirrored_track_extra_data(db_track_id, extra_data)
+                    else:
+                        extra_data = {
+                            'discovered': False,
+                            'discovery_attempted': True,
+                            'provider': discovery_source,
+                        }
+                        db.update_mirrored_track_extra_data(db_track_id, extra_data)
+                print(f"💾 Wrote discovery results to DB for {url_hash}")
+            except Exception as wb_err:
+                print(f"⚠️ Error writing discovery results to DB: {wb_err}")
+
         playlist_name = playlist['name']
         source_label = 'Spotify' if use_spotify else 'iTunes'
         add_activity_item("✅", f"YouTube Discovery Complete ({source_label})", f"'{playlist_name}' - {state['spotify_matches']}/{len(tracks)} tracks found", "Now")
@@ -29820,13 +29900,97 @@ def prepare_mirrored_discovery(playlist_id):
         url_hash = f"mirrored_{playlist_id}"
 
         # Build track list in the format the YouTube discovery worker expects
-        tracks = [{
+        tracks = []
-            'id': t.get('source_track_id') or f"mirrored_{t['id']}",
+        for t in tracks_data:
-            'name': t['track_name'],
+            # Parse extra_data if present
-            'artists': [t['artist_name']],
+            extra = None
-            'album': t.get('album_name', ''),
+            if t.get('extra_data'):
-            'duration_ms': t.get('duration_ms', 0)
+                try:
-        } for t in tracks_data]
+                    extra = json.loads(t['extra_data']) if isinstance(t['extra_data'], str) else t['extra_data']
+                except (json.JSONDecodeError, TypeError):
+                    pass
+            tracks.append({
+                'id': t.get('source_track_id') or f"mirrored_{t['id']}",
+                'db_track_id': t['id'],
+                'name': t['track_name'],
+                'artists': [t['artist_name']],
+                'album': t.get('album_name', ''),
+                'duration_ms': t.get('duration_ms', 0),
+                'extra_data': extra,
+            })
+
+        # Check for cached discovery results in extra_data
+        pre_discovered_results = []
+        pre_discovered_count = 0
+        has_pending = False
+
+        for idx, track in enumerate(tracks):
+            extra = track.get('extra_data')
+            if extra and extra.get('discovered'):
+                # Previously found match
+                matched = extra.get('matched_data', {})
+                artists_raw = matched.get('artists', [])
+                if artists_raw and isinstance(artists_raw[0], dict):
+                    artist_str = ', '.join(a.get('name', '') for a in artists_raw)
+                else:
+                    artist_str = ', '.join(str(a) for a in artists_raw) if artists_raw else ''
+                album_raw = matched.get('album', '')
+                album_str = album_raw.get('name', '') if isinstance(album_raw, dict) else (str(album_raw) if album_raw else '')
+                dur = track.get('duration_ms', 0)
+                result = {
+                    'index': idx,
+                    'yt_track': track['name'],
+                    'yt_artist': track['artists'][0] if track['artists'] else 'Unknown',
+                    'status': '✅ Found',
+                    'status_class': 'found',
+                    'spotify_track': matched.get('name', ''),
+                    'spotify_artist': artist_str,
+                    'spotify_album': album_str,
+                    'duration': f"{dur // 60000}:{(dur % 60000) // 1000:02d}" if dur else '0:00',
+                    'discovery_source': extra.get('provider', 'spotify'),
+                    'confidence': extra.get('confidence', 0),
+                    'matched_data': matched,
+                    'spotify_data': matched,
+                }
+                if extra.get('manual_match'):
+                    result['manual_match'] = True
+                pre_discovered_results.append(result)
+                pre_discovered_count += 1
+            elif extra and extra.get('discovery_attempted'):
+                # Previously attempted but not found
+                dur = track.get('duration_ms', 0)
+                pre_discovered_results.append({
+                    'index': idx,
+                    'yt_track': track['name'],
+                    'yt_artist': track['artists'][0] if track['artists'] else 'Unknown',
+                    'status': '❌ Not Found',
+                    'status_class': 'not-found',
+                    'spotify_track': '',
+                    'spotify_artist': '',
+                    'spotify_album': '',
+                    'duration': f"{dur // 60000}:{(dur % 60000) // 1000:02d}" if dur else '0:00',
+                    'discovery_source': extra.get('provider', 'spotify'),
+                    'confidence': 0,
+                })
+            elif not extra or (not extra.get('discovered') and not extra.get('discovery_attempted')):
+                # New track — no discovery data yet
+                has_pending = True
+                dur = track.get('duration_ms', 0)
+                pre_discovered_results.append({
+                    'index': idx,
+                    'yt_track': track['name'],
+                    'yt_artist': track['artists'][0] if track['artists'] else 'Unknown',
+                    'status': '🆕 Pending',
+                    'status_class': 'not-found',
+                    'spotify_track': '',
+                    'spotify_artist': '',
+                    'spotify_album': '',
+                    'duration': f"{dur // 60000}:{(dur % 60000) // 1000:02d}" if dur else '0:00',
+                    'confidence': 0,
+                })
+
+        # Only treat as cached if at least one track has been discovered or attempted
+        has_cached = any(t.get('extra_data') and (t['extra_data'].get('discovered') or t['extra_data'].get('discovery_attempted')) for t in tracks)
 
         playlist_data = {
             'id': url_hash,
@@ -29839,12 +30003,12 @@ def prepare_mirrored_discovery(playlist_id):
 
         youtube_playlist_states[url_hash] = {
             'playlist': playlist_data,
-            'phase': 'fresh',
+            'phase': 'discovered' if has_cached else 'fresh',
-            'discovery_results': [],
+            'discovery_results': pre_discovered_results if has_cached else [],
-            'discovery_progress': 0,
+            'discovery_progress': 100 if has_cached else 0,
-            'spotify_matches': 0,
+            'spotify_matches': pre_discovered_count if has_cached else 0,
             'spotify_total': len(tracks),
-            'status': 'parsed',
+            'status': 'complete' if has_cached else 'parsed',
             'url': playlist_data['url'],
             'sync_playlist_id': None,
             'converted_spotify_playlist_id': None,
@@ -29855,12 +30019,88 @@ def prepare_mirrored_discovery(playlist_id):
             'sync_progress': {}
         }
 
-        logger.info(f"Prepared mirrored playlist for discovery: {playlist['name']} ({len(tracks)} tracks)")
+        logger.info(f"Prepared mirrored playlist for discovery: {playlist['name']} ({len(tracks)} tracks, cached={has_cached}, matches={pre_discovered_count})")
-        return jsonify({"success": True, "url_hash": url_hash})
+        return jsonify({
+            "success": True,
+            "url_hash": url_hash,
+            "from_cache": has_cached,
+            "cached_matches": pre_discovered_count,
+            "total_tracks": len(tracks),
+            "has_pending": has_pending,
+        })
     except Exception as e:
         logger.error(f"Error preparing mirrored discovery: {e}")
         return jsonify({"error": str(e)}), 500
 
+@app.route('/api/mirrored-playlists/<int:playlist_id>/retry-failed-discovery', methods=['POST'])
+def retry_failed_mirrored_discovery(playlist_id):
+    """Re-run discovery only for tracks that failed or are pending in a mirrored playlist."""
+    try:
+        url_hash = f"mirrored_{playlist_id}"
+        state = youtube_playlist_states.get(url_hash)
+        if not state:
+            return jsonify({"error": "Discovery state not found. Run discovery first."}), 404
+
+        if state.get('phase') == 'discovering':
+            return jsonify({"error": "Discovery already in progress"}), 400
+
+        tracks = state['playlist']['tracks']
+        results = state.get('discovery_results', [])
+
+        # Build set of found track indices
+        found_indices = set()
+        kept_results = []
+        for r in results:
+            if r.get('status_class') == 'found':
+                found_indices.add(r.get('index', -1))
+                kept_results.append(r)
+
+        already_found = len(found_indices)
+        retry_count = len(tracks) - already_found
+
+        if retry_count == 0:
+            return jsonify({"success": True, "retry_count": 0, "already_found": already_found, "message": "All tracks already found"})
+
+        # Flag found tracks to skip, clear flag on others
+        for i, track in enumerate(tracks):
+            track['skip_discovery'] = (i in found_indices)
+
+        # Keep only found results, remove failed/pending
+        state['discovery_results'] = kept_results
+        state['phase'] = 'discovering'
+        state['status'] = 'discovering'
+        state['discovery_progress'] = 0
+        # spotify_matches stays at found count (already_found)
+        state['spotify_matches'] = already_found
+
+        # Clear discovery_attempted in DB for failed tracks so they're retryable
+        try:
+            db = get_database()
+            for i, track in enumerate(tracks):
+                if i not in found_indices:
+                    db_track_id = track.get('db_track_id')
+                    if db_track_id:
+                        db.update_mirrored_track_extra_data(db_track_id, {
+                            'discovered': False,
+                            'discovery_attempted': False,
+                        })
+        except Exception as db_err:
+            print(f"⚠️ Error clearing discovery_attempted in DB: {db_err}")
+
+        # Submit worker
+        future = youtube_discovery_executor.submit(_run_youtube_discovery_worker, url_hash)
+        state['discovery_future'] = future
+
+        print(f"🔄 Retrying failed discovery for {url_hash}: {retry_count} tracks to retry, {already_found} already found")
+        return jsonify({
+            "success": True,
+            "retry_count": retry_count,
+            "already_found": already_found,
+        })
+    except Exception as e:
+        logger.error(f"Error retrying failed discovery: {e}")
+        return jsonify({"error": str(e)}), 500
+
 @app.route('/api/mirrored-playlists/discovery-states', methods=['GET'])
 def get_mirrored_discovery_states():
     """Return discovery states for any mirrored playlists that have active/completed discoveries."""

webui/static/script.js

@@ -22681,6 +22681,15 @@ function getModalActionButtons(urlHash, phase, state = null) {
                 }
             }
 
+            // Retry Failed button for mirrored playlists
+            if (state && state.is_mirrored_playlist) {
+                const results = state.discovery_results || state.discoveryResults || [];
+                const failedCount = results.filter(r => r.status_class !== 'found').length;
+                if (failedCount > 0) {
+                    buttons += `<button class="modal-btn modal-btn-secondary" onclick="retryFailedMirroredDiscovery('${urlHash}')">🔄 Retry Failed (${failedCount})</button>`;
+                }
+            }
+
             if (!buttons) {
                 buttons = `<div class="modal-info">ℹ️ No Spotify matches found. Discovery complete but no tracks could be matched.</div>`;
             }
@@ -43291,30 +43300,68 @@ async function discoverMirroredPlaylist(playlistId) {
             duration_ms: t.duration_ms || 0
         }));
 
-        // Create state entry reusing the YouTube discovery infrastructure
+        // Check if backend returned cached results
-        youtubePlaylistStates[tempHash] = {
+        if (prepData.from_cache) {
-            playlist: {
+            // Fetch the pre-populated status from the backend
-                name: data.name,
+            const statusRes = await fetch(`/api/youtube/discovery/status/${tempHash}`);
-                tracks: tracks,
+            const statusData = await statusRes.json();
-                track_count: tracks.length
+            if (statusData.error) throw new Error(statusData.error);
-            },
+
-            phase: 'fresh',
+            youtubePlaylistStates[tempHash] = {
-            discovery_results: [],
+                playlist: {
-            discovery_progress: 0,
+                    name: data.name,
-            spotify_matches: 0,
+                    tracks: tracks,
-            spotify_total: tracks.length,
+                    track_count: tracks.length
-            status: 'parsed',
+                },
-            url: `mirrored://${data.source}/${data.source_playlist_id}`,
+                phase: statusData.phase || 'discovered',
-            sync_playlist_id: null,
+                discovery_results: statusData.results || [],
-            converted_spotify_playlist_id: null,
+                discoveryResults: statusData.results || [],
-            download_process_id: null,
+                discovery_progress: statusData.progress || 100,
-            created_at: Date.now() / 1000,
+                spotify_matches: statusData.spotify_matches || 0,
-            last_accessed: Date.now() / 1000,
+                spotifyMatches: statusData.spotify_matches || 0,
-            discovery_future: null,
+                spotify_total: tracks.length,
-            sync_progress: {},
+                status: statusData.status || 'complete',
-            is_mirrored_playlist: true,
+                url: `mirrored://${data.source}/${data.source_playlist_id}`,
-            mirrored_source: data.source
+                sync_playlist_id: null,
-        };
+                converted_spotify_playlist_id: null,
+                download_process_id: null,
+                created_at: Date.now() / 1000,
+                last_accessed: Date.now() / 1000,
+                discovery_future: null,
+                sync_progress: {},
+                is_mirrored_playlist: true,
+                mirrored_source: data.source
+            };
+
+            const cached = prepData.cached_matches || 0;
+            const total = prepData.total_tracks || tracks.length;
+            showToast(`Loaded ${cached}/${total} cached discovery results`, 'success');
+        } else {
+            // No cached data — fresh state
+            youtubePlaylistStates[tempHash] = {
+                playlist: {
+                    name: data.name,
+                    tracks: tracks,
+                    track_count: tracks.length
+                },
+                phase: 'fresh',
+                discovery_results: [],
+                discovery_progress: 0,
+                spotify_matches: 0,
+                spotify_total: tracks.length,
+                status: 'parsed',
+                url: `mirrored://${data.source}/${data.source_playlist_id}`,
+                sync_playlist_id: null,
+                converted_spotify_playlist_id: null,
+                download_process_id: null,
+                created_at: Date.now() / 1000,
+                last_accessed: Date.now() / 1000,
+                discovery_future: null,
+                sync_progress: {},
+                is_mirrored_playlist: true,
+                mirrored_source: data.source
+            };
+        }
 
         openYouTubeDiscoveryModal(tempHash);
     } catch (err) {
@@ -43327,6 +43374,41 @@ async function discoverMirroredPlaylist(playlistId) {
 // AUTOMATIONS — Visual Builder
 // ===============================
 
+async function retryFailedMirroredDiscovery(urlHash) {
+    // Extract playlist ID from url_hash (format: "mirrored_<id>")
+    const playlistId = urlHash.replace('mirrored_', '');
+    try {
+        const res = await fetch(`/api/mirrored-playlists/${playlistId}/retry-failed-discovery`, { method: 'POST' });
+        const data = await res.json();
+        if (data.error) {
+            showToast(`Error: ${data.error}`, 'error');
+            return;
+        }
+        if (data.retry_count === 0) {
+            showToast('All tracks already found!', 'success');
+            return;
+        }
+
+        // Update frontend state to discovering
+        const state = youtubePlaylistStates[urlHash];
+        if (state) {
+            state.phase = 'discovering';
+            state.status = 'discovering';
+            state.discovery_progress = 0;
+        }
+
+        // Update modal buttons to show discovering state
+        updateYouTubeModalButtons(urlHash, 'discovering');
+
+        // Start polling for progress
+        startYouTubeDiscoveryPolling(urlHash);
+
+        showToast(`Retrying ${data.retry_count} failed tracks...`, 'info');
+    } catch (err) {
+        showToast(`Error retrying discovery: ${err.message}`, 'error');
+    }
+}
+
 let _autoBlocks = null; // cached block definitions from /api/automations/blocks
 let _autoBuilder = { editId: null, when: null, do: null, then: [], isSystem: false };