danvics/soulsync
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 1

Playlist discovery pipeline with official metadata enforcement for automated sync

Browse source
This commit is contained in:
Broque Thomas 2026-03-05 18:32:18 -08:00
parent 4bd3e776bd
commit d57b48a62a
3 changed files with 441 additions and 44 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

44
database/music_database.py
View file

@ -6520,6 +6520,50 @@ class MusicDatabase:
logger.error(f"Error getting mirrored playlist tracks: {e}")
return []
def update_mirrored_track_extra_data(self, track_id: int, extra_data_dict: dict) -> bool:
"""Merge new data into a mirrored track's extra_data JSON field."""
try:
with self._get_connection() as conn:
cursor = conn.cursor()
cursor.execute(
"SELECT extra_data FROM mirrored_playlist_tracks WHERE id = ?",
(track_id,)
)
row = cursor.fetchone()
if not row:
return False
existing = {}
if row['extra_data']:
try:
existing = json.loads(row['extra_data'])
except (json.JSONDecodeError, TypeError):
pass
existing.update(extra_data_dict)
cursor.execute(
"UPDATE mirrored_playlist_tracks SET extra_data = ? WHERE id = ?",
(json.dumps(existing), track_id)
)
conn.commit()
return cursor.rowcount > 0
except Exception as e:
logger.error(f"Error updating mirrored track extra_data: {e}")
return False
def get_mirrored_tracks_extra_data_map(self, playlist_id: int) -> dict:
"""Return {source_track_id: extra_data_json_string} for a playlist.
Used to preserve discovery data across refreshes."""
try:
with self._get_connection() as conn:
cursor = conn.cursor()
cursor.execute("""
SELECT source_track_id, extra_data FROM mirrored_playlist_tracks
WHERE playlist_id = ? AND source_track_id IS NOT NULL AND extra_data IS NOT NULL
""", (playlist_id,))
return {row['source_track_id']: row['extra_data'] for row in cursor.fetchall()}
except Exception as e:
logger.error(f"Error getting extra_data map: {e}")
return {}
def delete_mirrored_playlist(self, playlist_id: int) -> bool:
"""Delete a mirrored playlist and its tracks (CASCADE)."""
try:

392
web_server.py
View file

@ -307,12 +307,42 @@ def _register_automation_handlers():
try:
source = pl.get('source', '')
source_id = pl.get('source_playlist_id', '')
tracks = None
if source == 'spotify' and spotify_client and spotify_client.is_spotify_authenticated():
playlist_obj = spotify_client.get_playlist_by_id(source_id)
if playlist_obj and playlist_obj.tracks:
tracks = []
for t in playlist_obj.tracks:
# Track.artists is List[str], not List[dict]
artist_name = t.artists[0] if t.artists else ''
track_dict = {
'track_name': t.name or '',
'artist_name': str(artist_name),
'album_name': t.album or '',
'duration_ms': t.duration_ms or 0,
'source_track_id': t.id or '',
}
# Spotify data IS official — auto-mark as discovered
if t.id:
track_dict['extra_data'] = json.dumps({
'discovered': True,
'provider': 'spotify',
'confidence': 1.0,
'matched_data': {
'id': t.id,
'name': t.name or '',
'artists': [{'name': str(a)} for a in (t.artists or [])],
'album': t.album or '',
'duration_ms': t.duration_ms or 0,
}
})
tracks.append(track_dict)
elif source == 'tidal' and tidal_client and tidal_client.is_authenticated():
full_playlist = tidal_client.get_playlist(source_id)
if full_playlist and full_playlist.tracks:
tracks = []
for t in full_playlist.tracks:
artist_name = t.artists[0] if t.artists else ''
tracks.append({
'track_name': t.name or '',
@ -321,42 +351,66 @@ def _register_automation_handlers():
'duration_ms': t.duration_ms or 0,
'source_track_id': t.id or '',
})
# Compare old vs new track IDs to detect changes
old_tracks = db.get_mirrored_playlist_tracks(pl['id']) if pl.get('id') else []
old_ids = {t.get('source_track_id') for t in old_tracks if t.get('source_track_id')}
new_ids = {t.get('source_track_id') for t in tracks if t.get('source_track_id')}
db.mirror_playlist(
source=source,
source_playlist_id=source_id,
name=pl['name'],
tracks=tracks,
profile_id=pl.get('profile_id', 1),
# Preserve existing image/owner — Playlist dataclass lacks image_url
owner=getattr(playlist_obj, 'owner', pl.get('owner')),
image_url=pl.get('image_url'),
)
refreshed += 1
elif source == 'youtube':
yt_url = f"https://www.youtube.com/playlist?list={source_id}"
playlist_data = parse_youtube_playlist(yt_url)
if playlist_data and playlist_data.get('tracks'):
tracks = []
for t in playlist_data['tracks']:
artist_name = t['artists'][0] if t.get('artists') else ''
tracks.append({
'track_name': t.get('name', ''),
'artist_name': str(artist_name),
'album_name': '',
'duration_ms': t.get('duration_ms', 0),
'source_track_id': t.get('id', ''),
})
# Emit playlist_changed if tracks actually changed
if old_ids != new_ids:
try:
if automation_engine:
automation_engine.emit('playlist_changed', {
'playlist_name': pl.get('name', ''),
'old_count': str(len(old_ids)),
'new_count': str(len(new_ids)),
'added': str(len(new_ids - old_ids)),
'removed': str(len(old_ids - new_ids)),
})
except Exception:
pass
if tracks is not None:
# Compare old vs new track IDs to detect changes
old_tracks = db.get_mirrored_playlist_tracks(pl['id']) if pl.get('id') else []
old_ids = {t.get('source_track_id') for t in old_tracks if t.get('source_track_id')}
new_ids = {t.get('source_track_id') for t in tracks if t.get('source_track_id')}
# Preserve existing discovery extra_data for tracks that still exist
old_extra_map = db.get_mirrored_tracks_extra_data_map(pl['id']) if pl.get('id') else {}
for t in tracks:
sid = t.get('source_track_id', '')
if sid and sid in old_extra_map and 'extra_data' not in t:
t['extra_data'] = old_extra_map[sid]
db.mirror_playlist(
source=source,
source_playlist_id=source_id,
name=pl['name'],
tracks=tracks,
profile_id=pl.get('profile_id', 1),
owner=pl.get('owner'),
image_url=pl.get('image_url'),
)
refreshed += 1
# Emit playlist_changed if tracks actually changed
if old_ids != new_ids:
try:
if automation_engine:
automation_engine.emit('playlist_changed', {
'playlist_name': pl.get('name', ''),
'old_count': str(len(old_ids)),
'new_count': str(len(new_ids)),
'added': str(len(new_ids - old_ids)),
'removed': str(len(old_ids - new_ids)),
})
except Exception:
pass
except Exception as e:
errors.append(f"{pl.get('name', '?')}: {str(e)}")
return {'status': 'completed', 'refreshed': str(refreshed), 'errors': str(len(errors))}
def _auto_sync_playlist(config):
"""Sync a mirrored playlist to media server."""
"""Sync a mirrored playlist to media server.
Uses discovered metadata when available, skips undiscovered tracks."""
playlist_id = config.get('playlist_id')
if not playlist_id:
return {'status': 'error', 'reason': 'No playlist specified'}
@ -371,15 +425,39 @@ def _register_automation_handlers():
return {'status': 'error', 'reason': 'No tracks in playlist'}
# Convert mirrored tracks to format expected by _run_sync_task
# Use discovered metadata when available, skip undiscovered tracks
tracks_json = []
skipped_count = 0
for t in tracks:
tracks_json.append({
'name': t.get('track_name', ''),
'artists': [{'name': t.get('artist_name', '')}],
'album': t.get('album_name', ''),
'duration_ms': t.get('duration_ms', 0),
'id': t.get('source_track_id', ''),
})
# Parse extra_data for discovery info
extra = {}
if t.get('extra_data'):
try:
extra = json.loads(t['extra_data']) if isinstance(t['extra_data'], str) else t['extra_data']
except (json.JSONDecodeError, TypeError):
pass
if extra.get('discovered') and extra.get('matched_data'):
# Use official discovered metadata
md = extra['matched_data']
tracks_json.append({
'name': md.get('name', ''),
'artists': md.get('artists', [{'name': t.get('artist_name', '')}]),
'album': md.get('album', ''),
'duration_ms': md.get('duration_ms', 0),
'id': md.get('id', ''),
})
else:
# NOT discovered — skip to prevent garbage in wishlist
skipped_count += 1
if not tracks_json:
return {
'status': 'skipped',
'reason': f'No discovered tracks to sync ({skipped_count} tracks need discovery first)',
'skipped_tracks': str(skipped_count),
}
sync_id = f"auto_mirror_{playlist_id}"
threading.Thread(
@ -388,10 +466,42 @@ def _register_automation_handlers():
daemon=True,
name=f'auto-sync-{playlist_id}'
).start()
return {'status': 'started', 'playlist_name': pl['name']}
return {
'status': 'started',
'playlist_name': pl['name'],
'discovered_tracks': str(len(tracks_json)),
'skipped_tracks': str(skipped_count),
}
def _auto_discover_playlist(config):
"""Discover official Spotify/iTunes metadata for mirrored playlist tracks."""
db = get_database()
playlist_id = config.get('playlist_id')
discover_all = config.get('all', False)
if discover_all:
playlists = db.get_mirrored_playlists()
elif playlist_id:
p = db.get_mirrored_playlist(int(playlist_id))
playlists = [p] if p else []
else:
return {'status': 'error', 'reason': 'No playlist specified'}
if not playlists:
return {'status': 'error', 'reason': 'No playlists found'}
threading.Thread(
target=_run_playlist_discovery_worker,
args=(playlists,),
daemon=True,
name='auto-discover-playlist'
).start()
names = ', '.join(p['name'] for p in playlists[:3])
return {'status': 'started', 'playlist_count': str(len(playlists)), 'playlists': names}
automation_engine.register_action_handler('refresh_mirrored', _auto_refresh_mirrored)
automation_engine.register_action_handler('sync_playlist', _auto_sync_playlist)
automation_engine.register_action_handler('discover_playlist', _auto_discover_playlist)
# --- Phase 3 action handlers ---
@ -3576,6 +3686,10 @@ def get_automation_blocks():
"has_conditions": True,
"condition_fields": ["playlist_name"],
"variables": ["playlist_name", "old_count", "new_count", "added", "removed"]},
{"type": "discovery_completed", "label": "Discovery Complete", "icon": "search", "description": "When playlist track discovery finishes", "available": True,
"has_conditions": True,
"condition_fields": ["playlist_name"],
"variables": ["playlist_name", "total_tracks", "discovered_count", "failed_count", "skipped_count"]},
# Phase 3 triggers
{"type": "wishlist_processing_completed", "label": "Wishlist Processed", "icon": "check-circle",
"description": "When auto-wishlist processing finishes", "available": True,
@ -3635,6 +3749,11 @@ def get_automation_blocks():
"config_fields": [
{"key": "playlist_id", "type": "mirrored_playlist_select", "label": "Playlist"}
]},
{"type": "discover_playlist", "label": "Discover Playlist", "icon": "search", "description": "Find official Spotify/iTunes metadata for mirrored playlist tracks", "available": True,
"config_fields": [
{"key": "playlist_id", "type": "mirrored_playlist_select", "label": "Playlist"},
{"key": "all", "type": "checkbox", "label": "Discover all mirrored playlists", "default": False}
]},
{"type": "notify_only", "label": "Notify Only", "icon": "bell", "description": "No action — just send notification", "available": True},
# Phase 3 actions
{"type": "start_database_update", "label": "Update Database", "icon": "database",
@ -19772,6 +19891,203 @@ def update_tidal_playlist_phase(playlist_id):
return jsonify({"error": str(e)}), 500
def _run_playlist_discovery_worker(playlists):
"""Background worker that discovers Spotify/iTunes metadata for undiscovered
mirrored playlist tracks. Stores results in extra_data for use by sync."""
try:
use_spotify = spotify_client and spotify_client.is_spotify_authenticated()
discovery_source = 'spotify' if use_spotify else 'itunes'
itunes_client_instance = None
if not use_spotify:
try:
from core.itunes_client import iTunesClient
itunes_client_instance = iTunesClient()
except Exception:
print("❌ Neither Spotify nor iTunes available for discovery")
return
total_discovered = 0
total_failed = 0
total_skipped = 0
total_tracks = 0
last_playlist_name = ''
for pl in playlists:
pl_id = pl['id']
pl_name = pl.get('name', '')
last_playlist_name = pl_name
source = pl.get('source', '')
# Spotify playlists are auto-discovered during refresh
if source == 'spotify':
continue
db = get_database()
tracks = db.get_mirrored_playlist_tracks(pl_id)
if not tracks:
continue
print(f"🔍 Starting discovery for playlist '{pl_name}' ({len(tracks)} tracks, using {discovery_source.upper()})")
for i, track in enumerate(tracks):
total_tracks += 1
track_id = track['id']
track_name = track.get('track_name', '')
artist_name = track.get('artist_name', '')
duration_ms = track.get('duration_ms', 0)
# Check if already discovered
existing_extra = {}
if track.get('extra_data'):
try:
existing_extra = json.loads(track['extra_data']) if isinstance(track['extra_data'], str) else track['extra_data']
except (json.JSONDecodeError, TypeError):
pass
if existing_extra.get('discovered'):
total_skipped += 1
continue
# Step 1: Check discovery cache
cache_key = _get_discovery_cache_key(track_name, artist_name)
try:
cached_match = db.get_discovery_cache_match(cache_key[0], cache_key[1], discovery_source)
if cached_match and _validate_discovery_cache_artist(artist_name, cached_match):
extra_data = {
'discovered': True,
'provider': discovery_source,
'confidence': cached_match.get('confidence', 0.85),
'matched_data': cached_match,
}
db.update_mirrored_track_extra_data(track_id, extra_data)
total_discovered += 1
print(f"⚡ CACHE [{i+1}/{len(tracks)}]: {track_name} → {cached_match.get('name', '?')}")
continue
except Exception:
pass
# Step 2: Generate search queries
try:
temp_track = type('TempTrack', (), {
'name': track_name,
'artists': [artist_name],
'album': None
})()
search_queries = matching_engine.generate_download_queries(temp_track)
except Exception:
search_queries = [f"{artist_name} {track_name}", track_name]
# Step 3: Search and score
best_match = None
best_confidence = 0.0
min_confidence = 0.7
for search_query in search_queries:
try:
if use_spotify:
results = spotify_client.search_tracks(search_query, limit=10)
else:
results = itunes_client_instance.search_tracks(search_query, limit=10)
if not results:
continue
match, confidence, _ = _discovery_score_candidates(
track_name, artist_name, duration_ms, results
)
if match and confidence > best_confidence:
best_confidence = confidence
best_match = match
if best_confidence >= 0.9:
break
except Exception:
continue
# Extended search fallback
if not best_match or best_confidence < min_confidence:
try:
query = f"{artist_name} {track_name}"
if use_spotify:
extended = spotify_client.search_tracks(query, limit=50)
else:
extended = itunes_client_instance.search_tracks(query, limit=50)
if extended:
match, confidence, _ = _discovery_score_candidates(
track_name, artist_name, duration_ms, extended
)
if match and confidence > best_confidence:
best_confidence = confidence
best_match = match
except Exception:
pass
# Step 4: Store results
if best_match and best_confidence >= min_confidence:
match_artists = best_match.artists if hasattr(best_match, 'artists') else []
matched_data = {
'id': best_match.id if hasattr(best_match, 'id') else '',
'name': best_match.name if hasattr(best_match, 'name') else '',
'artists': [{'name': a} if isinstance(a, str) else a for a in match_artists],
'album': best_match.album if hasattr(best_match, 'album') else '',
'duration_ms': best_match.duration_ms if hasattr(best_match, 'duration_ms') else 0,
'source': discovery_source,
}
extra_data = {
'discovered': True,
'provider': discovery_source,
'confidence': best_confidence,
'matched_data': matched_data,
}
db.update_mirrored_track_extra_data(track_id, extra_data)
total_discovered += 1
# Save to discovery cache
try:
db.save_discovery_cache_match(
cache_key[0], cache_key[1], discovery_source,
best_confidence, matched_data,
track_name, artist_name
)
except Exception:
pass
print(f"✅ [{i+1}/{len(tracks)}] {track_name} → {matched_data['name']} ({best_confidence:.2f})")
else:
extra_data = {
'discovered': False,
'discovery_attempted': True,
'provider': discovery_source,
}
db.update_mirrored_track_extra_data(track_id, extra_data)
total_failed += 1
print(f"❌ [{i+1}/{len(tracks)}] No match: {track_name} by {artist_name}")
time.sleep(0.15)
# Emit completion event
try:
if automation_engine:
automation_engine.emit('discovery_completed', {
'playlist_name': last_playlist_name if len(playlists) == 1 else f'{len(playlists)} playlists',
'total_tracks': str(total_tracks),
'discovered_count': str(total_discovered),
'failed_count': str(total_failed),
'skipped_count': str(total_skipped),
})
except Exception:
pass
print(f"✅ Playlist discovery complete: {total_discovered} discovered, {total_failed} failed, {total_skipped} skipped")
except Exception as e:
print(f"❌ Error in playlist discovery worker: {e}")
import traceback
traceback.print_exc()
def _get_discovery_cache_key(title, artist):
"""Normalize title/artist for discovery cache lookup using matching_engine."""
norm_title = matching_engine.clean_title(title)

49
webui/static/script.js
View file

@ -42137,6 +42137,7 @@ const _autoIcons = {
playlist_changed: '\u270F\uFE0F',
process_wishlist: '\uD83D\uDCCB', scan_watchlist: '\uD83D\uDC41\uFE0F',
scan_library: '\uD83D\uDD04', refresh_mirrored: '\uD83D\uDCC2', sync_playlist: '\uD83D\uDD01',
discover_playlist: '\uD83D\uDD0D', discovery_completed: '\uD83D\uDD0D',
notify_only: '\uD83D\uDD14', discord_webhook: '\uD83D\uDCAC', pushbullet: '\uD83D\uDD14', telegram: '\u2709\uFE0F',
// Phase 3
wishlist_processing_completed: '\u2705', watchlist_scan_completed: '\u2705',
@ -42246,7 +42247,7 @@ function _autoFormatTrigger(type, config) {
}
const labels = { app_started: 'App Started', track_downloaded: 'Track Downloaded', batch_complete: 'Batch Complete',
watchlist_new_release: 'New Release Found', playlist_synced: 'Playlist Synced',
playlist_changed: 'Playlist Changed',
playlist_changed: 'Playlist Changed', discovery_completed: 'Discovery Complete',
wishlist_processing_completed: 'Wishlist Processed', watchlist_scan_completed: 'Watchlist Scan Done',
database_update_completed: 'Database Updated', download_failed: 'Download Failed',
download_quarantined: 'File Quarantined', wishlist_item_added: 'Wishlist Item Added',
@ -42264,7 +42265,8 @@ function _autoFormatTrigger(type, config) {
function _autoFormatAction(type) {
const labels = { process_wishlist: 'Process Wishlist', scan_watchlist: 'Scan Watchlist',
scan_library: 'Scan Library', refresh_mirrored: 'Refresh Mirrored',
sync_playlist: 'Sync Playlist', notify_only: 'Notify Only',
sync_playlist: 'Sync Playlist', discover_playlist: 'Discover Playlist',
notify_only: 'Notify Only',
start_database_update: 'Update Database', run_duplicate_cleaner: 'Run Duplicate Cleaner',
clear_quarantine: 'Clear Quarantine', cleanup_wishlist: 'Clean Up Wishlist',
update_discovery_pool: 'Update Discovery', start_quality_scan: 'Run Quality Scan',
@ -42587,6 +42589,18 @@ function _renderBlockConfigFields(slotKey, blockType, config) {
</select>
</div>`;
}
if (blockType === 'discover_playlist') {
const allChecked = config.all ? ' checked' : '';
return `<div class="config-row">
<label>Playlist</label>
<select id="cfg-${slotKey}-playlist_id" class="mirrored-playlist-select" data-value="${_escAttr(config.playlist_id || '')}">
<option value="">Loading...</option>
</select>
</div>
<div class="config-row">
<label><input type="checkbox" id="cfg-${slotKey}-all"${allChecked} onchange="_autoTogglePlaylistSelect('${slotKey}')"> Discover all mirrored playlists</label>
</div>`;
}
// Shared variable tags builder for notification types
function _notifyVarHtml(slotKey) {
let allVars = ['time', 'name', 'run_count', 'status'];
@ -42683,21 +42697,28 @@ function _renderConditionBuilder(slotKey, blockDef, config) {
function _renderConditionRow(slotKey, index, fields, cond) {
const field = cond ? cond.field : (fields[0] || '');
const operator = cond ? cond.operator : 'contains';
const operator = cond ? cond.operator : 'equals';
const value = cond ? _escAttr(cond.value) : '';
let fieldOpts = '';
fields.forEach(f => { fieldOpts += `<option value="${f}"${f===field?' selected':''}>${f}</option>`; });
// For playlist-related triggers, use a mirrored playlist dropdown instead of free text
const triggerType = _autoBuilder.when ? _autoBuilder.when.type : '';
const usePlaylistSelect = ((triggerType === 'playlist_changed' || triggerType === 'discovery_completed') && field === 'playlist_name');
const valueHtml = usePlaylistSelect
? `<select class="cond-value mirrored-playlist-name-select" data-slot="${slotKey}" data-idx="${index}" data-value="${value}"></select>`
: `<input type="text" class="cond-value" data-slot="${slotKey}" data-idx="${index}" value="${value}" placeholder="value...">`;
return `<div class="condition-row" data-index="${index}">
<select class="cond-field" data-slot="${slotKey}" data-idx="${index}">${fieldOpts}</select>
<select class="cond-operator" data-slot="${slotKey}" data-idx="${index}">
<option value="contains"${operator==='contains'?' selected':''}>contains</option>
<option value="equals"${operator==='equals'?' selected':''}>equals</option>
<option value="contains"${operator==='contains'?' selected':''}>contains</option>
<option value="starts_with"${operator==='starts_with'?' selected':''}>starts with</option>
<option value="not_contains"${operator==='not_contains'?' selected':''}>not contains</option>
</select>
<input type="text" class="cond-value" data-slot="${slotKey}" data-idx="${index}" value="${value}" placeholder="value...">
${valueHtml}
<button class="remove-condition-btn" onclick="_autoRemoveCondition('${slotKey}',${index})">\u2715</button>
</div>`;
}
@ -42757,7 +42778,8 @@ function _autoTogglePlaylistSelect(slotKey) {
async function _autoLoadMirroredSelects() {
const selects = document.querySelectorAll('.mirrored-playlist-select');
if (!selects.length) return;
const nameSelects = document.querySelectorAll('.mirrored-playlist-name-select');
if (!selects.length && !nameSelects.length) return;
if (!_autoMirroredPlaylists) {
try {
@ -42773,6 +42795,14 @@ async function _autoLoadMirroredSelects() {
sel.innerHTML += `<option value="${p.id}"${String(p.id) === savedValue ? ' selected' : ''}>${_esc(p.name)}</option>`;
});
});
nameSelects.forEach(sel => {
const savedValue = sel.dataset.value || '';
sel.innerHTML = '<option value="">-- Select playlist --</option>';
_autoMirroredPlaylists.forEach(p => {
sel.innerHTML += `<option value="${_escAttr(p.name)}"${p.name === savedValue ? ' selected' : ''}>${_esc(p.name)}</option>`;
});
});
}
function _readPlacedConfig(slotKey) {
@ -42818,6 +42848,13 @@ function _readPlacedConfig(slotKey) {
if (type === 'sync_playlist') {
return { playlist_id: document.getElementById('cfg-' + slotKey + '-playlist_id')?.value || '' };
}
if (type === 'discover_playlist') {
const allCb = document.getElementById('cfg-' + slotKey + '-all');
return {
playlist_id: document.getElementById('cfg-' + slotKey + '-playlist_id')?.value || '',
all: allCb ? allCb.checked : false,
};
}
if (type === 'discord_webhook') {
return {
webhook_url: document.getElementById('cfg-' + slotKey + '-webhook_url')?.value?.trim() || '',