Add SoulID worker with API-based debut year disambiguation
SoulID worker generates deterministic soul IDs for all library entities:
- Artists: hash(name + debut_year) — searches iTunes + Deezer APIs,
verifies correct artist by matching discography against local DB
albums via MusicMatchingEngine, pools years from both sources and
picks the earliest. Falls back to hash(name) if no match found.
- Albums: hash(artist + album)
- Tracks: song ID hash(artist + track) + album ID hash(artist + album + track)
Dashboard button with trans2.png logo, rainbow spinner, hover tooltip.
Worker orb with rainbow effect. SoulSync badge on library artist cards.
DB migration adds soul_id columns with indexes to artists/albums/tracks.
Migration version flag auto-resets artist soul IDs when algorithm changes.
This commit is contained in:
Broque Thomas
parent
cc96af2cb1
commit
2d511d0a16
8 changed files with 921 additions and 1 deletions
579
core/soulid_worker.py
Normal file
579
core/soulid_worker.py
Normal file
|
|
@ -0,0 +1,579 @@
|
|||
"""
|
||||
SoulID Worker — generates deterministic soul IDs for artists, albums, and tracks.
|
||||
|
||||
Runs as a background worker that processes library entries without soul IDs,
|
||||
computes a deterministic hash from normalized metadata, and stores the result.
|
||||
|
||||
Hash inputs (all lowercased, stripped of accents/punctuation, collapsed):
|
||||
- Artist: normalize(artist_name) + normalize(debut_year) if known
|
||||
Debut year is sourced from iTunes + Deezer APIs (not local DB)
|
||||
to ensure deterministic results across all SoulSync nodes.
|
||||
- Album: normalize(artist_name) + normalize(album_name)
|
||||
- Track (song): normalize(artist_name) + normalize(track_name)
|
||||
- Track (album): normalize(artist_name) + normalize(album_name) + normalize(track_name)
|
||||
|
||||
The "song" soul ID links different versions of the same song (single vs album).
|
||||
The "album track" soul ID is specific to a track on a particular release.
|
||||
"""
|
||||
|
||||
import hashlib
|
||||
import re
|
||||
import threading
|
||||
import time
|
||||
import unicodedata
|
||||
from typing import Dict, Any, List, Optional
|
||||
|
||||
from utils.logging_config import get_logger
|
||||
|
||||
logger = get_logger("soulid_worker")
|
||||
|
||||
|
||||
def normalize_for_soul_id(text: str) -> str:
|
||||
"""Aggressively normalize a string for deterministic hashing.
|
||||
|
||||
- Lowercase
|
||||
- Strip accents/diacritics (Beyoncé → beyonce)
|
||||
- Remove parentheticals: (feat. X), (Deluxe), (Remastered), [Live], etc.
|
||||
- Remove all non-alphanumeric characters
|
||||
- Collapse whitespace
|
||||
"""
|
||||
if not text:
|
||||
return ''
|
||||
s = text.lower()
|
||||
# Decompose unicode and strip combining marks (accents)
|
||||
s = unicodedata.normalize('NFD', s)
|
||||
s = ''.join(c for c in s if unicodedata.category(c) != 'Mn')
|
||||
# Remove parenthetical/bracket suffixes
|
||||
s = re.sub(r'\s*[\(\[][^)\]]*[\)\]]', '', s)
|
||||
# Remove all non-alphanumeric
|
||||
s = re.sub(r'[^a-z0-9]', '', s)
|
||||
return s
|
||||
|
||||
|
||||
def generate_soul_id(*parts: str) -> str:
|
||||
"""Generate a soul ID from normalized parts.
|
||||
|
||||
Returns a 'soul_' prefixed hex string (first 16 chars of SHA-256).
|
||||
"""
|
||||
combined = ''.join(normalize_for_soul_id(p) for p in parts if p)
|
||||
if not combined:
|
||||
return ''
|
||||
digest = hashlib.sha256(combined.encode('utf-8')).hexdigest()[:16]
|
||||
return f'soul_{digest}'
|
||||
|
||||
|
||||
class SoulIDWorker:
|
||||
"""Background worker that generates soul IDs for all library entities.
|
||||
|
||||
Artists are processed one at a time (API calls to iTunes/Deezer for
|
||||
deterministic debut year). Albums and tracks are processed in batches
|
||||
(local DB only, no API calls).
|
||||
"""
|
||||
|
||||
def __init__(self, database):
|
||||
self.db = database
|
||||
|
||||
# Worker state
|
||||
self.running = False
|
||||
self.paused = False
|
||||
self.should_stop = False
|
||||
self.thread = None
|
||||
self.current_item = None
|
||||
|
||||
# API clients (lazy-initialized)
|
||||
self._itunes_client = None
|
||||
self._deezer_client = None
|
||||
self._matching_engine = None
|
||||
|
||||
# Statistics
|
||||
self.stats = {
|
||||
'artists_processed': 0,
|
||||
'albums_processed': 0,
|
||||
'tracks_processed': 0,
|
||||
'errors': 0,
|
||||
'pending': 0,
|
||||
}
|
||||
|
||||
# Processing config
|
||||
self.batch_size = 100 # For albums/tracks (no API calls)
|
||||
self.artist_sleep = 3.0 # Between artist API lookups (rate limit courtesy)
|
||||
self.inter_batch_sleep = 0.5 # Between album/track batches
|
||||
self.idle_sleep = 30
|
||||
self.album_match_threshold = 0.80
|
||||
|
||||
logger.info("SoulID worker initialized")
|
||||
|
||||
def _get_itunes_client(self):
|
||||
if self._itunes_client is None:
|
||||
try:
|
||||
from core.itunes_client import iTunesClient
|
||||
self._itunes_client = iTunesClient()
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to init iTunes client: {e}")
|
||||
return self._itunes_client
|
||||
|
||||
def _get_deezer_client(self):
|
||||
if self._deezer_client is None:
|
||||
try:
|
||||
from core.deezer_client import DeezerClient
|
||||
self._deezer_client = DeezerClient()
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to init Deezer client: {e}")
|
||||
return self._deezer_client
|
||||
|
||||
def _get_matching_engine(self):
|
||||
if self._matching_engine is None:
|
||||
try:
|
||||
from core.matching_engine import MusicMatchingEngine
|
||||
self._matching_engine = MusicMatchingEngine()
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to init matching engine: {e}")
|
||||
return self._matching_engine
|
||||
|
||||
def start(self):
|
||||
if self.running:
|
||||
return
|
||||
self.running = True
|
||||
self.should_stop = False
|
||||
self.thread = threading.Thread(target=self._run, daemon=True)
|
||||
self.thread.start()
|
||||
logger.info("SoulID worker started")
|
||||
|
||||
def stop(self):
|
||||
if not self.running:
|
||||
return
|
||||
self.should_stop = True
|
||||
self.running = False
|
||||
if self.thread:
|
||||
self.thread.join(timeout=5)
|
||||
logger.info("SoulID worker stopped")
|
||||
|
||||
def pause(self):
|
||||
self.paused = True
|
||||
|
||||
def resume(self):
|
||||
self.paused = False
|
||||
|
||||
def get_stats(self) -> Dict[str, Any]:
|
||||
self.stats['pending'] = self._count_pending()
|
||||
is_running = self.running and self.thread is not None and self.thread.is_alive()
|
||||
is_idle = is_running and not self.paused and self.stats['pending'] == 0
|
||||
return {
|
||||
'enabled': True,
|
||||
'running': is_running and not self.paused,
|
||||
'paused': self.paused,
|
||||
'idle': is_idle,
|
||||
'current_item': self.current_item,
|
||||
'stats': self.stats.copy(),
|
||||
}
|
||||
|
||||
# ── Main loop ──
|
||||
|
||||
def _run(self):
|
||||
logger.info("SoulID worker thread started")
|
||||
|
||||
# One-time migration: reset artist soul IDs when algorithm changes
|
||||
self._migrate_artist_soul_ids()
|
||||
|
||||
while not self.should_stop:
|
||||
try:
|
||||
if self.paused:
|
||||
time.sleep(1)
|
||||
continue
|
||||
|
||||
processed = 0
|
||||
processed += self._process_next_artist()
|
||||
processed += self._process_albums()
|
||||
processed += self._process_tracks()
|
||||
|
||||
if processed == 0:
|
||||
self.current_item = None
|
||||
time.sleep(self.idle_sleep)
|
||||
else:
|
||||
# Albums/tracks get inter_batch_sleep, artists get their
|
||||
# own sleep inside _process_next_artist
|
||||
time.sleep(self.inter_batch_sleep)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error in SoulID worker loop: {e}", exc_info=True)
|
||||
self.stats['errors'] += 1
|
||||
time.sleep(5)
|
||||
|
||||
self.current_item = None
|
||||
logger.info("SoulID worker thread finished")
|
||||
|
||||
# ── Artist processing (one at a time, API-based) ──
|
||||
|
||||
def _process_next_artist(self) -> int:
|
||||
"""Process a single artist — queries iTunes + Deezer for debut year."""
|
||||
conn = None
|
||||
try:
|
||||
conn = self.db._get_connection()
|
||||
cursor = conn.cursor()
|
||||
cursor.execute("""
|
||||
SELECT id, name FROM artists
|
||||
WHERE (soul_id IS NULL OR soul_id = '')
|
||||
AND name IS NOT NULL AND name != ''
|
||||
AND id IS NOT NULL
|
||||
LIMIT 1
|
||||
""")
|
||||
row = cursor.fetchone()
|
||||
if not row:
|
||||
return 0
|
||||
|
||||
artist_id, name = row
|
||||
self.current_item = f"Artist: {name}"
|
||||
|
||||
# Get this artist's album names from our DB for cross-referencing
|
||||
cursor.execute("""
|
||||
SELECT title FROM albums
|
||||
WHERE artist_id = ? AND title IS NOT NULL AND title != ''
|
||||
""", (artist_id,))
|
||||
db_album_names = [r[0] for r in cursor.fetchall()]
|
||||
|
||||
# Look up debut year from iTunes + Deezer APIs
|
||||
debut_year = self._lookup_debut_year(name, db_album_names)
|
||||
|
||||
if debut_year:
|
||||
soul_id = generate_soul_id(name, debut_year)
|
||||
self.current_item = f"Artist: {name} ({debut_year})"
|
||||
else:
|
||||
soul_id = generate_soul_id(name)
|
||||
|
||||
if not soul_id:
|
||||
soul_id = f'soul_unnamed_{artist_id}'
|
||||
|
||||
cursor.execute(
|
||||
"UPDATE artists SET soul_id = ? WHERE id = ? AND (soul_id IS NULL OR soul_id = '')",
|
||||
(soul_id, artist_id)
|
||||
)
|
||||
conn.commit()
|
||||
self.stats['artists_processed'] += 1
|
||||
logger.info(f"Generated soul ID for artist: {name}" + (f" (debut {debut_year})" if debut_year else ""))
|
||||
|
||||
# Rate limit courtesy for API calls
|
||||
time.sleep(self.artist_sleep)
|
||||
return 1
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error processing artist: {e}")
|
||||
self.stats['errors'] += 1
|
||||
if conn:
|
||||
try:
|
||||
conn.rollback()
|
||||
except Exception:
|
||||
pass
|
||||
return 0
|
||||
finally:
|
||||
if conn:
|
||||
conn.close()
|
||||
|
||||
def _lookup_debut_year(self, artist_name: str, db_album_names: List[str]) -> Optional[str]:
|
||||
"""Look up an artist's debut year from iTunes and Deezer.
|
||||
|
||||
Searches both sources for the artist, verifies the match by comparing
|
||||
their discography against our DB albums, then pools all album years
|
||||
from both matched sources and returns the earliest.
|
||||
|
||||
Args:
|
||||
artist_name: Artist name to search for
|
||||
db_album_names: Album names from our DB for this artist
|
||||
|
||||
Returns:
|
||||
Earliest release year as string (e.g. '2011'), or None
|
||||
"""
|
||||
if not db_album_names:
|
||||
# No albums to cross-reference — can't verify which artist is correct
|
||||
return None
|
||||
|
||||
matching = self._get_matching_engine()
|
||||
if not matching:
|
||||
return None
|
||||
|
||||
# Search both sources
|
||||
itunes = self._get_itunes_client()
|
||||
deezer = self._get_deezer_client()
|
||||
|
||||
itunes_results = []
|
||||
deezer_results = []
|
||||
|
||||
try:
|
||||
if itunes:
|
||||
itunes_results = itunes.search_artists(artist_name, limit=5) or []
|
||||
except Exception as e:
|
||||
logger.debug(f"iTunes artist search failed for '{artist_name}': {e}")
|
||||
|
||||
try:
|
||||
if deezer:
|
||||
deezer_results = deezer.search_artists(artist_name, limit=5) or []
|
||||
except Exception as e:
|
||||
logger.debug(f"Deezer artist search failed for '{artist_name}': {e}")
|
||||
|
||||
# Each source independently steps through its results to find a match
|
||||
itunes_discog = self._find_matching_discography(itunes, itunes_results, db_album_names, matching, 'iTunes')
|
||||
deezer_discog = self._find_matching_discography(deezer, deezer_results, db_album_names, matching, 'Deezer')
|
||||
|
||||
# Pool all albums from both matched sources
|
||||
all_years = []
|
||||
for discog in (itunes_discog, deezer_discog):
|
||||
if discog:
|
||||
for album in discog:
|
||||
year = self._extract_year(album)
|
||||
if year:
|
||||
all_years.append(year)
|
||||
|
||||
if all_years:
|
||||
return min(all_years)
|
||||
|
||||
return None
|
||||
|
||||
def _find_matching_discography(self, client, artist_results, db_album_names: List[str],
|
||||
matching, source_name: str) -> Optional[list]:
|
||||
"""Step through artist search results, return the discography of the first
|
||||
one whose albums overlap with our DB albums.
|
||||
|
||||
Args:
|
||||
client: iTunes or Deezer client
|
||||
artist_results: List of Artist dataclass objects from search
|
||||
db_album_names: Our DB album names for comparison
|
||||
matching: MatchingEngine instance
|
||||
source_name: 'iTunes' or 'Deezer' for logging
|
||||
|
||||
Returns:
|
||||
List of Album objects from the matched artist's discography, or None
|
||||
"""
|
||||
if not client or not artist_results:
|
||||
return None
|
||||
|
||||
for artist in artist_results:
|
||||
try:
|
||||
discog = client.get_artist_albums(artist.id, album_type='album,single', limit=50)
|
||||
if not discog:
|
||||
continue
|
||||
|
||||
# Check if any discography album matches any DB album
|
||||
for api_album in discog:
|
||||
api_name = api_album.name if hasattr(api_album, 'name') else str(api_album)
|
||||
for db_name in db_album_names:
|
||||
score = matching.similarity_score(
|
||||
matching.normalize_string(api_name),
|
||||
matching.normalize_string(db_name)
|
||||
)
|
||||
if score >= self.album_match_threshold:
|
||||
logger.debug(f" {source_name}: matched '{artist.name}' via album '{api_name}' ↔ '{db_name}' (score={score:.2f})")
|
||||
return discog
|
||||
|
||||
except Exception as e:
|
||||
logger.debug(f" {source_name}: discography fetch failed for '{artist.name}': {e}")
|
||||
continue
|
||||
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def _extract_year(album) -> Optional[str]:
|
||||
"""Extract a 4-digit year from an Album object's release_date."""
|
||||
release_date = getattr(album, 'release_date', '') or ''
|
||||
release_date = str(release_date)
|
||||
if len(release_date) >= 4:
|
||||
year = release_date[:4]
|
||||
if year.isdigit() and int(year) > 1900:
|
||||
return year
|
||||
return None
|
||||
|
||||
# ── Album processing (batch, local DB only) ──
|
||||
|
||||
def _process_albums(self) -> int:
|
||||
conn = None
|
||||
try:
|
||||
conn = self.db._get_connection()
|
||||
cursor = conn.cursor()
|
||||
cursor.execute("""
|
||||
SELECT al.id, al.title, ar.name as artist_name
|
||||
FROM albums al
|
||||
JOIN artists ar ON ar.id = al.artist_id
|
||||
WHERE (al.soul_id IS NULL OR al.soul_id = '')
|
||||
AND al.title IS NOT NULL AND al.title != ''
|
||||
AND ar.name IS NOT NULL AND ar.name != ''
|
||||
AND al.id IS NOT NULL
|
||||
LIMIT ?
|
||||
""", (self.batch_size,))
|
||||
rows = cursor.fetchall()
|
||||
if not rows:
|
||||
return 0
|
||||
|
||||
count = 0
|
||||
for album_id, title, artist_name in rows:
|
||||
if self.should_stop:
|
||||
break
|
||||
soul_id = generate_soul_id(artist_name, title)
|
||||
if not soul_id:
|
||||
soul_id = f'soul_unnamed_{album_id}'
|
||||
cursor.execute(
|
||||
"UPDATE albums SET soul_id = ? WHERE id = ? AND (soul_id IS NULL OR soul_id = '')",
|
||||
(soul_id, album_id)
|
||||
)
|
||||
count += 1
|
||||
self.current_item = f"Album: {artist_name} - {title}"
|
||||
|
||||
if count > 0:
|
||||
conn.commit()
|
||||
self.stats['albums_processed'] += count
|
||||
logger.info(f"Generated soul IDs for {count} albums")
|
||||
return count
|
||||
except Exception as e:
|
||||
logger.error(f"Error processing albums: {e}")
|
||||
self.stats['errors'] += 1
|
||||
if conn:
|
||||
try:
|
||||
conn.rollback()
|
||||
except Exception:
|
||||
pass
|
||||
return 0
|
||||
finally:
|
||||
if conn:
|
||||
conn.close()
|
||||
|
||||
# ── Track processing (batch, local DB only) ──
|
||||
|
||||
def _process_tracks(self) -> int:
|
||||
conn = None
|
||||
try:
|
||||
conn = self.db._get_connection()
|
||||
cursor = conn.cursor()
|
||||
cursor.execute("""
|
||||
SELECT t.id, t.title, ar.name as artist_name, al.title as album_title
|
||||
FROM tracks t
|
||||
JOIN artists ar ON ar.id = t.artist_id
|
||||
LEFT JOIN albums al ON al.id = t.album_id
|
||||
WHERE (t.soul_id IS NULL OR t.soul_id = '')
|
||||
AND t.title IS NOT NULL AND t.title != ''
|
||||
AND ar.name IS NOT NULL AND ar.name != ''
|
||||
AND t.id IS NOT NULL
|
||||
LIMIT ?
|
||||
""", (self.batch_size,))
|
||||
rows = cursor.fetchall()
|
||||
if not rows:
|
||||
return 0
|
||||
|
||||
count = 0
|
||||
for track_id, title, artist_name, album_title in rows:
|
||||
if self.should_stop:
|
||||
break
|
||||
|
||||
# Song soul ID: artist + track (links singles to album versions)
|
||||
song_soul_id = generate_soul_id(artist_name, title)
|
||||
|
||||
# Album track soul ID: artist + album + track (specific to this release)
|
||||
album_soul_id = ''
|
||||
if album_title:
|
||||
album_soul_id = generate_soul_id(artist_name, album_title, title)
|
||||
|
||||
if not song_soul_id:
|
||||
song_soul_id = f'soul_unnamed_{track_id}'
|
||||
cursor.execute(
|
||||
"UPDATE tracks SET soul_id = ?, album_soul_id = ? WHERE id = ? AND (soul_id IS NULL OR soul_id = '')",
|
||||
(song_soul_id, album_soul_id or None, track_id)
|
||||
)
|
||||
count += 1
|
||||
self.current_item = f"Track: {artist_name} - {title}"
|
||||
|
||||
if count > 0:
|
||||
conn.commit()
|
||||
self.stats['tracks_processed'] += count
|
||||
logger.info(f"Generated soul IDs for {count} tracks")
|
||||
return count
|
||||
except Exception as e:
|
||||
logger.error(f"Error processing tracks: {e}")
|
||||
self.stats['errors'] += 1
|
||||
if conn:
|
||||
try:
|
||||
conn.rollback()
|
||||
except Exception:
|
||||
pass
|
||||
return 0
|
||||
finally:
|
||||
if conn:
|
||||
conn.close()
|
||||
|
||||
# ── Migrations ──
|
||||
|
||||
def _migrate_artist_soul_ids(self):
|
||||
"""One-time reset: clear all artist soul IDs when algorithm changes.
|
||||
Uses a versioned metadata flag to run only once per algorithm version."""
|
||||
conn = None
|
||||
try:
|
||||
conn = self.db._get_connection()
|
||||
cursor = conn.cursor()
|
||||
|
||||
cursor.execute("SELECT value FROM metadata WHERE key = 'soulid_artist_version' LIMIT 1")
|
||||
row = cursor.fetchone()
|
||||
if row and row[0] == 'debut_year_api_v2':
|
||||
return # Already on latest version
|
||||
|
||||
# Reset all artist soul IDs for regeneration
|
||||
cursor.execute("UPDATE artists SET soul_id = NULL WHERE soul_id IS NOT NULL")
|
||||
reset_count = cursor.rowcount
|
||||
|
||||
# Mark current version
|
||||
cursor.execute("INSERT OR REPLACE INTO metadata (key, value) VALUES ('soulid_artist_version', 'debut_year_api_v2')")
|
||||
# Clean up old flags
|
||||
cursor.execute("DELETE FROM metadata WHERE key IN ('soulid_artist_v2', 'soulid_artist_v3')")
|
||||
conn.commit()
|
||||
|
||||
if reset_count > 0:
|
||||
logger.info(f"SoulID migration: reset {reset_count} artist soul IDs for API-based debut year regeneration")
|
||||
except Exception as e:
|
||||
logger.error(f"SoulID migration failed: {e}")
|
||||
if conn:
|
||||
try:
|
||||
conn.rollback()
|
||||
except Exception:
|
||||
pass
|
||||
finally:
|
||||
if conn:
|
||||
conn.close()
|
||||
|
||||
# ── Helpers ──
|
||||
|
||||
def _count_pending(self) -> int:
|
||||
conn = None
|
||||
try:
|
||||
conn = self.db._get_connection()
|
||||
cursor = conn.cursor()
|
||||
total = 0
|
||||
|
||||
# Must match the WHERE clauses in _process_* methods exactly
|
||||
cursor.execute("""
|
||||
SELECT COUNT(*) FROM artists
|
||||
WHERE (soul_id IS NULL OR soul_id = '')
|
||||
AND name IS NOT NULL AND name != ''
|
||||
AND id IS NOT NULL
|
||||
""")
|
||||
total += (cursor.fetchone() or [0])[0]
|
||||
|
||||
cursor.execute("""
|
||||
SELECT COUNT(*) FROM albums al
|
||||
JOIN artists ar ON ar.id = al.artist_id
|
||||
WHERE (al.soul_id IS NULL OR al.soul_id = '')
|
||||
AND al.title IS NOT NULL AND al.title != ''
|
||||
AND ar.name IS NOT NULL AND ar.name != ''
|
||||
AND al.id IS NOT NULL
|
||||
""")
|
||||
total += (cursor.fetchone() or [0])[0]
|
||||
|
||||
cursor.execute("""
|
||||
SELECT COUNT(*) FROM tracks t
|
||||
JOIN artists ar ON ar.id = t.artist_id
|
||||
WHERE (t.soul_id IS NULL OR t.soul_id = '')
|
||||
AND t.title IS NOT NULL AND t.title != ''
|
||||
AND ar.name IS NOT NULL AND ar.name != ''
|
||||
AND t.id IS NOT NULL
|
||||
""")
|
||||
total += (cursor.fetchone() or [0])[0]
|
||||
|
||||
return total
|
||||
except Exception:
|
||||
return 0
|
||||
finally:
|
||||
if conn:
|
||||
conn.close()
|
||||
|
|
@ -356,6 +356,7 @@ class MusicDatabase:
|
|||
self._add_profile_settings(cursor)
|
||||
self._add_profile_listenbrainz_support(cursor)
|
||||
self._add_profile_service_credentials(cursor)
|
||||
self._add_soul_id_columns(cursor)
|
||||
|
||||
# Spotify library cache
|
||||
self._add_spotify_library_cache_table(cursor)
|
||||
|
|
@ -2625,6 +2626,42 @@ class MusicDatabase:
|
|||
except Exception as e:
|
||||
logger.error(f"Error in per-profile service credentials migration: {e}")
|
||||
|
||||
def _add_soul_id_columns(self, cursor):
|
||||
"""Add soul_id columns to artists, albums, and tracks tables."""
|
||||
try:
|
||||
# Artists: soul_id
|
||||
cursor.execute("PRAGMA table_info(artists)")
|
||||
artist_cols = [c[1] for c in cursor.fetchall()]
|
||||
if 'soul_id' not in artist_cols:
|
||||
cursor.execute("ALTER TABLE artists ADD COLUMN soul_id TEXT DEFAULT NULL")
|
||||
logger.info("Added soul_id column to artists table")
|
||||
|
||||
# Albums: soul_id
|
||||
cursor.execute("PRAGMA table_info(albums)")
|
||||
album_cols = [c[1] for c in cursor.fetchall()]
|
||||
if 'soul_id' not in album_cols:
|
||||
cursor.execute("ALTER TABLE albums ADD COLUMN soul_id TEXT DEFAULT NULL")
|
||||
logger.info("Added soul_id column to albums table")
|
||||
|
||||
# Tracks: soul_id (song-level) + album_soul_id (release-specific)
|
||||
cursor.execute("PRAGMA table_info(tracks)")
|
||||
track_cols = [c[1] for c in cursor.fetchall()]
|
||||
if 'soul_id' not in track_cols:
|
||||
cursor.execute("ALTER TABLE tracks ADD COLUMN soul_id TEXT DEFAULT NULL")
|
||||
logger.info("Added soul_id column to tracks table")
|
||||
if 'album_soul_id' not in track_cols:
|
||||
cursor.execute("ALTER TABLE tracks ADD COLUMN album_soul_id TEXT DEFAULT NULL")
|
||||
logger.info("Added album_soul_id column to tracks table")
|
||||
|
||||
# Indexes for lookups
|
||||
cursor.execute("CREATE INDEX IF NOT EXISTS idx_artists_soul_id ON artists (soul_id)")
|
||||
cursor.execute("CREATE INDEX IF NOT EXISTS idx_albums_soul_id ON albums (soul_id)")
|
||||
cursor.execute("CREATE INDEX IF NOT EXISTS idx_tracks_soul_id ON tracks (soul_id)")
|
||||
cursor.execute("CREATE INDEX IF NOT EXISTS idx_tracks_album_soul_id ON tracks (album_soul_id)")
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error adding soul_id columns: {e}")
|
||||
|
||||
def set_profile_spotify(self, profile_id: int, client_id: str, client_secret: str,
|
||||
redirect_uri: str = '') -> bool:
|
||||
"""Save Spotify API credentials for a profile (encrypted)."""
|
||||
|
|
@ -7138,6 +7175,7 @@ class MusicDatabase:
|
|||
a.genius_url,
|
||||
a.tidal_id,
|
||||
a.qobuz_id,
|
||||
a.soul_id,
|
||||
COUNT(DISTINCT al.id) as album_count,
|
||||
COUNT(DISTINCT t.id) as track_count
|
||||
FROM artists a
|
||||
|
|
@ -7148,7 +7186,7 @@ class MusicDatabase:
|
|||
WHERE {where_clause}
|
||||
AND a.id = (SELECT MIN(a2.id) FROM artists a2
|
||||
WHERE a2.name = a.name AND a2.server_source = a.server_source)
|
||||
GROUP BY a.id, a.name, a.thumb_url, a.genres, a.musicbrainz_id, a.spotify_artist_id, a.itunes_artist_id, a.deezer_id, a.audiodb_id, a.lastfm_url, a.genius_url, a.tidal_id, a.qobuz_id
|
||||
GROUP BY a.id, a.name, a.thumb_url, a.genres, a.musicbrainz_id, a.spotify_artist_id, a.itunes_artist_id, a.deezer_id, a.audiodb_id, a.lastfm_url, a.genius_url, a.tidal_id, a.qobuz_id, a.soul_id
|
||||
ORDER BY a.name COLLATE NOCASE
|
||||
LIMIT ? OFFSET ?
|
||||
"""
|
||||
|
|
@ -7200,6 +7238,7 @@ class MusicDatabase:
|
|||
'genius_url': row['genius_url'],
|
||||
'tidal_id': row['tidal_id'],
|
||||
'qobuz_id': row['qobuz_id'],
|
||||
'soul_id': row['soul_id'],
|
||||
'album_count': row['album_count'] or 0,
|
||||
'track_count': row['track_count'] or 0,
|
||||
'is_watched': bool(is_watched)
|
||||
|
|
|
|||
|
|
@ -41343,6 +41343,37 @@ def hydrabase_worker_resume():
|
|||
|
||||
from core.repair_worker import RepairWorker
|
||||
|
||||
# ===================================================================
|
||||
# SoulID Worker — generates deterministic soul IDs for library entities
|
||||
# ===================================================================
|
||||
soulid_worker = None
|
||||
try:
|
||||
from core.soulid_worker import SoulIDWorker
|
||||
from database.music_database import MusicDatabase
|
||||
soulid_db = MusicDatabase()
|
||||
soulid_worker = SoulIDWorker(database=soulid_db)
|
||||
soulid_worker.start()
|
||||
print("✅ SoulID worker initialized and started")
|
||||
except Exception as e:
|
||||
print(f"⚠️ SoulID worker initialization failed: {e}")
|
||||
soulid_worker = None
|
||||
|
||||
@app.route('/api/soulid/status', methods=['GET'])
|
||||
def soulid_status():
|
||||
"""Get SoulID worker status for UI polling."""
|
||||
try:
|
||||
if soulid_worker is None:
|
||||
return jsonify({
|
||||
'enabled': False, 'running': False, 'paused': False, 'idle': False,
|
||||
'current_item': None, 'stats': {}
|
||||
})
|
||||
return jsonify(soulid_worker.get_stats())
|
||||
except Exception as e:
|
||||
return jsonify({'error': str(e)}), 500
|
||||
|
||||
# ===================================================================
|
||||
# Repair Worker — Library maintenance and repair jobs
|
||||
# ===================================================================
|
||||
repair_worker = None
|
||||
try:
|
||||
from database.music_database import MusicDatabase
|
||||
|
|
@ -42858,6 +42889,7 @@ def _emit_enrichment_status_loop():
|
|||
'tidal-enrichment': lambda: tidal_enrichment_worker,
|
||||
'qobuz-enrichment': lambda: qobuz_enrichment_worker,
|
||||
'hydrabase': lambda: hydrabase_worker,
|
||||
'soulid': lambda: soulid_worker,
|
||||
'repair': lambda: repair_worker,
|
||||
}
|
||||
while True:
|
||||
|
|
|
|||
|
|
@ -546,6 +546,27 @@
|
|||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<!-- SoulID Worker Status Icon -->
|
||||
<div class="soulid-button-container">
|
||||
<button class="soulid-button" id="soulid-button" title="SoulID Generator">
|
||||
<img src="/static/trans2.png" alt="SoulID" class="soulid-logo">
|
||||
<div class="soulid-spinner"></div>
|
||||
</button>
|
||||
<div class="soulid-tooltip" id="soulid-tooltip">
|
||||
<div class="soulid-tooltip-content">
|
||||
<div class="soulid-tooltip-header">SoulID Generator</div>
|
||||
<div class="soulid-tooltip-body" id="soulid-tooltip-body">
|
||||
<div class="tooltip-status">Status: <span
|
||||
id="soulid-tooltip-status">Idle</span>
|
||||
</div>
|
||||
<div class="tooltip-current" id="soulid-tooltip-current">No items processing
|
||||
</div>
|
||||
<div class="tooltip-progress" id="soulid-tooltip-progress">Progress: 0 pending
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<button class="import-button" id="import-button" onclick="navigateToPage('import')"
|
||||
title="Import Music from Staging">
|
||||
<img src="https://cdn-icons-png.flaticon.com/512/8765/8765164.png" alt="Import"
|
||||
|
|
|
|||
|
|
@ -349,6 +349,7 @@ function initializeWebSocket() {
|
|||
socket.on('enrichment:qobuz-enrichment', (data) => updateQobuzEnrichmentStatusFromData(data));
|
||||
socket.on('enrichment:hydrabase', (data) => updateHydrabaseStatusFromData(data));
|
||||
socket.on('enrichment:repair', (data) => updateRepairStatusFromData(data));
|
||||
socket.on('enrichment:soulid', (data) => updateSoulIDStatusFromData(data));
|
||||
socket.on('repair:progress', (data) => updateRepairJobProgressFromData(data));
|
||||
|
||||
// Phase 4 event listeners (tool progress)
|
||||
|
|
@ -38289,6 +38290,9 @@ function createLibraryArtistCard(artist) {
|
|||
if (artist.qobuz_id) {
|
||||
badgeSources.push({ cls: 'qobuz-card-icon', logo: QOBUZ_LOGO_URL, fallback: 'Qz', title: 'View on Qobuz', url: `https://www.qobuz.com/artist/${artist.qobuz_id}` });
|
||||
}
|
||||
if (artist.soul_id && !artist.soul_id.startsWith('soul_unnamed_')) {
|
||||
badgeSources.push({ cls: 'soulid-card-icon', logo: '/static/trans2.png', fallback: 'SS', title: `SoulID: ${artist.soul_id}`, url: null });
|
||||
}
|
||||
// Add watchlist indicator — only if artist has a usable ID for the active source
|
||||
const hasActiveSourceId = currentMusicSourceName === 'Apple Music'
|
||||
? (artist.itunes_artist_id || artist.spotify_artist_id)
|
||||
|
|
@ -53173,6 +53177,51 @@ function updateRepairStatusFromData(data) {
|
|||
}
|
||||
}
|
||||
|
||||
// ── SoulID Worker Status ──
|
||||
|
||||
function updateSoulIDStatusFromData(data) {
|
||||
const button = document.getElementById('soulid-button');
|
||||
if (!button) return;
|
||||
|
||||
button.classList.remove('active', 'complete');
|
||||
if (data.idle) {
|
||||
button.classList.add('complete');
|
||||
} else if (data.running && !data.paused) {
|
||||
button.classList.add('active');
|
||||
}
|
||||
|
||||
const tooltipStatus = document.getElementById('soulid-tooltip-status');
|
||||
const tooltipCurrent = document.getElementById('soulid-tooltip-current');
|
||||
const tooltipProgress = document.getElementById('soulid-tooltip-progress');
|
||||
|
||||
if (tooltipStatus) {
|
||||
if (data.idle) tooltipStatus.textContent = 'Complete';
|
||||
else if (data.running && !data.paused) tooltipStatus.textContent = 'Running';
|
||||
else if (data.paused) tooltipStatus.textContent = 'Paused';
|
||||
else tooltipStatus.textContent = 'Idle';
|
||||
}
|
||||
|
||||
if (tooltipCurrent) {
|
||||
if (data.current_item) {
|
||||
tooltipCurrent.textContent = data.current_item;
|
||||
} else if (data.idle) {
|
||||
tooltipCurrent.textContent = 'All entities have soul IDs';
|
||||
} else {
|
||||
tooltipCurrent.textContent = 'No items processing';
|
||||
}
|
||||
}
|
||||
|
||||
if (tooltipProgress && data.stats) {
|
||||
const s = data.stats;
|
||||
const parts = [];
|
||||
if (s.artists_processed) parts.push(`Artists: ${s.artists_processed}`);
|
||||
if (s.albums_processed) parts.push(`Albums: ${s.albums_processed}`);
|
||||
if (s.tracks_processed) parts.push(`Tracks: ${s.tracks_processed}`);
|
||||
if (s.pending > 0) parts.push(`Pending: ${s.pending}`);
|
||||
tooltipProgress.textContent = parts.length ? parts.join(' · ') : 'No items processed yet';
|
||||
}
|
||||
}
|
||||
|
||||
// ── Repair Modal State ──
|
||||
let _repairCurrentTab = 'jobs';
|
||||
let _repairFindingsPage = 0;
|
||||
|
|
|
|||
|
|
@ -31813,6 +31813,205 @@ body {
|
|||
border-bottom: 8px solid rgba(30, 30, 30, 0.98);
|
||||
}
|
||||
|
||||
/* ============================================================================
|
||||
SOULID WORKER BUTTON + TOOLTIP
|
||||
Uses SoulSync logo (whisoul.png) with rainbow spinner like repair worker
|
||||
============================================================================ */
|
||||
|
||||
.soulid-button-container {
|
||||
position: relative;
|
||||
display: flex;
|
||||
align-items: center;
|
||||
}
|
||||
|
||||
.soulid-button {
|
||||
position: relative;
|
||||
width: 44px;
|
||||
height: 44px;
|
||||
background: linear-gradient(135deg,
|
||||
rgba(29, 185, 84, 0.12) 0%,
|
||||
rgba(100, 80, 220, 0.18) 100%);
|
||||
backdrop-filter: blur(20px) saturate(1.4);
|
||||
-webkit-backdrop-filter: blur(20px) saturate(1.4);
|
||||
border: 1.5px solid rgba(29, 185, 84, 0.25);
|
||||
border-radius: 50%;
|
||||
cursor: pointer;
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
transition: all 0.3s cubic-bezier(0.4, 0, 0.2, 1);
|
||||
box-shadow:
|
||||
0 4px 16px rgba(29, 185, 84, 0.15),
|
||||
0 2px 8px rgba(0, 0, 0, 0.15),
|
||||
inset 0 1px 0 rgba(255, 255, 255, 0.08);
|
||||
}
|
||||
|
||||
.soulid-button:hover {
|
||||
background: linear-gradient(135deg,
|
||||
rgba(29, 185, 84, 0.18) 0%,
|
||||
rgba(100, 80, 220, 0.25) 100%);
|
||||
border-color: rgba(29, 185, 84, 0.4);
|
||||
transform: scale(1.05);
|
||||
box-shadow:
|
||||
0 6px 20px rgba(29, 185, 84, 0.25),
|
||||
0 3px 12px rgba(0, 0, 0, 0.2),
|
||||
inset 0 1px 0 rgba(255, 255, 255, 0.12);
|
||||
}
|
||||
|
||||
.soulid-button:active {
|
||||
transform: scale(0.95);
|
||||
}
|
||||
|
||||
.soulid-logo {
|
||||
width: 24px;
|
||||
height: 24px;
|
||||
object-fit: cover;
|
||||
border-radius: 50%;
|
||||
opacity: 0.85;
|
||||
transition: opacity 0.3s ease;
|
||||
filter: drop-shadow(0 2px 4px rgba(0, 0, 0, 0.3));
|
||||
}
|
||||
|
||||
.soulid-button:hover .soulid-logo {
|
||||
opacity: 1;
|
||||
}
|
||||
|
||||
.soulid-spinner {
|
||||
position: absolute;
|
||||
width: 44px;
|
||||
height: 44px;
|
||||
border: 3px solid transparent;
|
||||
border-top-color: rgba(29, 185, 84, 0.8);
|
||||
border-right-color: rgba(100, 80, 220, 0.5);
|
||||
border-radius: 50%;
|
||||
opacity: 0;
|
||||
transition: opacity 0.3s ease;
|
||||
pointer-events: none;
|
||||
}
|
||||
|
||||
/* Active State — Rainbow spinner */
|
||||
.soulid-button.active .soulid-spinner {
|
||||
opacity: 1;
|
||||
animation: repair-spin 1.2s linear infinite, rainbow-spinner 3s linear infinite;
|
||||
}
|
||||
|
||||
.soulid-button.active {
|
||||
animation: rainbow-glow 3s linear infinite;
|
||||
background: linear-gradient(135deg,
|
||||
rgba(29, 185, 84, 0.22) 0%,
|
||||
rgba(100, 80, 220, 0.28) 100%);
|
||||
}
|
||||
|
||||
.soulid-button.active .soulid-logo {
|
||||
opacity: 1;
|
||||
filter: drop-shadow(0 0 8px rgba(29, 185, 84, 0.6));
|
||||
}
|
||||
|
||||
/* Complete/Idle State */
|
||||
.soulid-button.complete {
|
||||
background: linear-gradient(135deg,
|
||||
rgba(76, 175, 80, 0.15) 0%,
|
||||
rgba(56, 142, 60, 0.20) 100%);
|
||||
border-color: rgba(76, 175, 80, 0.35);
|
||||
box-shadow:
|
||||
0 4px 16px rgba(76, 175, 80, 0.2),
|
||||
0 2px 8px rgba(0, 0, 0, 0.15),
|
||||
inset 0 1px 0 rgba(255, 255, 255, 0.08);
|
||||
}
|
||||
|
||||
.soulid-button.complete .soulid-logo {
|
||||
opacity: 0.85;
|
||||
filter: drop-shadow(0 0 6px rgba(76, 175, 80, 0.5));
|
||||
}
|
||||
|
||||
.soulid-button.complete .soulid-spinner {
|
||||
opacity: 0;
|
||||
}
|
||||
|
||||
/* Tooltip */
|
||||
.soulid-tooltip {
|
||||
position: absolute;
|
||||
left: 50%;
|
||||
top: calc(100% + 12px);
|
||||
transform: translateX(-50%) translateY(-5px);
|
||||
z-index: 1000;
|
||||
opacity: 0;
|
||||
visibility: hidden;
|
||||
transition: all 0.3s cubic-bezier(0.4, 0, 0.2, 1);
|
||||
pointer-events: none;
|
||||
}
|
||||
|
||||
.soulid-button:hover+.soulid-tooltip {
|
||||
opacity: 1;
|
||||
visibility: visible;
|
||||
transform: translateX(-50%) translateY(0);
|
||||
}
|
||||
|
||||
.soulid-tooltip-content {
|
||||
min-width: 260px;
|
||||
background: linear-gradient(135deg,
|
||||
rgba(30, 30, 30, 0.98) 0%,
|
||||
rgba(20, 20, 20, 0.99) 100%);
|
||||
backdrop-filter: blur(40px) saturate(1.6);
|
||||
-webkit-backdrop-filter: blur(40px) saturate(1.6);
|
||||
border: 1px solid rgba(29, 185, 84, 0.3);
|
||||
border-radius: 16px;
|
||||
padding: 16px 18px;
|
||||
box-shadow:
|
||||
0 12px 40px rgba(0, 0, 0, 0.5),
|
||||
0 6px 20px rgba(29, 185, 84, 0.15),
|
||||
inset 0 1px 0 rgba(255, 255, 255, 0.1);
|
||||
}
|
||||
|
||||
.soulid-tooltip-header {
|
||||
font-family: 'SF Pro Display', -apple-system, sans-serif;
|
||||
font-size: 13px;
|
||||
font-weight: 600;
|
||||
color: rgba(29, 185, 84, 0.95);
|
||||
letter-spacing: -0.2px;
|
||||
margin-bottom: 12px;
|
||||
padding-bottom: 10px;
|
||||
border-bottom: 1px solid rgba(255, 255, 255, 0.08);
|
||||
}
|
||||
|
||||
.soulid-tooltip-body {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: 8px;
|
||||
}
|
||||
|
||||
#soulid-tooltip-status {
|
||||
color: rgb(var(--accent-light-rgb));
|
||||
font-weight: 600;
|
||||
}
|
||||
|
||||
/* Tooltip Arrow */
|
||||
.soulid-tooltip-content::before {
|
||||
content: '';
|
||||
position: absolute;
|
||||
left: 50%;
|
||||
top: -8px;
|
||||
transform: translateX(-50%);
|
||||
width: 0;
|
||||
height: 0;
|
||||
border-left: 6px solid transparent;
|
||||
border-right: 6px solid transparent;
|
||||
border-bottom: 8px solid rgba(29, 185, 84, 0.3);
|
||||
}
|
||||
|
||||
.soulid-tooltip-content::after {
|
||||
content: '';
|
||||
position: absolute;
|
||||
left: 50%;
|
||||
top: -7px;
|
||||
transform: translateX(-50%);
|
||||
width: 0;
|
||||
height: 0;
|
||||
border-left: 6px solid transparent;
|
||||
border-right: 6px solid transparent;
|
||||
border-bottom: 8px solid rgba(30, 30, 30, 0.98);
|
||||
}
|
||||
|
||||
/* MusicBrainz Integration */
|
||||
.release-card {
|
||||
position: relative !important;
|
||||
|
|
|
|||
BIN
webui/static/trans.png
Normal file
BIN
webui/static/trans.png
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 189 KiB |
|
|
@ -21,6 +21,7 @@
|
|||
{ container: '.tidal-enrich-button-container', color: [180, 180, 255] },
|
||||
{ container: '.qobuz-enrich-button-container', color: [1, 112, 239] },
|
||||
{ container: '.hydrabase-button-container', color: [200, 200, 200] },
|
||||
{ container: '.soulid-button-container', color: [29, 185, 84], rainbow: true },
|
||||
{ container: '.repair-button-container', color: [180, 130, 255], rainbow: true },
|
||||
];
|
||||
|
||||
|
|
|
|||
Loading…
Reference in a new issue