Dashboard 'enrichment bubbles' could pause/hover but offered no way to
*manage* a worker. This adds a full management modal opened from a new
header button, covering all 11 enrichment sources.
Backend (testable core helper + seam tests; no live-DB dependency):
- core/enrichment/unmatched.py: pure, whitelisted SQL builders for the
unmatched browser. service/entity validated against a support map (never
interpolated raw); search + pagination bound as params; tracks join albums
for artwork; limit capped at 200.
- database/music_database.py: get_enrichment_unmatched() +
get_enrichment_breakdown() (the breakdown splits matched/not_found/pending,
which the existing get_stats().progress lumps together).
- core/enrichment/api.py: GET /api/enrichment/<id>/{unmatched,breakdown} on
the existing blueprint + a db_getter hook.
- web_server.py: wire db_getter=get_database.
- tests/enrichment/test_unmatched.py: 19 tests across builders, DB methods,
and Flask routes.
Frontend (vanilla, matches app conventions):
- webui/static/enrichment-manager.js: worker rail with live status + coverage
micro-bars, accent-themed detail panel (hero header, segmented matched/
not_found/pending stat cards, current item, pause/resume), and a searchable
paginated unmatched browser with inline manual match (reusing
search-service + manual-match) and retry (clear-match re-queues).
- Polish: entrance/exit motion, scroll-lock, Escape, refresh control,
flicker-free polling (in-place updates), skeleton loaders, relative
timestamps, per-worker accent theming, real dashboard logos reused at
runtime (with the same invert/circle treatment), responsive rail.
- index.html: header button + script include. style.css: full styling.
Reuses existing pause/resume, status, and manual search+assign endpoints.
Backend tests green (19 new + 11 existing enrichment tests).
214 lines
7.2 KiB
Python
214 lines
7.2 KiB
Python
"""Read-side helpers for browsing the items an enrichment source hasn't matched.
|
|
|
|
The dashboard "Manage Enrichment Workers" modal lists, per source, the
|
|
artists / albums / tracks whose ``<service>_match_status`` is ``'not_found'``
|
|
(or still pending = ``NULL``) so the user can manually match them. Every
|
|
enrichment source writes a uniform ``<service>_match_status`` column, so one
|
|
parametric query serves all 11 workers.
|
|
|
|
This module owns the column mapping and SQL construction. ``service`` and
|
|
``entity_type`` are whitelisted against :data:`SERVICE_ENTITY_SUPPORT` and the
|
|
entity table map before any column name is interpolated — user-supplied values
|
|
(the search term, pagination) are always bound parameters, never interpolated.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
from typing import List, Optional, Tuple
|
|
|
|
# Which entity types each enrichment source covers. Mirrors the authoritative
|
|
# ``_SERVICE_ID_COLUMNS`` map in web_server.py (used by manual-match), kept here
|
|
# so the unmatched browser is self-contained and unit-testable. Singular keys
|
|
# ('artist'/'album'/'track') match the manual-match entity_type vocabulary.
|
|
SERVICE_ENTITY_SUPPORT = {
|
|
'spotify': ('artist', 'album', 'track'),
|
|
'musicbrainz': ('artist', 'album', 'track'),
|
|
'deezer': ('artist', 'album', 'track'),
|
|
'audiodb': ('artist', 'album', 'track'),
|
|
'discogs': ('artist', 'album'), # no track-level id column
|
|
'itunes': ('artist', 'album', 'track'),
|
|
'lastfm': ('artist', 'album', 'track'),
|
|
'genius': ('artist', 'track'), # no album-level id column
|
|
'tidal': ('artist', 'album', 'track'),
|
|
'qobuz': ('artist', 'album', 'track'),
|
|
'amazon': ('artist', 'album', 'track'),
|
|
}
|
|
|
|
# entity_type -> table / display-name column / image expression / optional join.
|
|
# tracks carry no artwork column of their own, so we borrow the parent album's.
|
|
_ENTITY_TABLE = {
|
|
'artist': {
|
|
'table': 'artists', 'name': 'name',
|
|
'image': 'artists.thumb_url', 'join': '',
|
|
},
|
|
'album': {
|
|
'table': 'albums', 'name': 'title',
|
|
'image': 'albums.thumb_url', 'join': '',
|
|
},
|
|
'track': {
|
|
'table': 'tracks', 'name': 'title',
|
|
'image': 'al.thumb_url',
|
|
'join': 'LEFT JOIN albums al ON tracks.album_id = al.id',
|
|
},
|
|
}
|
|
|
|
# 'unmatched' = not yet matched at all (pending OR explicitly not_found).
|
|
VALID_STATUSES = ('not_found', 'pending', 'unmatched')
|
|
|
|
# Hard cap so a malicious/buggy caller can't ask for the whole library at once.
|
|
MAX_LIMIT = 200
|
|
|
|
|
|
class UnmatchedQueryError(ValueError):
|
|
"""Raised for an unknown service / unsupported entity type / bad status."""
|
|
|
|
|
|
def supported_entity_types(service: str) -> Tuple[str, ...]:
|
|
"""Return the entity types a source enriches, or () for an unknown source."""
|
|
return SERVICE_ENTITY_SUPPORT.get(service, ())
|
|
|
|
|
|
def match_status_column(service: str) -> str:
|
|
return f"{service}_match_status"
|
|
|
|
|
|
def last_attempted_column(service: str) -> str:
|
|
return f"{service}_last_attempted"
|
|
|
|
|
|
def _validate(service: str, entity_type: str) -> None:
|
|
support = SERVICE_ENTITY_SUPPORT.get(service)
|
|
if support is None:
|
|
raise UnmatchedQueryError(f"Unknown enrichment service: {service!r}")
|
|
if entity_type not in support:
|
|
raise UnmatchedQueryError(
|
|
f"{service} does not enrich {entity_type!r} entities"
|
|
)
|
|
if entity_type not in _ENTITY_TABLE: # defensive — support map drift
|
|
raise UnmatchedQueryError(f"No table mapping for entity type {entity_type!r}")
|
|
|
|
|
|
def _status_predicate(service: str, status: str, qualifier: str) -> str:
|
|
"""SQL predicate selecting rows in the requested match state.
|
|
|
|
``qualifier`` (the table name/alias) is always prefixed so the predicate is
|
|
unambiguous even when the query joins a second table that also carries a
|
|
``<service>_match_status`` column (tracks LEFT JOIN albums).
|
|
"""
|
|
col = f"{qualifier}.{match_status_column(service)}"
|
|
if status == 'not_found':
|
|
return f"{col} = 'not_found'"
|
|
if status == 'pending':
|
|
return f"{col} IS NULL"
|
|
# 'unmatched'
|
|
return f"({col} IS NULL OR {col} = 'not_found')"
|
|
|
|
|
|
def build_unmatched_query(
|
|
service: str,
|
|
entity_type: str,
|
|
status: str = 'not_found',
|
|
query: Optional[str] = None,
|
|
limit: int = 50,
|
|
offset: int = 0,
|
|
) -> Tuple[str, List]:
|
|
"""Build the paginated SELECT for one (service, entity_type, status) view.
|
|
|
|
Returns ``(sql, params)``. Selected columns: id, name, image_url, status,
|
|
last_attempted.
|
|
"""
|
|
_validate(service, entity_type)
|
|
if status not in VALID_STATUSES:
|
|
raise UnmatchedQueryError(f"Invalid status: {status!r}")
|
|
|
|
meta = _ENTITY_TABLE[entity_type]
|
|
table, name_col, image_expr, join = (
|
|
meta['table'], meta['name'], meta['image'], meta['join'],
|
|
)
|
|
ms = match_status_column(service)
|
|
la = last_attempted_column(service)
|
|
|
|
where = [_status_predicate(service, status, table)]
|
|
params: List = []
|
|
if query:
|
|
where.append(f"{table}.{name_col} LIKE ?")
|
|
params.append(f"%{query}%")
|
|
|
|
sql = (
|
|
f"SELECT {table}.id AS id, {table}.{name_col} AS name, "
|
|
f"{image_expr} AS image_url, {table}.{ms} AS status, "
|
|
f"{table}.{la} AS last_attempted "
|
|
f"FROM {table} {join} "
|
|
f"WHERE {' AND '.join(where)} "
|
|
f"ORDER BY {table}.{name_col} COLLATE NOCASE "
|
|
f"LIMIT ? OFFSET ?"
|
|
).replace(' ', ' ')
|
|
|
|
params.append(_clamp_limit(limit))
|
|
params.append(max(int(offset or 0), 0))
|
|
return sql, params
|
|
|
|
|
|
def build_count_query(
|
|
service: str,
|
|
entity_type: str,
|
|
status: str = 'not_found',
|
|
query: Optional[str] = None,
|
|
) -> Tuple[str, List]:
|
|
"""Build the COUNT(*) matching :func:`build_unmatched_query`'s filters."""
|
|
_validate(service, entity_type)
|
|
if status not in VALID_STATUSES:
|
|
raise UnmatchedQueryError(f"Invalid status: {status!r}")
|
|
|
|
meta = _ENTITY_TABLE[entity_type]
|
|
table, name_col = meta['table'], meta['name']
|
|
|
|
where = [_status_predicate(service, status, table)]
|
|
params: List = []
|
|
if query:
|
|
where.append(f"{table}.{name_col} LIKE ?")
|
|
params.append(f"%{query}%")
|
|
|
|
sql = f"SELECT COUNT(*) FROM {table} WHERE {' AND '.join(where)}"
|
|
return sql, params
|
|
|
|
|
|
def build_breakdown_query(service: str, entity_type: str) -> Tuple[str, List]:
|
|
"""Build the matched / not_found / pending / total tally for one entity type."""
|
|
_validate(service, entity_type)
|
|
meta = _ENTITY_TABLE[entity_type]
|
|
table = meta['table']
|
|
ms = f"{table}.{match_status_column(service)}"
|
|
sql = (
|
|
"SELECT "
|
|
f"SUM(CASE WHEN {ms} = 'matched' THEN 1 ELSE 0 END) AS matched, "
|
|
f"SUM(CASE WHEN {ms} = 'not_found' THEN 1 ELSE 0 END) AS not_found, "
|
|
f"SUM(CASE WHEN {ms} IS NULL THEN 1 ELSE 0 END) AS pending, "
|
|
f"COUNT(*) AS total "
|
|
f"FROM {table}"
|
|
)
|
|
return sql, []
|
|
|
|
|
|
def _clamp_limit(limit) -> int:
|
|
try:
|
|
n = int(limit)
|
|
except (TypeError, ValueError):
|
|
return 50
|
|
if n <= 0:
|
|
return 50
|
|
return min(n, MAX_LIMIT)
|
|
|
|
|
|
__all__ = [
|
|
'SERVICE_ENTITY_SUPPORT',
|
|
'VALID_STATUSES',
|
|
'MAX_LIMIT',
|
|
'UnmatchedQueryError',
|
|
'supported_entity_types',
|
|
'match_status_column',
|
|
'last_attempted_column',
|
|
'build_unmatched_query',
|
|
'build_count_query',
|
|
'build_breakdown_query',
|
|
]
|