Merge pull request #778 from Nezreka/feature/enrichment-workers-manager
Feature/enrichment workers manager
This commit is contained in:
BoulderBadgeDad
GitHub
commit
73a8882ad7
22 changed files with 2735 additions and 5 deletions
|
|
@ -174,6 +174,16 @@ class AmazonWorker:
|
|||
cursor = conn.cursor()
|
||||
self._ensure_amazon_schema(cursor)
|
||||
|
||||
# Pinned-group override (Manage Enrichment Workers): process one
|
||||
# entity type first, then fall through to the normal chain. Unset or
|
||||
# exhausted ⇒ default artist→album→track order, unchanged.
|
||||
from core.worker_utils import read_enrichment_priority, priority_pending_item
|
||||
_prio = read_enrichment_priority('amazon')
|
||||
if _prio:
|
||||
_pi = priority_pending_item(cursor, 'amazon', _prio)
|
||||
if _pi:
|
||||
return _pi
|
||||
|
||||
# Priority 1: Unattempted artists
|
||||
cursor.execute("""
|
||||
SELECT id, name FROM artists
|
||||
|
|
|
|||
|
|
@ -162,6 +162,16 @@ class AudioDBWorker:
|
|||
conn = self.db._get_connection()
|
||||
cursor = conn.cursor()
|
||||
|
||||
# Pinned-group override (Manage Enrichment Workers): process one
|
||||
# entity type first, then fall through to the normal chain. Unset or
|
||||
# exhausted ⇒ default artist→album→track order, unchanged.
|
||||
from core.worker_utils import read_enrichment_priority, priority_pending_item
|
||||
_prio = read_enrichment_priority('audiodb')
|
||||
if _prio:
|
||||
_pi = priority_pending_item(cursor, 'audiodb', _prio)
|
||||
if _pi:
|
||||
return _pi
|
||||
|
||||
# Priority 1: Unattempted artists
|
||||
cursor.execute("""
|
||||
SELECT id, name
|
||||
|
|
|
|||
|
|
@ -163,6 +163,16 @@ class DeezerWorker:
|
|||
conn = self.db._get_connection()
|
||||
cursor = conn.cursor()
|
||||
|
||||
# Pinned-group override (Manage Enrichment Workers): process one
|
||||
# entity type first, then fall through to the normal chain. Unset or
|
||||
# exhausted ⇒ default artist→album→track order, unchanged.
|
||||
from core.worker_utils import read_enrichment_priority, priority_pending_item
|
||||
_prio = read_enrichment_priority('deezer')
|
||||
if _prio:
|
||||
_pi = priority_pending_item(cursor, 'deezer', _prio)
|
||||
if _pi:
|
||||
return _pi
|
||||
|
||||
# Priority 1: Unattempted artists
|
||||
cursor.execute("""
|
||||
SELECT id, name
|
||||
|
|
|
|||
|
|
@ -174,6 +174,16 @@ class DiscogsWorker:
|
|||
conn = self.db._get_connection()
|
||||
cursor = conn.cursor()
|
||||
|
||||
# Pinned-group override (Manage Enrichment Workers): process one
|
||||
# entity type first, then fall through to the normal chain. Discogs
|
||||
# has no track endpoint, so only artist/album are honored.
|
||||
from core.worker_utils import read_enrichment_priority, priority_pending_item
|
||||
_prio = read_enrichment_priority('discogs')
|
||||
if _prio in ('artist', 'album'):
|
||||
_pi = priority_pending_item(cursor, 'discogs', _prio)
|
||||
if _pi:
|
||||
return _pi
|
||||
|
||||
# Priority 1: Unattempted artists
|
||||
cursor.execute("""
|
||||
SELECT id, name FROM artists
|
||||
|
|
|
|||
|
|
@ -18,9 +18,14 @@ from __future__ import annotations
|
|||
|
||||
from typing import Any, Callable, Optional
|
||||
|
||||
from flask import Blueprint, jsonify
|
||||
from flask import Blueprint, jsonify, request
|
||||
|
||||
from core.enrichment.services import EnrichmentService, get_service
|
||||
from core.enrichment.unmatched import (
|
||||
SERVICE_ENTITY_SUPPORT,
|
||||
UnmatchedQueryError,
|
||||
supported_entity_types,
|
||||
)
|
||||
from utils.logging_config import get_logger
|
||||
|
||||
|
||||
|
|
@ -30,25 +35,33 @@ logger = get_logger("enrichment.api")
|
|||
# Hooks the host wires up so the blueprint can persist pause state and
|
||||
# clean up auto-pause / yield-override sets without circular imports.
|
||||
_config_set: Optional[Callable[[str, Any], None]] = None
|
||||
_config_get: Optional[Callable[[str, Any], Any]] = None
|
||||
_auto_paused_discard: Optional[Callable[[str], None]] = None
|
||||
_yield_override_add: Optional[Callable[[str], None]] = None
|
||||
_db_getter: Optional[Callable[[], Any]] = None
|
||||
|
||||
|
||||
def configure(
|
||||
*,
|
||||
config_set: Optional[Callable[[str, Any], None]] = None,
|
||||
config_get: Optional[Callable[[str, Any], Any]] = None,
|
||||
auto_paused_discard: Optional[Callable[[str], None]] = None,
|
||||
yield_override_add: Optional[Callable[[str], None]] = None,
|
||||
db_getter: Optional[Callable[[], Any]] = None,
|
||||
) -> None:
|
||||
"""Wire host-side mutators that the generic routes call after pause/resume.
|
||||
|
||||
Each is optional — pass None for hosts that don't have a corresponding
|
||||
mechanism (e.g. tests).
|
||||
mechanism (e.g. tests). ``db_getter`` returns the live ``MusicDatabase``
|
||||
for the unmatched-browser routes; ``config_get``/``config_set`` read and
|
||||
write the per-worker priority override.
|
||||
"""
|
||||
global _config_set, _auto_paused_discard, _yield_override_add
|
||||
global _config_set, _config_get, _auto_paused_discard, _yield_override_add, _db_getter
|
||||
_config_set = config_set
|
||||
_config_get = config_get
|
||||
_auto_paused_discard = auto_paused_discard
|
||||
_yield_override_add = yield_override_add
|
||||
_db_getter = db_getter
|
||||
|
||||
|
||||
def _persist_paused(service: EnrichmentService, paused: bool) -> None:
|
||||
|
|
@ -153,4 +166,133 @@ def create_blueprint() -> Blueprint:
|
|||
logger.error("Error resuming %s worker: %s", service.id, e)
|
||||
return jsonify({'error': str(e)}), 500
|
||||
|
||||
@bp.route('/api/enrichment/<service_id>/breakdown', methods=['GET'])
|
||||
def enrichment_breakdown(service_id: str):
|
||||
"""matched / not_found / pending tallies per entity type for the modal."""
|
||||
if service_id not in SERVICE_ENTITY_SUPPORT:
|
||||
return jsonify({'error': f'Unknown enrichment service: {service_id}'}), 404
|
||||
if _db_getter is None:
|
||||
return jsonify({'error': 'database unavailable'}), 503
|
||||
try:
|
||||
db = _db_getter()
|
||||
breakdown = {
|
||||
entity: db.get_enrichment_breakdown(service_id, entity)
|
||||
for entity in supported_entity_types(service_id)
|
||||
}
|
||||
return jsonify({'service': service_id, 'breakdown': breakdown}), 200
|
||||
except UnmatchedQueryError as e:
|
||||
return jsonify({'error': str(e)}), 400
|
||||
except Exception as e:
|
||||
logger.error("Error building %s enrichment breakdown: %s", service_id, e)
|
||||
return jsonify({'error': str(e)}), 500
|
||||
|
||||
@bp.route('/api/enrichment/<service_id>/unmatched', methods=['GET'])
|
||||
def enrichment_unmatched(service_id: str):
|
||||
"""Paginated list of items this source hasn't matched (for manual match).
|
||||
|
||||
Query params: ``entity_type`` (artist|album|track), ``status``
|
||||
(not_found|pending|unmatched), ``q`` (name search), ``limit``, ``offset``.
|
||||
"""
|
||||
if service_id not in SERVICE_ENTITY_SUPPORT:
|
||||
return jsonify({'error': f'Unknown enrichment service: {service_id}'}), 404
|
||||
if _db_getter is None:
|
||||
return jsonify({'error': 'database unavailable'}), 503
|
||||
|
||||
entity_type = (request.args.get('entity_type') or 'artist').strip()
|
||||
status = (request.args.get('status') or 'not_found').strip()
|
||||
query = (request.args.get('q') or '').strip() or None
|
||||
try:
|
||||
limit = int(request.args.get('limit', 50))
|
||||
offset = int(request.args.get('offset', 0))
|
||||
except (TypeError, ValueError):
|
||||
return jsonify({'error': 'limit/offset must be integers'}), 400
|
||||
|
||||
try:
|
||||
result = _db_getter().get_enrichment_unmatched(
|
||||
service_id, entity_type, status, query, limit, offset
|
||||
)
|
||||
except UnmatchedQueryError as e:
|
||||
return jsonify({'error': str(e)}), 400
|
||||
except Exception as e:
|
||||
logger.error("Error listing %s unmatched %ss: %s", service_id, entity_type, e)
|
||||
return jsonify({'error': str(e)}), 500
|
||||
|
||||
result.update({
|
||||
'service': service_id,
|
||||
'entity_type': entity_type,
|
||||
'status': status,
|
||||
'limit': limit,
|
||||
'offset': offset,
|
||||
'entity_types': list(supported_entity_types(service_id)),
|
||||
})
|
||||
return jsonify(result), 200
|
||||
|
||||
@bp.route('/api/enrichment/<service_id>/retry', methods=['POST'])
|
||||
def enrichment_retry(service_id: str):
|
||||
"""Re-queue item(s) so the worker re-attempts them.
|
||||
|
||||
Body: ``entity_type`` (artist|album|track), ``scope`` (item|failed),
|
||||
``entity_id`` (required when scope='item'). 'failed' re-queues every
|
||||
not_found item of that entity type.
|
||||
"""
|
||||
if service_id not in SERVICE_ENTITY_SUPPORT:
|
||||
return jsonify({'error': f'Unknown enrichment service: {service_id}'}), 404
|
||||
if _db_getter is None:
|
||||
return jsonify({'error': 'database unavailable'}), 503
|
||||
|
||||
data = request.get_json(silent=True) or {}
|
||||
entity_type = (data.get('entity_type') or 'artist').strip()
|
||||
scope = (data.get('scope') or 'item').strip()
|
||||
entity_id = data.get('entity_id')
|
||||
try:
|
||||
count = _db_getter().reset_enrichment(service_id, entity_type, scope, entity_id)
|
||||
except UnmatchedQueryError as e:
|
||||
return jsonify({'error': str(e)}), 400
|
||||
except Exception as e:
|
||||
logger.error("Error re-queuing %s %s (%s): %s", service_id, entity_type, scope, e)
|
||||
return jsonify({'error': str(e)}), 500
|
||||
return jsonify({'success': True, 'reset': count, 'service': service_id,
|
||||
'entity_type': entity_type, 'scope': scope}), 200
|
||||
|
||||
@bp.route('/api/enrichment/<service_id>/priority', methods=['GET'])
|
||||
def enrichment_get_priority(service_id: str):
|
||||
"""Return the pinned 'process this group first' entity for a worker."""
|
||||
if service_id not in SERVICE_ENTITY_SUPPORT:
|
||||
return jsonify({'error': f'Unknown enrichment service: {service_id}'}), 404
|
||||
priority = ''
|
||||
if _config_get is not None:
|
||||
try:
|
||||
priority = (_config_get(f'{service_id}_enrichment_priority', '') or '').strip().lower()
|
||||
except Exception as e:
|
||||
logger.debug("reading %s priority: %s", service_id, e)
|
||||
if priority not in supported_entity_types(service_id):
|
||||
priority = ''
|
||||
return jsonify({'service': service_id, 'priority': priority,
|
||||
'entity_types': list(supported_entity_types(service_id))}), 200
|
||||
|
||||
@bp.route('/api/enrichment/<service_id>/priority', methods=['POST'])
|
||||
def enrichment_set_priority(service_id: str):
|
||||
"""Pin (or clear) the entity type the worker should process first.
|
||||
|
||||
Body: ``entity`` = 'artist'|'album'|'track' to pin, or '' / null / 'none'
|
||||
to clear. Must be an entity type the source actually enriches.
|
||||
"""
|
||||
if service_id not in SERVICE_ENTITY_SUPPORT:
|
||||
return jsonify({'error': f'Unknown enrichment service: {service_id}'}), 404
|
||||
if _config_set is None:
|
||||
return jsonify({'error': 'config unavailable'}), 503
|
||||
data = request.get_json(silent=True) or {}
|
||||
entity = (data.get('entity') or '').strip().lower()
|
||||
if entity in ('none', 'clear'):
|
||||
entity = ''
|
||||
if entity and entity not in supported_entity_types(service_id):
|
||||
return jsonify({'error': f'{service_id} does not enrich {entity!r}'}), 400
|
||||
try:
|
||||
_config_set(f'{service_id}_enrichment_priority', entity)
|
||||
except Exception as e:
|
||||
logger.error("setting %s priority: %s", service_id, e)
|
||||
return jsonify({'error': str(e)}), 500
|
||||
logger.info("%s enrichment priority set to %r via UI", service_id, entity or '(none)')
|
||||
return jsonify({'success': True, 'service': service_id, 'priority': entity}), 200
|
||||
|
||||
return bp
|
||||
|
|
|
|||
263
core/enrichment/unmatched.py
Normal file
263
core/enrichment/unmatched.py
Normal file
|
|
@ -0,0 +1,263 @@
|
|||
"""Read-side helpers for browsing the items an enrichment source hasn't matched.
|
||||
|
||||
The dashboard "Manage Enrichment Workers" modal lists, per source, the
|
||||
artists / albums / tracks whose ``<service>_match_status`` is ``'not_found'``
|
||||
(or still pending = ``NULL``) so the user can manually match them. Every
|
||||
enrichment source writes a uniform ``<service>_match_status`` column, so one
|
||||
parametric query serves all 11 workers.
|
||||
|
||||
This module owns the column mapping and SQL construction. ``service`` and
|
||||
``entity_type`` are whitelisted against :data:`SERVICE_ENTITY_SUPPORT` and the
|
||||
entity table map before any column name is interpolated — user-supplied values
|
||||
(the search term, pagination) are always bound parameters, never interpolated.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import List, Optional, Tuple
|
||||
|
||||
# Which entity types each enrichment source covers. Mirrors the authoritative
|
||||
# ``_SERVICE_ID_COLUMNS`` map in web_server.py (used by manual-match), kept here
|
||||
# so the unmatched browser is self-contained and unit-testable. Singular keys
|
||||
# ('artist'/'album'/'track') match the manual-match entity_type vocabulary.
|
||||
SERVICE_ENTITY_SUPPORT = {
|
||||
'spotify': ('artist', 'album', 'track'),
|
||||
'musicbrainz': ('artist', 'album', 'track'),
|
||||
'deezer': ('artist', 'album', 'track'),
|
||||
'audiodb': ('artist', 'album', 'track'),
|
||||
'discogs': ('artist', 'album'), # no track-level id column
|
||||
'itunes': ('artist', 'album', 'track'),
|
||||
'lastfm': ('artist', 'album', 'track'),
|
||||
'genius': ('artist', 'track'), # no album-level id column
|
||||
'tidal': ('artist', 'album', 'track'),
|
||||
'qobuz': ('artist', 'album', 'track'),
|
||||
'amazon': ('artist', 'album', 'track'),
|
||||
}
|
||||
|
||||
# entity_type -> table / display-name column / image expression / optional join
|
||||
# / parent-context expression (the artist an album belongs to; the album a
|
||||
# track belongs to) so the UI can disambiguate same-named items.
|
||||
# tracks carry no artwork column of their own, so we borrow the parent album's.
|
||||
_ENTITY_TABLE = {
|
||||
'artist': {
|
||||
'table': 'artists', 'name': 'name',
|
||||
'image': 'artists.thumb_url', 'join': '', 'parent': None,
|
||||
},
|
||||
'album': {
|
||||
'table': 'albums', 'name': 'title',
|
||||
'image': 'albums.thumb_url',
|
||||
'join': 'LEFT JOIN artists par ON albums.artist_id = par.id',
|
||||
'parent': 'par.name',
|
||||
},
|
||||
'track': {
|
||||
'table': 'tracks', 'name': 'title',
|
||||
'image': 'al.thumb_url',
|
||||
'join': 'LEFT JOIN albums al ON tracks.album_id = al.id',
|
||||
'parent': 'al.title',
|
||||
},
|
||||
}
|
||||
|
||||
# 'unmatched' = not yet matched at all (pending OR explicitly not_found).
|
||||
VALID_STATUSES = ('not_found', 'pending', 'unmatched')
|
||||
|
||||
# Hard cap so a malicious/buggy caller can't ask for the whole library at once.
|
||||
MAX_LIMIT = 200
|
||||
|
||||
|
||||
class UnmatchedQueryError(ValueError):
|
||||
"""Raised for an unknown service / unsupported entity type / bad status."""
|
||||
|
||||
|
||||
def supported_entity_types(service: str) -> Tuple[str, ...]:
|
||||
"""Return the entity types a source enriches, or () for an unknown source."""
|
||||
return SERVICE_ENTITY_SUPPORT.get(service, ())
|
||||
|
||||
|
||||
def match_status_column(service: str) -> str:
|
||||
return f"{service}_match_status"
|
||||
|
||||
|
||||
def last_attempted_column(service: str) -> str:
|
||||
return f"{service}_last_attempted"
|
||||
|
||||
|
||||
def _validate(service: str, entity_type: str) -> None:
|
||||
support = SERVICE_ENTITY_SUPPORT.get(service)
|
||||
if support is None:
|
||||
raise UnmatchedQueryError(f"Unknown enrichment service: {service!r}")
|
||||
if entity_type not in support:
|
||||
raise UnmatchedQueryError(
|
||||
f"{service} does not enrich {entity_type!r} entities"
|
||||
)
|
||||
if entity_type not in _ENTITY_TABLE: # defensive — support map drift
|
||||
raise UnmatchedQueryError(f"No table mapping for entity type {entity_type!r}")
|
||||
|
||||
|
||||
def _status_predicate(service: str, status: str, qualifier: str) -> str:
|
||||
"""SQL predicate selecting rows in the requested match state.
|
||||
|
||||
``qualifier`` (the table name/alias) is always prefixed so the predicate is
|
||||
unambiguous even when the query joins a second table that also carries a
|
||||
``<service>_match_status`` column (tracks LEFT JOIN albums).
|
||||
"""
|
||||
col = f"{qualifier}.{match_status_column(service)}"
|
||||
if status == 'not_found':
|
||||
return f"{col} = 'not_found'"
|
||||
if status == 'pending':
|
||||
return f"{col} IS NULL"
|
||||
# 'unmatched'
|
||||
return f"({col} IS NULL OR {col} = 'not_found')"
|
||||
|
||||
|
||||
def build_unmatched_query(
|
||||
service: str,
|
||||
entity_type: str,
|
||||
status: str = 'not_found',
|
||||
query: Optional[str] = None,
|
||||
limit: int = 50,
|
||||
offset: int = 0,
|
||||
) -> Tuple[str, List]:
|
||||
"""Build the paginated SELECT for one (service, entity_type, status) view.
|
||||
|
||||
Returns ``(sql, params)``. Selected columns: id, name, image_url, status,
|
||||
last_attempted.
|
||||
"""
|
||||
_validate(service, entity_type)
|
||||
if status not in VALID_STATUSES:
|
||||
raise UnmatchedQueryError(f"Invalid status: {status!r}")
|
||||
|
||||
meta = _ENTITY_TABLE[entity_type]
|
||||
table, name_col, image_expr, join = (
|
||||
meta['table'], meta['name'], meta['image'], meta['join'],
|
||||
)
|
||||
ms = match_status_column(service)
|
||||
la = last_attempted_column(service)
|
||||
|
||||
where = [_status_predicate(service, status, table)]
|
||||
params: List = []
|
||||
if query:
|
||||
where.append(f"{table}.{name_col} LIKE ?")
|
||||
params.append(f"%{query}%")
|
||||
|
||||
parent_expr = meta.get('parent')
|
||||
parent_select = f"{parent_expr} AS parent" if parent_expr else "NULL AS parent"
|
||||
sql = (
|
||||
f"SELECT {table}.id AS id, {table}.{name_col} AS name, "
|
||||
f"{image_expr} AS image_url, {parent_select}, {table}.{ms} AS status, "
|
||||
f"{table}.{la} AS last_attempted "
|
||||
f"FROM {table} {join} "
|
||||
f"WHERE {' AND '.join(where)} "
|
||||
f"ORDER BY {table}.{name_col} COLLATE NOCASE "
|
||||
f"LIMIT ? OFFSET ?"
|
||||
).replace(' ', ' ')
|
||||
|
||||
params.append(_clamp_limit(limit))
|
||||
params.append(max(int(offset or 0), 0))
|
||||
return sql, params
|
||||
|
||||
|
||||
def build_count_query(
|
||||
service: str,
|
||||
entity_type: str,
|
||||
status: str = 'not_found',
|
||||
query: Optional[str] = None,
|
||||
) -> Tuple[str, List]:
|
||||
"""Build the COUNT(*) matching :func:`build_unmatched_query`'s filters."""
|
||||
_validate(service, entity_type)
|
||||
if status not in VALID_STATUSES:
|
||||
raise UnmatchedQueryError(f"Invalid status: {status!r}")
|
||||
|
||||
meta = _ENTITY_TABLE[entity_type]
|
||||
table, name_col = meta['table'], meta['name']
|
||||
|
||||
where = [_status_predicate(service, status, table)]
|
||||
params: List = []
|
||||
if query:
|
||||
where.append(f"{table}.{name_col} LIKE ?")
|
||||
params.append(f"%{query}%")
|
||||
|
||||
sql = f"SELECT COUNT(*) FROM {table} WHERE {' AND '.join(where)}"
|
||||
return sql, params
|
||||
|
||||
|
||||
# Reset scopes for re-queuing items so the worker re-attempts them.
|
||||
RESET_SCOPES = ('item', 'failed')
|
||||
|
||||
|
||||
def build_reset_query(
|
||||
service: str,
|
||||
entity_type: str,
|
||||
scope: str = 'item',
|
||||
entity_id=None,
|
||||
) -> Tuple[str, List]:
|
||||
"""Build the UPDATE that re-queues item(s) for enrichment.
|
||||
|
||||
Re-queuing means clearing ``<service>_match_status`` back to NULL (and
|
||||
``<service>_last_attempted`` to NULL): every worker's pending query selects
|
||||
``match_status IS NULL`` first, so the item is retried on the next pass.
|
||||
Nulling last_attempted alone is NOT enough — the not_found retry path uses
|
||||
``last_attempted < cutoff`` and ``NULL < cutoff`` is false, so the item
|
||||
would never be picked up.
|
||||
|
||||
* scope='item' -> a single row (requires entity_id)
|
||||
* scope='failed' -> every 'not_found' row for this entity type
|
||||
"""
|
||||
_validate(service, entity_type)
|
||||
if scope not in RESET_SCOPES:
|
||||
raise UnmatchedQueryError(f"Invalid reset scope: {scope!r}")
|
||||
|
||||
meta = _ENTITY_TABLE[entity_type]
|
||||
table = meta['table']
|
||||
ms = match_status_column(service)
|
||||
la = last_attempted_column(service)
|
||||
set_clause = f"SET {ms} = NULL, {la} = NULL"
|
||||
|
||||
if scope == 'item':
|
||||
if not entity_id:
|
||||
raise UnmatchedQueryError("entity_id is required for an item reset")
|
||||
return f"UPDATE {table} {set_clause} WHERE id = ?", [entity_id]
|
||||
# 'failed' — re-queue everything this source explicitly gave up on.
|
||||
return f"UPDATE {table} {set_clause} WHERE {ms} = 'not_found'", []
|
||||
|
||||
|
||||
def build_breakdown_query(service: str, entity_type: str) -> Tuple[str, List]:
|
||||
"""Build the matched / not_found / pending / total tally for one entity type."""
|
||||
_validate(service, entity_type)
|
||||
meta = _ENTITY_TABLE[entity_type]
|
||||
table = meta['table']
|
||||
ms = f"{table}.{match_status_column(service)}"
|
||||
sql = (
|
||||
"SELECT "
|
||||
f"SUM(CASE WHEN {ms} = 'matched' THEN 1 ELSE 0 END) AS matched, "
|
||||
f"SUM(CASE WHEN {ms} = 'not_found' THEN 1 ELSE 0 END) AS not_found, "
|
||||
f"SUM(CASE WHEN {ms} IS NULL THEN 1 ELSE 0 END) AS pending, "
|
||||
f"COUNT(*) AS total "
|
||||
f"FROM {table}"
|
||||
)
|
||||
return sql, []
|
||||
|
||||
|
||||
def _clamp_limit(limit) -> int:
|
||||
try:
|
||||
n = int(limit)
|
||||
except (TypeError, ValueError):
|
||||
return 50
|
||||
if n <= 0:
|
||||
return 50
|
||||
return min(n, MAX_LIMIT)
|
||||
|
||||
|
||||
__all__ = [
|
||||
'SERVICE_ENTITY_SUPPORT',
|
||||
'VALID_STATUSES',
|
||||
'MAX_LIMIT',
|
||||
'UnmatchedQueryError',
|
||||
'supported_entity_types',
|
||||
'match_status_column',
|
||||
'last_attempted_column',
|
||||
'build_unmatched_query',
|
||||
'build_count_query',
|
||||
'build_breakdown_query',
|
||||
'build_reset_query',
|
||||
'RESET_SCOPES',
|
||||
]
|
||||
|
|
@ -178,6 +178,16 @@ class GeniusWorker:
|
|||
conn = self.db._get_connection()
|
||||
cursor = conn.cursor()
|
||||
|
||||
# Pinned-group override (Manage Enrichment Workers): process one
|
||||
# entity type first, then fall through to the normal chain. Genius
|
||||
# is artist/track only, so albums are not honored.
|
||||
from core.worker_utils import read_enrichment_priority, priority_pending_item
|
||||
_prio = read_enrichment_priority('genius')
|
||||
if _prio in ('artist', 'track'):
|
||||
_pi = priority_pending_item(cursor, 'genius', _prio)
|
||||
if _pi:
|
||||
return _pi
|
||||
|
||||
# Priority 1: Unattempted artists
|
||||
cursor.execute("""
|
||||
SELECT id, name
|
||||
|
|
|
|||
|
|
@ -172,6 +172,17 @@ class iTunesWorker:
|
|||
conn = self.db._get_connection()
|
||||
cursor = conn.cursor()
|
||||
|
||||
# Pinned-group override (Manage Enrichment Workers): process one
|
||||
# entity type first, then fall through to the normal chain. Unset or
|
||||
# exhausted ⇒ default artist→album→track order, unchanged.
|
||||
from core.worker_utils import read_enrichment_priority, priority_pending_item
|
||||
_prio = read_enrichment_priority('itunes')
|
||||
if _prio:
|
||||
_pi = priority_pending_item(cursor, 'itunes', _prio,
|
||||
{'album': 'album_individual', 'track': 'track_individual'})
|
||||
if _pi:
|
||||
return _pi
|
||||
|
||||
# Priority 1: Unattempted artists
|
||||
cursor.execute("""
|
||||
SELECT id, name
|
||||
|
|
|
|||
|
|
@ -177,6 +177,16 @@ class LastFMWorker:
|
|||
conn = self.db._get_connection()
|
||||
cursor = conn.cursor()
|
||||
|
||||
# Pinned-group override (Manage Enrichment Workers): process one
|
||||
# entity type first, then fall through to the normal chain. Unset or
|
||||
# exhausted ⇒ default artist→album→track order, unchanged.
|
||||
from core.worker_utils import read_enrichment_priority, priority_pending_item
|
||||
_prio = read_enrichment_priority('lastfm')
|
||||
if _prio:
|
||||
_pi = priority_pending_item(cursor, 'lastfm', _prio)
|
||||
if _pi:
|
||||
return _pi
|
||||
|
||||
# Priority 1: Unattempted artists
|
||||
cursor.execute("""
|
||||
SELECT id, name
|
||||
|
|
|
|||
|
|
@ -166,6 +166,16 @@ class MusicBrainzWorker:
|
|||
conn = self.db._get_connection()
|
||||
cursor = conn.cursor()
|
||||
|
||||
# Pinned-group override (Manage Enrichment Workers): process one
|
||||
# entity type first, then fall through to the normal chain. Unset or
|
||||
# exhausted ⇒ default artist→album→track order, unchanged.
|
||||
from core.worker_utils import read_enrichment_priority, priority_pending_item
|
||||
_prio = read_enrichment_priority('musicbrainz')
|
||||
if _prio:
|
||||
_pi = priority_pending_item(cursor, 'musicbrainz', _prio)
|
||||
if _pi:
|
||||
return _pi
|
||||
|
||||
# Priority 1: Unattempted artists
|
||||
cursor.execute("""
|
||||
SELECT id, name
|
||||
|
|
|
|||
|
|
@ -186,6 +186,16 @@ class QobuzWorker:
|
|||
conn = self.db._get_connection()
|
||||
cursor = conn.cursor()
|
||||
|
||||
# Pinned-group override (Manage Enrichment Workers): process one
|
||||
# entity type first, then fall through to the normal chain. Unset or
|
||||
# exhausted ⇒ default artist→album→track order, unchanged.
|
||||
from core.worker_utils import read_enrichment_priority, priority_pending_item
|
||||
_prio = read_enrichment_priority('qobuz')
|
||||
if _prio:
|
||||
_pi = priority_pending_item(cursor, 'qobuz', _prio)
|
||||
if _pi:
|
||||
return _pi
|
||||
|
||||
# Priority 1: Unattempted artists
|
||||
cursor.execute("""
|
||||
SELECT id, name
|
||||
|
|
|
|||
|
|
@ -260,6 +260,17 @@ class SpotifyWorker:
|
|||
conn = self.db._get_connection()
|
||||
cursor = conn.cursor()
|
||||
|
||||
# Pinned-group override (Manage Enrichment Workers): process one
|
||||
# entity type first, then fall through to the normal chain. Unset or
|
||||
# exhausted ⇒ default artist→album→track order, unchanged.
|
||||
from core.worker_utils import read_enrichment_priority, priority_pending_item
|
||||
_prio = read_enrichment_priority('spotify')
|
||||
if _prio:
|
||||
_pi = priority_pending_item(cursor, 'spotify', _prio,
|
||||
{'album': 'album_individual', 'track': 'track_individual'})
|
||||
if _pi:
|
||||
return _pi
|
||||
|
||||
# Priority 1: Unattempted artists
|
||||
cursor.execute("""
|
||||
SELECT id, name
|
||||
|
|
|
|||
|
|
@ -198,6 +198,16 @@ class TidalWorker:
|
|||
conn = self.db._get_connection()
|
||||
cursor = conn.cursor()
|
||||
|
||||
# Pinned-group override (Manage Enrichment Workers): process one
|
||||
# entity type first, then fall through to the normal chain. Unset or
|
||||
# exhausted ⇒ default artist→album→track order, unchanged.
|
||||
from core.worker_utils import read_enrichment_priority, priority_pending_item
|
||||
_prio = read_enrichment_priority('tidal')
|
||||
if _prio:
|
||||
_pi = priority_pending_item(cursor, 'tidal', _prio)
|
||||
if _pi:
|
||||
return _pi
|
||||
|
||||
# Priority 1: Unattempted artists
|
||||
cursor.execute("""
|
||||
SELECT id, name
|
||||
|
|
|
|||
|
|
@ -78,3 +78,67 @@ def set_album_api_track_count(cursor, album_id, count):
|
|||
logger.warning(
|
||||
"Failed to cache api_track_count for album %s: %s", album_id, e
|
||||
)
|
||||
|
||||
|
||||
# --- Enrichment "process this group first" override -----------------------
|
||||
# Each enrichment worker normally processes artist -> album -> track. A user
|
||||
# can pin one entity type to run first via the Manage Enrichment Workers modal;
|
||||
# the choice is stored in config as "<service>_enrichment_priority" and read
|
||||
# at the top of each worker's _get_next_item so it takes effect live. When the
|
||||
# pinned group is exhausted (or unset), the worker falls back to its normal
|
||||
# chain — so the default path is unchanged.
|
||||
|
||||
PRIORITY_ENTITIES = ('artist', 'album', 'track')
|
||||
|
||||
|
||||
def read_enrichment_priority(service: str) -> str:
|
||||
"""Return the pinned entity ('artist'|'album'|'track') for a worker, or ''.
|
||||
|
||||
Read every loop so the override applies without restarting the worker.
|
||||
Any error / unset / invalid value yields '' (no override)."""
|
||||
try:
|
||||
from config.settings import config_manager
|
||||
val = (config_manager.get(f'{service}_enrichment_priority', '') or '')
|
||||
val = str(val).strip().lower()
|
||||
return val if val in PRIORITY_ENTITIES else ''
|
||||
except Exception:
|
||||
return ''
|
||||
|
||||
|
||||
def priority_pending_item(cursor, service, entity, type_overrides=None):
|
||||
"""Return one pending (NULL match_status) item of `entity`, or None.
|
||||
|
||||
`service` is the column prefix (e.g. 'spotify' -> spotify_match_status) and
|
||||
MUST be a trusted worker-supplied literal (it is interpolated into SQL).
|
||||
`type_overrides` maps the canonical entity to the worker's dispatch 'type'
|
||||
string — Spotify/iTunes process individual items as 'album_individual' /
|
||||
'track_individual', the other workers use 'album' / 'track'. The returned
|
||||
dict matches the shape those workers already return from _get_next_item."""
|
||||
if not str(service).isalpha() or entity not in PRIORITY_ENTITIES:
|
||||
return None
|
||||
type_overrides = type_overrides or {}
|
||||
ms = f"{service}_match_status"
|
||||
|
||||
if entity == 'artist':
|
||||
cursor.execute(
|
||||
f"SELECT id, name FROM artists WHERE {ms} IS NULL AND id IS NOT NULL "
|
||||
f"ORDER BY id ASC LIMIT 1"
|
||||
)
|
||||
r = cursor.fetchone()
|
||||
return {'type': type_overrides.get('artist', 'artist'), 'id': r[0], 'name': r[1]} if r else None
|
||||
|
||||
if entity == 'album':
|
||||
cursor.execute(
|
||||
f"SELECT a.id, a.title, ar.name FROM albums a JOIN artists ar ON a.artist_id = ar.id "
|
||||
f"WHERE a.{ms} IS NULL AND a.id IS NOT NULL ORDER BY a.id ASC LIMIT 1"
|
||||
)
|
||||
r = cursor.fetchone()
|
||||
return {'type': type_overrides.get('album', 'album'), 'id': r[0], 'name': r[1], 'artist': r[2]} if r else None
|
||||
|
||||
# track
|
||||
cursor.execute(
|
||||
f"SELECT t.id, t.title, ar.name FROM tracks t JOIN artists ar ON t.artist_id = ar.id "
|
||||
f"WHERE t.{ms} IS NULL AND t.id IS NOT NULL ORDER BY t.id ASC LIMIT 1"
|
||||
)
|
||||
r = cursor.fetchone()
|
||||
return {'type': type_overrides.get('track', 'track'), 'id': r[0], 'name': r[1], 'artist': r[2]} if r else None
|
||||
|
|
|
|||
|
|
@ -1021,6 +1021,82 @@ class MusicDatabase:
|
|||
finally:
|
||||
conn.close()
|
||||
|
||||
def get_enrichment_unmatched(
|
||||
self,
|
||||
service: str,
|
||||
entity_type: str,
|
||||
status: str = 'not_found',
|
||||
query: str = None,
|
||||
limit: int = 50,
|
||||
offset: int = 0,
|
||||
) -> dict:
|
||||
"""List items a given enrichment source hasn't matched, paginated.
|
||||
|
||||
Powers the "Manage Enrichment Workers" modal's unmatched browser.
|
||||
Returns ``{'total': int, 'items': [{id, name, image_url, status,
|
||||
last_attempted}]}``. Raises ``UnmatchedQueryError`` for an unknown
|
||||
service / unsupported entity type / bad status (the caller maps that to
|
||||
an HTTP 400)."""
|
||||
from core.enrichment.unmatched import (
|
||||
build_count_query,
|
||||
build_unmatched_query,
|
||||
)
|
||||
|
||||
sql, params = build_unmatched_query(
|
||||
service, entity_type, status, query, limit, offset
|
||||
)
|
||||
count_sql, count_params = build_count_query(service, entity_type, status, query)
|
||||
conn = self._get_connection()
|
||||
try:
|
||||
cursor = conn.cursor()
|
||||
total = cursor.execute(count_sql, count_params).fetchone()[0]
|
||||
rows = cursor.execute(sql, params).fetchall()
|
||||
items = [dict(row) for row in rows]
|
||||
return {'total': total or 0, 'items': items}
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
def get_enrichment_breakdown(self, service: str, entity_type: str) -> dict:
|
||||
"""Return ``{matched, not_found, pending, total}`` for a source/entity.
|
||||
|
||||
The per-worker ``get_stats().progress`` lumps matched + not_found into a
|
||||
single 'processed' count; this splits them so the modal can show the
|
||||
real match rate. Raises ``UnmatchedQueryError`` on bad input."""
|
||||
from core.enrichment.unmatched import build_breakdown_query
|
||||
|
||||
sql, params = build_breakdown_query(service, entity_type)
|
||||
conn = self._get_connection()
|
||||
try:
|
||||
row = conn.cursor().execute(sql, params).fetchone()
|
||||
if not row:
|
||||
return {'matched': 0, 'not_found': 0, 'pending': 0, 'total': 0}
|
||||
return {
|
||||
'matched': row[0] or 0,
|
||||
'not_found': row[1] or 0,
|
||||
'pending': row[2] or 0,
|
||||
'total': row[3] or 0,
|
||||
}
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
def reset_enrichment(self, service: str, entity_type: str, scope: str = 'item', entity_id=None) -> int:
|
||||
"""Re-queue item(s) for a source by clearing match_status back to NULL.
|
||||
|
||||
scope='item' resets one row (entity_id); scope='failed' resets every
|
||||
'not_found' row for that entity type. Returns the number of rows reset.
|
||||
Raises ``UnmatchedQueryError`` on bad input."""
|
||||
from core.enrichment.unmatched import build_reset_query
|
||||
|
||||
sql, params = build_reset_query(service, entity_type, scope, entity_id)
|
||||
conn = self._get_connection()
|
||||
try:
|
||||
cursor = conn.cursor()
|
||||
cursor.execute(sql, params)
|
||||
conn.commit()
|
||||
return cursor.rowcount or 0
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
def _add_mirrored_playlist_explored_column(self, cursor):
|
||||
"""Add explored_at column to mirrored_playlists to persist explore badge."""
|
||||
try:
|
||||
|
|
|
|||
263
tests/enrichment/test_unmatched.py
Normal file
263
tests/enrichment/test_unmatched.py
Normal file
|
|
@ -0,0 +1,263 @@
|
|||
"""Unmatched-browser backend for the Manage Enrichment Workers modal.
|
||||
|
||||
Three seams:
|
||||
* pure SQL builders + validation (core.enrichment.unmatched)
|
||||
* the MusicDatabase read methods against a temp DB
|
||||
* the Flask routes via a test client
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
from flask import Flask
|
||||
|
||||
from core.enrichment import api as enrichment_api
|
||||
from core.enrichment.unmatched import (
|
||||
MAX_LIMIT,
|
||||
UnmatchedQueryError,
|
||||
build_breakdown_query,
|
||||
build_count_query,
|
||||
build_reset_query,
|
||||
build_unmatched_query,
|
||||
supported_entity_types,
|
||||
)
|
||||
from database.music_database import MusicDatabase
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# Pure builders / validation
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
def test_unknown_service_rejected():
|
||||
with pytest.raises(UnmatchedQueryError):
|
||||
build_unmatched_query('not-a-service', 'artist')
|
||||
|
||||
|
||||
def test_unsupported_entity_type_rejected():
|
||||
# Genius enriches artists + tracks but has no album-level id column.
|
||||
assert 'album' not in supported_entity_types('genius')
|
||||
with pytest.raises(UnmatchedQueryError):
|
||||
build_unmatched_query('genius', 'album')
|
||||
with pytest.raises(UnmatchedQueryError):
|
||||
build_breakdown_query('discogs', 'track') # discogs has no track column
|
||||
|
||||
|
||||
def test_bad_status_rejected():
|
||||
with pytest.raises(UnmatchedQueryError):
|
||||
build_unmatched_query('spotify', 'artist', status='bogus')
|
||||
|
||||
|
||||
def test_status_predicates():
|
||||
nf, _ = build_count_query('spotify', 'artist', 'not_found')
|
||||
pend, _ = build_count_query('spotify', 'artist', 'pending')
|
||||
un, _ = build_count_query('spotify', 'artist', 'unmatched')
|
||||
assert "artists.spotify_match_status = 'not_found'" in nf
|
||||
assert "artists.spotify_match_status IS NULL" in pend
|
||||
assert "IS NULL OR" in un and "= 'not_found'" in un
|
||||
|
||||
|
||||
def test_track_query_qualifies_status_to_avoid_join_ambiguity():
|
||||
# tracks LEFT JOIN albums for artwork — both carry spotify_match_status,
|
||||
# so the predicate must be qualified or SQLite errors "ambiguous column".
|
||||
sql, _ = build_unmatched_query('spotify', 'track', 'not_found')
|
||||
assert 'LEFT JOIN albums al' in sql
|
||||
assert 'tracks.spotify_match_status' in sql
|
||||
assert 'al.thumb_url AS image_url' in sql
|
||||
|
||||
|
||||
def test_search_adds_like_param():
|
||||
sql, params = build_unmatched_query('spotify', 'artist', 'not_found', query='dragons')
|
||||
assert 'LIKE ?' in sql
|
||||
assert '%dragons%' in params
|
||||
|
||||
|
||||
def test_limit_is_clamped():
|
||||
_, params = build_unmatched_query('spotify', 'artist', 'not_found', limit=99999)
|
||||
assert params[-2] == MAX_LIMIT # limit
|
||||
assert params[-1] == 0 # offset
|
||||
_, params2 = build_unmatched_query('spotify', 'artist', 'not_found', limit=0)
|
||||
assert params2[-2] == 50 # invalid -> default
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# MusicDatabase integration (temp DB)
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
def _seed(db: MusicDatabase):
|
||||
conn = db._get_connection()
|
||||
cur = conn.cursor()
|
||||
# 3 artists: matched / not_found / pending(NULL)
|
||||
cur.execute("INSERT INTO artists (id, name, spotify_match_status) VALUES ('a1','Matched Artist','matched')")
|
||||
cur.execute("INSERT INTO artists (id, name, spotify_match_status) VALUES ('a2','Failed Dragons','not_found')")
|
||||
cur.execute("INSERT INTO artists (id, name) VALUES ('a3','Pending Person')") # NULL status
|
||||
# album + track to exercise the join-for-artwork path
|
||||
cur.execute("INSERT INTO albums (id, artist_id, title, thumb_url, spotify_match_status) "
|
||||
"VALUES ('al1','a2','Evolve','http://img/evolve.jpg','not_found')")
|
||||
cur.execute("INSERT INTO tracks (id, album_id, artist_id, title, spotify_match_status) "
|
||||
"VALUES ('t1','al1','a2','Believer','not_found')")
|
||||
conn.commit()
|
||||
conn.close()
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def db(tmp_path):
|
||||
d = MusicDatabase(str(tmp_path / 'enrich.db'))
|
||||
_seed(d)
|
||||
return d
|
||||
|
||||
|
||||
def test_breakdown_splits_matched_notfound_pending(db):
|
||||
bd = db.get_enrichment_breakdown('spotify', 'artist')
|
||||
assert bd == {'matched': 1, 'not_found': 1, 'pending': 1, 'total': 3}
|
||||
|
||||
|
||||
def test_unmatched_not_found_only(db):
|
||||
res = db.get_enrichment_unmatched('spotify', 'artist', status='not_found')
|
||||
assert res['total'] == 1
|
||||
assert [i['name'] for i in res['items']] == ['Failed Dragons']
|
||||
assert res['items'][0]['status'] == 'not_found'
|
||||
|
||||
|
||||
def test_unmatched_pending_only(db):
|
||||
res = db.get_enrichment_unmatched('spotify', 'artist', status='pending')
|
||||
assert res['total'] == 1
|
||||
assert res['items'][0]['name'] == 'Pending Person'
|
||||
|
||||
|
||||
def test_unmatched_combined(db):
|
||||
res = db.get_enrichment_unmatched('spotify', 'artist', status='unmatched')
|
||||
assert res['total'] == 2
|
||||
assert {i['name'] for i in res['items']} == {'Failed Dragons', 'Pending Person'}
|
||||
|
||||
|
||||
def test_unmatched_search_filters_by_name(db):
|
||||
res = db.get_enrichment_unmatched('spotify', 'artist', status='unmatched', query='dragons')
|
||||
assert res['total'] == 1
|
||||
assert res['items'][0]['name'] == 'Failed Dragons'
|
||||
|
||||
|
||||
def test_unmatched_pagination(db):
|
||||
page = db.get_enrichment_unmatched('spotify', 'artist', status='unmatched', limit=1, offset=0)
|
||||
assert page['total'] == 2 and len(page['items']) == 1
|
||||
page2 = db.get_enrichment_unmatched('spotify', 'artist', status='unmatched', limit=1, offset=1)
|
||||
assert page2['items'][0]['name'] != page['items'][0]['name']
|
||||
|
||||
|
||||
def test_track_unmatched_borrows_album_artwork(db):
|
||||
res = db.get_enrichment_unmatched('spotify', 'track', status='not_found')
|
||||
assert res['total'] == 1
|
||||
assert res['items'][0]['name'] == 'Believer'
|
||||
assert res['items'][0]['image_url'] == 'http://img/evolve.jpg'
|
||||
|
||||
|
||||
def test_unmatched_includes_parent_context(db):
|
||||
# album's parent is its artist; track's parent is its album
|
||||
album = db.get_enrichment_unmatched('spotify', 'album', status='not_found')['items'][0]
|
||||
assert album['parent'] == 'Failed Dragons'
|
||||
track = db.get_enrichment_unmatched('spotify', 'track', status='not_found')['items'][0]
|
||||
assert track['parent'] == 'Evolve'
|
||||
artist = db.get_enrichment_unmatched('spotify', 'artist', status='not_found')['items'][0]
|
||||
assert artist['parent'] is None
|
||||
|
||||
|
||||
def test_db_raises_on_bad_input(db):
|
||||
with pytest.raises(UnmatchedQueryError):
|
||||
db.get_enrichment_unmatched('spotify', 'artist', status='bogus')
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# Reset / retry (re-queue) — must clear match_status to NULL so the worker
|
||||
# re-attempts (nulling last_attempted alone leaves not_found in limbo).
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
def test_reset_builder_requires_id_for_item():
|
||||
with pytest.raises(UnmatchedQueryError):
|
||||
build_reset_query('spotify', 'artist', 'item') # no entity_id
|
||||
|
||||
|
||||
def test_reset_builder_bad_scope():
|
||||
with pytest.raises(UnmatchedQueryError):
|
||||
build_reset_query('spotify', 'artist', 'bogus', entity_id='x')
|
||||
|
||||
|
||||
def test_reset_builder_nulls_status_not_just_attempted():
|
||||
sql, _ = build_reset_query('spotify', 'artist', 'failed')
|
||||
assert 'spotify_match_status = NULL' in sql
|
||||
assert 'spotify_last_attempted = NULL' in sql
|
||||
assert "WHERE spotify_match_status = 'not_found'" in sql
|
||||
|
||||
|
||||
def test_reset_item_requeues_to_pending(db):
|
||||
n = db.reset_enrichment('spotify', 'artist', 'item', entity_id='a2') # was not_found
|
||||
assert n == 1
|
||||
# not_found dropped by 1, pending gained 1
|
||||
bd = db.get_enrichment_breakdown('spotify', 'artist')
|
||||
assert bd == {'matched': 1, 'not_found': 0, 'pending': 2, 'total': 3}
|
||||
|
||||
|
||||
def test_reset_failed_requeues_all(db):
|
||||
n = db.reset_enrichment('spotify', 'album', 'failed') # one not_found album
|
||||
assert n == 1
|
||||
bd = db.get_enrichment_breakdown('spotify', 'album')
|
||||
assert bd['not_found'] == 0 and bd['pending'] == 1
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# Flask routes
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
@pytest.fixture
|
||||
def client(db):
|
||||
enrichment_api.configure(db_getter=lambda: db)
|
||||
app = Flask(__name__)
|
||||
app.register_blueprint(enrichment_api.create_blueprint())
|
||||
with app.test_client() as c:
|
||||
yield c
|
||||
enrichment_api.configure(db_getter=None) # reset module global
|
||||
|
||||
|
||||
def test_route_unknown_service_404(client):
|
||||
assert client.get('/api/enrichment/bogus/unmatched').status_code == 404
|
||||
|
||||
|
||||
def test_route_bad_entity_type_400(client):
|
||||
# genius has no album column -> 400, not a 500
|
||||
r = client.get('/api/enrichment/genius/unmatched?entity_type=album')
|
||||
assert r.status_code == 400
|
||||
|
||||
|
||||
def test_route_happy_path(client):
|
||||
r = client.get('/api/enrichment/spotify/unmatched?entity_type=artist&status=unmatched')
|
||||
assert r.status_code == 200
|
||||
body = r.get_json()
|
||||
assert body['total'] == 2
|
||||
assert body['service'] == 'spotify'
|
||||
assert body['entity_types'] == ['artist', 'album', 'track']
|
||||
|
||||
|
||||
def test_route_breakdown(client):
|
||||
r = client.get('/api/enrichment/spotify/breakdown')
|
||||
assert r.status_code == 200
|
||||
bd = r.get_json()['breakdown']
|
||||
assert bd['artist'] == {'matched': 1, 'not_found': 1, 'pending': 1, 'total': 3}
|
||||
|
||||
|
||||
def test_route_retry_item(client):
|
||||
r = client.post('/api/enrichment/spotify/retry',
|
||||
json={'entity_type': 'artist', 'scope': 'item', 'entity_id': 'a2'})
|
||||
assert r.status_code == 200
|
||||
body = r.get_json()
|
||||
assert body['success'] is True and body['reset'] == 1
|
||||
|
||||
|
||||
def test_route_retry_failed_bulk(client):
|
||||
r = client.post('/api/enrichment/spotify/retry',
|
||||
json={'entity_type': 'artist', 'scope': 'failed'})
|
||||
assert r.status_code == 200
|
||||
assert r.get_json()['reset'] == 1 # one not_found artist re-queued
|
||||
|
||||
|
||||
def test_route_retry_item_missing_id_400(client):
|
||||
r = client.post('/api/enrichment/spotify/retry',
|
||||
json={'entity_type': 'artist', 'scope': 'item'})
|
||||
assert r.status_code == 400
|
||||
138
tests/enrichment/test_worker_priority.py
Normal file
138
tests/enrichment/test_worker_priority.py
Normal file
|
|
@ -0,0 +1,138 @@
|
|||
"""Priority 'process this group first' helper for enrichment workers.
|
||||
|
||||
The shared helper returns one pending item of a chosen entity type in the
|
||||
shape the worker's dispatch already expects (with Spotify/iTunes mapped to
|
||||
their album_individual / track_individual types). Default path (no override)
|
||||
is exercised by the workers themselves and unchanged.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
|
||||
from core.worker_utils import (
|
||||
PRIORITY_ENTITIES,
|
||||
priority_pending_item,
|
||||
read_enrichment_priority,
|
||||
)
|
||||
from database.music_database import MusicDatabase
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def db(tmp_path):
|
||||
d = MusicDatabase(str(tmp_path / 'prio.db'))
|
||||
conn = d._get_connection()
|
||||
cur = conn.cursor()
|
||||
cur.execute("INSERT INTO artists (id, name) VALUES ('a1', 'Pending Artist')") # NULL status
|
||||
cur.execute("INSERT INTO artists (id, name, spotify_match_status) VALUES ('a2','Done','matched')")
|
||||
cur.execute("INSERT INTO albums (id, artist_id, title) VALUES ('al1','a2','Pending Album')") # NULL status
|
||||
cur.execute("INSERT INTO tracks (id, album_id, artist_id, title) VALUES ('t1','al1','a2','Pending Track')")
|
||||
conn.commit()
|
||||
conn.close()
|
||||
return d
|
||||
|
||||
|
||||
def _cur(db):
|
||||
return db._get_connection().cursor()
|
||||
|
||||
|
||||
def test_priority_artist_shape(db):
|
||||
item = priority_pending_item(_cur(db), 'spotify', 'artist')
|
||||
assert item == {'type': 'artist', 'id': 'a1', 'name': 'Pending Artist'}
|
||||
|
||||
|
||||
def test_priority_album_default_type(db):
|
||||
item = priority_pending_item(_cur(db), 'spotify', 'album')
|
||||
assert item['id'] == 'al1' and item['name'] == 'Pending Album' and item['artist'] == 'Done'
|
||||
assert item['type'] == 'album' # default type string
|
||||
|
||||
|
||||
def test_priority_album_type_override_for_spotify_itunes(db):
|
||||
item = priority_pending_item(_cur(db), 'spotify', 'album',
|
||||
{'album': 'album_individual', 'track': 'track_individual'})
|
||||
assert item['type'] == 'album_individual' # matches Spotify/iTunes dispatch
|
||||
|
||||
|
||||
def test_priority_track_shape(db):
|
||||
item = priority_pending_item(_cur(db), 'spotify', 'track')
|
||||
assert item['id'] == 't1' and item['type'] == 'track' and item['artist'] == 'Done'
|
||||
|
||||
|
||||
def test_priority_returns_none_when_group_exhausted(db):
|
||||
# No pending artists once a1 is matched -> None, so worker resumes its chain.
|
||||
conn = db._get_connection(); conn.execute("UPDATE artists SET spotify_match_status='matched' WHERE id='a1'"); conn.commit(); conn.close()
|
||||
assert priority_pending_item(_cur(db), 'spotify', 'artist') is None
|
||||
|
||||
|
||||
def test_priority_rejects_bad_entity_and_service(db):
|
||||
assert priority_pending_item(_cur(db), 'spotify', 'bogus') is None
|
||||
assert priority_pending_item(_cur(db), 'spot;drop', 'artist') is None # non-alpha service blocked
|
||||
|
||||
|
||||
def test_read_priority_unset_is_empty():
|
||||
# Unknown/unset key -> '' (no override). Uses the real config_manager.
|
||||
assert read_enrichment_priority('definitely_not_a_service') == ''
|
||||
|
||||
|
||||
def test_read_priority_roundtrip():
|
||||
from config.settings import config_manager
|
||||
key = 'spotify_enrichment_priority'
|
||||
old = config_manager.get(key, '')
|
||||
try:
|
||||
config_manager.set(key, 'album')
|
||||
assert read_enrichment_priority('spotify') == 'album'
|
||||
config_manager.set(key, 'bogus')
|
||||
assert read_enrichment_priority('spotify') == '' # invalid -> ignored
|
||||
finally:
|
||||
config_manager.set(key, old)
|
||||
|
||||
|
||||
def test_priority_entities_constant():
|
||||
assert PRIORITY_ENTITIES == ('artist', 'album', 'track')
|
||||
|
||||
|
||||
# --- priority GET/POST routes ---------------------------------------------
|
||||
|
||||
@pytest.fixture
|
||||
def client():
|
||||
from flask import Flask
|
||||
from core.enrichment import api as enrichment_api
|
||||
store = {}
|
||||
enrichment_api.configure(
|
||||
config_get=lambda k, d=None: store.get(k, d),
|
||||
config_set=lambda k, v: store.__setitem__(k, v),
|
||||
db_getter=lambda: None,
|
||||
)
|
||||
app = Flask(__name__)
|
||||
app.register_blueprint(enrichment_api.create_blueprint())
|
||||
with app.test_client() as c:
|
||||
c._store = store
|
||||
yield c
|
||||
enrichment_api.configure(config_get=None, config_set=None, db_getter=None)
|
||||
|
||||
|
||||
def test_route_priority_get_default_empty(client):
|
||||
r = client.get('/api/enrichment/spotify/priority')
|
||||
assert r.status_code == 200
|
||||
assert r.get_json()['priority'] == ''
|
||||
|
||||
|
||||
def test_route_priority_set_and_get(client):
|
||||
assert client.post('/api/enrichment/spotify/priority', json={'entity': 'album'}).status_code == 200
|
||||
assert client._store['spotify_enrichment_priority'] == 'album'
|
||||
assert client.get('/api/enrichment/spotify/priority').get_json()['priority'] == 'album'
|
||||
|
||||
|
||||
def test_route_priority_clear(client):
|
||||
client.post('/api/enrichment/spotify/priority', json={'entity': 'album'})
|
||||
client.post('/api/enrichment/spotify/priority', json={'entity': 'none'})
|
||||
assert client.get('/api/enrichment/spotify/priority').get_json()['priority'] == ''
|
||||
|
||||
|
||||
def test_route_priority_rejects_unsupported_entity(client):
|
||||
# Genius has no albums -> 400
|
||||
assert client.post('/api/enrichment/genius/priority', json={'entity': 'album'}).status_code == 400
|
||||
|
||||
|
||||
def test_route_priority_unknown_service_404(client):
|
||||
assert client.get('/api/enrichment/bogus/priority').status_code == 404
|
||||
|
|
@ -51,7 +51,8 @@ SPLIT_MODULES = [
|
|||
]
|
||||
|
||||
# Other JS files that exist in static/ but are NOT part of the split
|
||||
NON_SPLIT_JS = {"setup-wizard.js", "docs.js", "helper.js", "particles.js", "worker-orbs.js"}
|
||||
NON_SPLIT_JS = {"setup-wizard.js", "docs.js", "helper.js", "particles.js", "worker-orbs.js",
|
||||
"enrichment-manager.js"}
|
||||
|
||||
# Pre-existing duplicate helper functions that lived in the original monolith.
|
||||
# In a plain <script> context the last-loaded declaration wins. These are NOT
|
||||
|
|
@ -216,7 +217,7 @@ class TestOnclickCoverage:
|
|||
self.all_fns.update(_all_function_decls(text))
|
||||
|
||||
# Also include non-split JS files that are loaded
|
||||
for extra in ("setup-wizard.js", "docs.js", "helper.js"):
|
||||
for extra in ("setup-wizard.js", "docs.js", "helper.js", "enrichment-manager.js"):
|
||||
path = _STATIC / extra
|
||||
if path.exists():
|
||||
self.all_fns.update(_all_function_decls(_read(path)))
|
||||
|
|
|
|||
|
|
@ -34643,8 +34643,10 @@ _register_enrichment_services([
|
|||
|
||||
_configure_enrichment_api(
|
||||
config_set=lambda key, value: config_manager.set(key, value),
|
||||
config_get=lambda key, default=None: config_manager.get(key, default),
|
||||
auto_paused_discard=lambda token: _download_auto_paused.discard(token),
|
||||
yield_override_add=lambda token: _download_yield_override.add(token),
|
||||
db_getter=get_database,
|
||||
)
|
||||
|
||||
app.register_blueprint(_create_enrichment_blueprint())
|
||||
|
|
|
|||
|
|
@ -625,6 +625,13 @@
|
|||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<!-- Manage Enrichment Workers — opens the full management modal -->
|
||||
<button class="em-manage-btn" id="manage-enrichment-btn"
|
||||
title="Manage enrichment workers — stats, unmatched items, manual matching"
|
||||
onclick="openEnrichmentManager()">
|
||||
<span class="em-manage-btn-icon">🧬</span>
|
||||
<span class="em-manage-btn-label">Manage Workers</span>
|
||||
</button>
|
||||
</div>
|
||||
<!-- Watchlist / Wishlist quick-nav (top-right corner) -->
|
||||
<div class="header-quick-nav">
|
||||
|
|
@ -8032,6 +8039,7 @@
|
|||
<script src="{{ url_for('static', filename='discover-section-controller.js', v=static_v) }}"></script>
|
||||
<script src="{{ url_for('static', filename='discover.js', v=static_v) }}"></script>
|
||||
<script src="{{ url_for('static', filename='enrichment.js', v=static_v) }}"></script>
|
||||
<script src="{{ url_for('static', filename='enrichment-manager.js', v=static_v) }}"></script>
|
||||
<script src="{{ url_for('static', filename='stats-automations.js', v=static_v) }}"></script>
|
||||
<script src="{{ url_for('static', filename='auto-sync.js', v=static_v) }}"></script>
|
||||
<script src="{{ url_for('static', filename='pages-extra.js', v=static_v) }}"></script>
|
||||
|
|
|
|||
1160
webui/static/enrichment-manager.js
Normal file
1160
webui/static/enrichment-manager.js
Normal file
File diff suppressed because it is too large
Load diff
|
|
@ -64744,3 +64744,504 @@ body.reduce-effects .dash-card::after {
|
|||
background: rgba(var(--accent-rgb), 0.3);
|
||||
background-clip: padding-box;
|
||||
}
|
||||
|
||||
/* ===========================================================================
|
||||
Manage Enrichment Workers modal (enrichment-manager.js)
|
||||
=========================================================================== */
|
||||
.em-manage-btn {
|
||||
display: inline-flex;
|
||||
align-items: center;
|
||||
gap: 9px;
|
||||
height: 44px;
|
||||
padding: 0 18px 0 8px;
|
||||
margin-left: 10px;
|
||||
background: linear-gradient(135deg, rgba(var(--accent-rgb), 0.16) 0%, rgba(var(--accent-rgb), 0.07) 100%);
|
||||
backdrop-filter: blur(20px) saturate(1.4);
|
||||
-webkit-backdrop-filter: blur(20px) saturate(1.4);
|
||||
border: 1.5px solid rgba(var(--accent-rgb), 0.28);
|
||||
border-radius: 999px;
|
||||
color: #fff;
|
||||
font-size: 13.5px;
|
||||
font-weight: 700;
|
||||
letter-spacing: 0.2px;
|
||||
cursor: pointer;
|
||||
transition: all 0.3s cubic-bezier(0.4, 0, 0.2, 1);
|
||||
white-space: nowrap;
|
||||
box-shadow:
|
||||
0 4px 16px rgba(var(--accent-rgb), 0.18),
|
||||
0 2px 8px rgba(0,0,0,0.15),
|
||||
inset 0 1px 0 rgba(255,255,255,0.08);
|
||||
}
|
||||
.em-manage-btn:hover {
|
||||
border-color: rgba(var(--accent-rgb), 0.5);
|
||||
transform: scale(1.04);
|
||||
box-shadow:
|
||||
0 6px 22px rgba(var(--accent-rgb), 0.32),
|
||||
0 3px 12px rgba(0,0,0,0.2),
|
||||
inset 0 1px 0 rgba(255,255,255,0.12);
|
||||
}
|
||||
.em-manage-btn-icon {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
width: 30px;
|
||||
height: 30px;
|
||||
border-radius: 50%;
|
||||
font-size: 16px;
|
||||
background: linear-gradient(135deg, rgba(var(--accent-rgb), 0.9), rgba(var(--accent-rgb), 0.55));
|
||||
box-shadow: 0 0 12px rgba(var(--accent-rgb), 0.5);
|
||||
}
|
||||
.em-manage-btn-label { background: linear-gradient(90deg, #fff, rgba(255,255,255,0.85)); -webkit-background-clip: text; background-clip: text; }
|
||||
|
||||
.enrichment-manager-modal {
|
||||
position: relative;
|
||||
background:
|
||||
radial-gradient(120% 80% at 0% 0%, rgba(var(--accent-rgb), 0.10), transparent 55%),
|
||||
radial-gradient(100% 70% at 100% 0%, rgba(255,255,255,0.04), transparent 50%),
|
||||
linear-gradient(150deg, #1c1c1f 0%, #131316 55%, #0f0f12 100%);
|
||||
border-radius: 18px;
|
||||
border: 1px solid rgba(255,255,255,0.09);
|
||||
width: 1150px;
|
||||
max-width: 95vw;
|
||||
height: 82vh;
|
||||
max-height: 860px;
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
box-shadow:
|
||||
0 30px 90px rgba(0,0,0,0.65),
|
||||
0 0 0 1px rgba(var(--accent-rgb), 0.12),
|
||||
inset 0 1px 0 rgba(255,255,255,0.06);
|
||||
overflow: hidden;
|
||||
}
|
||||
/* Hairline top accent line across the whole modal. */
|
||||
.enrichment-manager-modal::before {
|
||||
content: ''; position: absolute; top: 0; left: 0; right: 0; height: 1px;
|
||||
background: linear-gradient(90deg, transparent, rgba(var(--accent-rgb), 0.6), transparent);
|
||||
z-index: 2;
|
||||
}
|
||||
/* Top bar (rebuilt header) */
|
||||
.em-topbar {
|
||||
flex: 0 0 auto; display: flex; align-items: center; gap: 16px; flex-wrap: wrap;
|
||||
padding: 15px 18px; position: relative;
|
||||
background: linear-gradient(120deg, rgba(var(--accent-rgb), 0.22), rgba(var(--accent-rgb), 0.05) 45%, transparent 78%);
|
||||
border-bottom: 1px solid rgba(255,255,255,0.08);
|
||||
}
|
||||
.em-topbar-icon {
|
||||
flex: 0 0 auto; width: 46px; height: 46px; border-radius: 13px;
|
||||
display: flex; align-items: center; justify-content: center; font-size: 22px;
|
||||
background: linear-gradient(135deg, rgba(var(--accent-rgb), 0.95), rgba(var(--accent-rgb), 0.5));
|
||||
box-shadow: 0 6px 20px rgba(var(--accent-rgb), 0.45), inset 0 1px 0 rgba(255,255,255,0.25);
|
||||
}
|
||||
.em-topbar-icon span { filter: drop-shadow(0 1px 2px rgba(0,0,0,0.3)); }
|
||||
.em-topbar-titles { flex: 1 1 auto; min-width: 0; }
|
||||
.em-topbar-title {
|
||||
margin: 0; font-size: 19px; font-weight: 800; letter-spacing: 0.2px;
|
||||
background: linear-gradient(90deg, #fff, rgba(255,255,255,0.72));
|
||||
-webkit-background-clip: text; background-clip: text; color: transparent;
|
||||
}
|
||||
.em-topbar-sub { font-size: 12px; color: rgba(255,255,255,0.5); margin-top: 1px; }
|
||||
|
||||
/* Global 'process first everywhere' control */
|
||||
.em-global { display: flex; align-items: center; gap: 11px; flex: 0 0 auto;
|
||||
padding: 6px 6px 6px 12px; border-radius: 13px;
|
||||
background: rgba(0,0,0,0.25); border: 1px solid rgba(255,255,255,0.08); }
|
||||
.em-global-label { font-size: 9px; font-weight: 800; text-transform: uppercase; letter-spacing: 0.6px;
|
||||
color: rgba(255,255,255,0.4); line-height: 1.25; text-align: right; }
|
||||
.em-global-label span { color: rgba(255,255,255,0.7); }
|
||||
.em-global-tabs { display: inline-flex; gap: 3px; padding: 3px; background: rgba(255,255,255,0.05);
|
||||
border-radius: 9px; border: 1px solid rgba(255,255,255,0.07); }
|
||||
.em-global-tabs button { padding: 6px 12px; border: none; background: transparent; border-radius: 7px;
|
||||
color: rgba(255,255,255,0.6); font-size: 12px; font-weight: 700; cursor: pointer; transition: all 0.15s ease; }
|
||||
.em-global-tabs button:hover { color: #fff; background: rgba(255,255,255,0.06); }
|
||||
.em-global-tabs button.active { background: rgb(var(--accent-rgb)); color: #fff; box-shadow: 0 2px 10px rgba(var(--accent-rgb), 0.5); }
|
||||
|
||||
.em-topbar-actions { display: flex; gap: 8px; flex: 0 0 auto; }
|
||||
.em-icon-btn--close { font-size: 20px; }
|
||||
|
||||
.em-body { display: flex; flex: 1 1 auto; min-height: 0; }
|
||||
|
||||
/* Left rail */
|
||||
.em-rail {
|
||||
flex: 0 0 230px;
|
||||
border-right: 1px solid rgba(255,255,255,0.07);
|
||||
padding: 12px;
|
||||
overflow-y: auto;
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: 4px;
|
||||
}
|
||||
.em-worker-row {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 10px;
|
||||
padding: 9px 10px;
|
||||
background: transparent;
|
||||
border: 1px solid transparent;
|
||||
border-radius: 10px;
|
||||
cursor: pointer;
|
||||
text-align: left;
|
||||
transition: background 0.15s ease, border-color 0.15s ease;
|
||||
}
|
||||
.em-worker-row { position: relative; }
|
||||
.em-worker-row:hover { background: rgba(255,255,255,0.05); }
|
||||
.em-worker-row.active {
|
||||
background: rgba(var(--accent-rgb), 0.14);
|
||||
border-color: rgba(var(--accent-rgb), 0.4);
|
||||
}
|
||||
.em-worker-row.active::before {
|
||||
content: '';
|
||||
position: absolute; left: -1px; top: 8px; bottom: 8px; width: 3px;
|
||||
border-radius: 0 3px 3px 0;
|
||||
background: rgb(var(--accent-rgb));
|
||||
box-shadow: 0 0 10px rgba(var(--accent-rgb), 0.7);
|
||||
}
|
||||
/* Circular glowing logo chip — mirrors the dashboard enrichment bubbles. */
|
||||
.em-icon {
|
||||
flex: 0 0 auto;
|
||||
width: 34px; height: 34px;
|
||||
border-radius: 50%;
|
||||
display: flex; align-items: center; justify-content: center;
|
||||
background: linear-gradient(135deg, rgba(255,255,255,0.07), rgba(255,255,255,0.02));
|
||||
border: 1.5px solid color-mix(in srgb, var(--em-accent, #888) 45%, transparent);
|
||||
box-shadow: 0 0 12px color-mix(in srgb, var(--em-accent, #888) 30%, transparent),
|
||||
inset 0 1px 0 rgba(255,255,255,0.08);
|
||||
overflow: hidden;
|
||||
transition: transform 0.25s cubic-bezier(0.4, 0, 0.2, 1);
|
||||
}
|
||||
.em-icon--lg { width: 52px; height: 52px; border-width: 2px; }
|
||||
.em-icon-img { width: 66%; height: 66%; object-fit: contain; display: block; }
|
||||
.em-icon-letter { font-weight: 800; font-size: 15px; color: var(--em-accent, #fff); text-shadow: 0 1px 2px rgba(0,0,0,0.5); }
|
||||
.em-icon--lg .em-icon-letter { font-size: 22px; }
|
||||
.em-worker-row:hover .em-icon { transform: scale(1.08); }
|
||||
.em-worker-meta { flex: 1 1 auto; min-width: 0; display: flex; flex-direction: column; gap: 2px; }
|
||||
.em-worker-name { color: #eee; font-size: 13px; font-weight: 600; white-space: nowrap; overflow: hidden; text-overflow: ellipsis; }
|
||||
.em-worker-sub { font-size: 10.5px; color: rgba(255,255,255,0.4); white-space: nowrap; overflow: hidden; text-overflow: ellipsis; }
|
||||
.em-rail-cov { display: block; height: 3px; border-radius: 999px; background: rgba(255,255,255,0.1); overflow: hidden; margin-top: 2px; }
|
||||
.em-rail-cov-fill { display: block; height: 100%; border-radius: 999px; background: color-mix(in srgb, var(--accent, #4ade80) 70%, transparent); background: rgb(var(--accent-rgb)); transition: width 0.5s cubic-bezier(0.4,0,0.2,1); }
|
||||
.em-dot { flex: 0 0 auto; width: 9px; height: 9px; border-radius: 50%; background: #555; }
|
||||
.em-dot--running { background: #1db954; box-shadow: 0 0 8px #1db954; }
|
||||
.em-dot--idle { background: #4a90d9; }
|
||||
.em-dot--paused { background: #e0a93b; }
|
||||
.em-dot--ratelimited { background: #e05b5b; box-shadow: 0 0 8px #e05b5b; }
|
||||
.em-dot--disabled, .em-dot--stopped { background: #555; }
|
||||
|
||||
/* Right panel */
|
||||
.em-panel {
|
||||
flex: 1 1 auto;
|
||||
min-width: 0;
|
||||
padding: 18px 22px;
|
||||
overflow-y: auto;
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: 16px;
|
||||
}
|
||||
.em-panel-header { flex: 0 0 auto; }
|
||||
.em-ph-top { display: flex; align-items: center; gap: 14px; }
|
||||
.em-ph-titles { flex: 1 1 auto; min-width: 0; }
|
||||
.em-ph-name { font-size: 19px; font-weight: 800; color: #fff; }
|
||||
.em-ph-sub { font-size: 13px; color: rgba(255,255,255,0.7); margin-top: 2px; }
|
||||
.em-ph-actions { display: flex; align-items: center; gap: 8px; flex-wrap: wrap; justify-content: flex-end; }
|
||||
.em-muted { color: rgba(255,255,255,0.45); }
|
||||
|
||||
.em-pill {
|
||||
padding: 4px 11px; border-radius: 999px; font-size: 11px; font-weight: 700;
|
||||
text-transform: uppercase; letter-spacing: 0.4px;
|
||||
}
|
||||
.em-pill--running { background: rgba(29,185,84,0.18); color: #4ade80; }
|
||||
.em-pill--idle { background: rgba(74,144,217,0.18); color: #7fb5ec; }
|
||||
.em-pill--paused { background: rgba(224,169,59,0.18); color: #f0c060; }
|
||||
.em-pill--ratelimited { background: rgba(224,91,91,0.2); color: #ff8b8b; }
|
||||
.em-pill--disabled, .em-pill--stopped { background: rgba(255,255,255,0.08); color: rgba(255,255,255,0.5); }
|
||||
|
||||
.em-chip {
|
||||
padding: 4px 9px; border-radius: 8px; font-size: 11px; font-weight: 600;
|
||||
background: rgba(255,255,255,0.08); color: rgba(255,255,255,0.75);
|
||||
}
|
||||
.em-chip--err { background: rgba(224,91,91,0.18); color: #ff8b8b; }
|
||||
.em-chip--nf { background: rgba(224,91,91,0.15); color: #ff9b9b; }
|
||||
.em-chip--pend { background: rgba(224,169,59,0.15); color: #f0c060; }
|
||||
|
||||
.em-btn {
|
||||
padding: 8px 14px; border-radius: 9px; border: 1px solid rgba(var(--accent-rgb), 0.4);
|
||||
background: rgba(var(--accent-rgb), 0.15); color: #fff; font-size: 13px; font-weight: 600;
|
||||
cursor: pointer; transition: all 0.15s ease;
|
||||
}
|
||||
.em-btn:hover:not(:disabled) { background: rgba(var(--accent-rgb), 0.28); }
|
||||
.em-btn:disabled { opacity: 0.4; cursor: not-allowed; }
|
||||
.em-btn--go { background: rgba(29,185,84,0.2); border-color: rgba(29,185,84,0.5); }
|
||||
.em-btn--sm { padding: 5px 10px; font-size: 12px; }
|
||||
.em-btn--ghost { background: transparent; border-color: rgba(255,255,255,0.18); color: rgba(255,255,255,0.7); }
|
||||
.em-btn--ghost:hover:not(:disabled) { background: rgba(255,255,255,0.08); }
|
||||
|
||||
/* Stat cards */
|
||||
.em-stats { display: grid; grid-template-columns: repeat(auto-fit, minmax(190px, 1fr)); gap: 12px; flex: 0 0 auto; }
|
||||
.em-stat-card {
|
||||
background: linear-gradient(160deg, rgba(255,255,255,0.06), rgba(255,255,255,0.02));
|
||||
border: 1px solid rgba(255,255,255,0.08);
|
||||
border-radius: 12px; padding: 14px;
|
||||
transition: transform 0.2s ease, border-color 0.2s ease;
|
||||
}
|
||||
.em-stat-card:hover { transform: translateY(-2px); border-color: rgba(var(--accent-rgb), 0.3); }
|
||||
.em-bar-fill { box-shadow: 0 0 10px rgba(var(--accent-rgb), 0.5); }
|
||||
.em-stat-head { display: flex; justify-content: space-between; align-items: baseline; margin-bottom: 8px; }
|
||||
.em-stat-title { font-size: 12px; font-weight: 700; text-transform: uppercase; letter-spacing: 0.5px; color: rgba(255,255,255,0.6); }
|
||||
.em-stat-pct { font-size: 20px; font-weight: 800; color: #fff; }
|
||||
.em-bar { height: 7px; border-radius: 999px; background: rgba(255,255,255,0.1); overflow: hidden; }
|
||||
.em-bar-fill { height: 100%; border-radius: 999px; background: linear-gradient(90deg, rgba(var(--accent-rgb),0.7), rgba(var(--accent-rgb),1)); transition: width 0.4s ease; }
|
||||
.em-stat-legend { display: flex; gap: 10px; margin-top: 9px; font-size: 11px; flex-wrap: wrap; }
|
||||
.em-leg--matched { color: #4ade80; }
|
||||
.em-leg--nf { color: #ff9b9b; }
|
||||
.em-leg--pend { color: #f0c060; }
|
||||
|
||||
/* Unmatched browser */
|
||||
.em-unmatched { flex: 1 1 auto; display: flex; flex-direction: column; min-height: 0; gap: 10px; }
|
||||
.em-unmatched-controls { display: flex; justify-content: space-between; align-items: center; gap: 12px; flex-wrap: wrap; flex: 0 0 auto; }
|
||||
.em-tabs { display: flex; gap: 6px; }
|
||||
.em-tab {
|
||||
padding: 7px 14px; border-radius: 8px; border: 1px solid rgba(255,255,255,0.1);
|
||||
background: transparent; color: rgba(255,255,255,0.65); font-size: 13px; font-weight: 600; cursor: pointer;
|
||||
}
|
||||
.em-tab.active { background: rgba(var(--accent-rgb), 0.16); border-color: rgba(var(--accent-rgb), 0.4); color: #fff; }
|
||||
.em-filter-row { display: flex; gap: 8px; align-items: center; }
|
||||
.em-select, .em-search {
|
||||
padding: 8px 12px; background: rgba(255,255,255,0.06); border: 1px solid rgba(255,255,255,0.12);
|
||||
border-radius: 8px; color: #fff; font-size: 13px;
|
||||
}
|
||||
.em-search { min-width: 200px; }
|
||||
.em-search:focus, .em-select:focus { outline: none; border-color: rgba(var(--accent-rgb), 0.6); }
|
||||
|
||||
.em-unmatched-list { flex: 1 1 auto; overflow-y: auto; display: flex; flex-direction: column; gap: 6px; padding-right: 4px; }
|
||||
.em-row {
|
||||
display: flex; align-items: center; gap: 13px; padding: 10px 14px 10px 16px;
|
||||
background: linear-gradient(100deg, rgba(255,255,255,0.045), rgba(255,255,255,0.02));
|
||||
border: 1px solid rgba(255,255,255,0.07); border-radius: 12px;
|
||||
position: relative;
|
||||
transition: transform 0.15s ease, border-color 0.15s ease, background 0.15s ease;
|
||||
}
|
||||
/* status stripe down the left edge */
|
||||
.em-row::before {
|
||||
content: ''; position: absolute; left: 0; top: 0; bottom: 0; width: 3px;
|
||||
background: rgba(255,255,255,0.15);
|
||||
}
|
||||
.em-row--nf::before { background: #e0586b; }
|
||||
.em-row--pend::before { background: #f0c060; }
|
||||
.em-row:hover {
|
||||
transform: translateX(2px);
|
||||
background: linear-gradient(100deg, rgba(255,255,255,0.07), rgba(255,255,255,0.03));
|
||||
border-color: rgba(var(--em-accent-rgb, 255,255,255), 0.22);
|
||||
}
|
||||
.em-row-img { width: 46px; height: 46px; border-radius: 10px; object-fit: cover; flex: 0 0 auto; }
|
||||
.em-row-img--ph {
|
||||
position: relative; display: flex; align-items: center; justify-content: center;
|
||||
background: rgba(255,255,255,0.045); border: 1px solid rgba(255,255,255,0.06);
|
||||
color: rgba(255,255,255,0.28); font-size: 18px;
|
||||
}
|
||||
.em-row-img-pic { position: absolute; inset: 0; width: 100%; height: 100%; object-fit: cover; border-radius: inherit; }
|
||||
.em-row-info { flex: 1 1 auto; min-width: 0; }
|
||||
.em-row-name { font-size: 14.5px; font-weight: 600; color: #fff; white-space: nowrap; overflow: hidden; text-overflow: ellipsis; }
|
||||
.em-row-meta { display: flex; gap: 8px; align-items: center; margin-top: 4px; font-size: 11px; }
|
||||
.em-row-actions { display: flex; gap: 6px; flex: 0 0 auto; opacity: 0.85; transition: opacity 0.15s ease; }
|
||||
.em-row:hover .em-row-actions { opacity: 1; }
|
||||
.em-empty { text-align: center; padding: 48px 20px; color: rgba(255,255,255,0.55); font-size: 15px; }
|
||||
.em-empty-emoji { font-size: 38px; margin-bottom: 10px; opacity: 0.85; }
|
||||
.em-pager { display: flex; align-items: center; justify-content: center; gap: 14px; flex: 0 0 auto; padding-top: 4px; font-size: 12px; }
|
||||
|
||||
/* --- Motion: entrance / exit + scroll lock --- */
|
||||
body.em-scroll-lock { overflow: hidden; }
|
||||
.em-overlay { transition: opacity 0.22s ease, backdrop-filter 0.22s ease; }
|
||||
.em-overlay.em-closing { opacity: 0; }
|
||||
@keyframes em-pop-in {
|
||||
from { opacity: 0; transform: translateY(14px) scale(0.97); }
|
||||
to { opacity: 1; transform: translateY(0) scale(1); }
|
||||
}
|
||||
.em-in { animation: em-pop-in 0.28s cubic-bezier(0.16, 1, 0.3, 1) both; }
|
||||
.em-overlay.em-closing .enrichment-manager-modal { transform: scale(0.98); opacity: 0; transition: all 0.16s ease; }
|
||||
.enrichment-manager-modal:focus { outline: none; }
|
||||
|
||||
/* --- Header actions / refresh --- */
|
||||
.em-header-actions { display: flex; align-items: center; gap: 8px; }
|
||||
.em-icon-btn {
|
||||
width: 32px; height: 32px; border-radius: 50%;
|
||||
background: rgba(255,255,255,0.08); color: rgba(255,255,255,0.8);
|
||||
border: none; cursor: pointer; font-size: 16px; line-height: 1;
|
||||
display: flex; align-items: center; justify-content: center;
|
||||
transition: background 0.2s ease, transform 0.2s ease;
|
||||
}
|
||||
.em-icon-btn:hover { background: rgba(255,255,255,0.16); transform: rotate(15deg); }
|
||||
.em-icon-btn.em-spinning { animation: em-spin 0.6s linear; }
|
||||
@keyframes em-spin { to { transform: rotate(360deg); } }
|
||||
|
||||
.em-stat-pct-sym { font-size: 13px; opacity: 0.6; margin-left: 1px; }
|
||||
|
||||
/* --- Skeleton loaders --- */
|
||||
.em-skel {
|
||||
position: relative; overflow: hidden;
|
||||
background: rgba(255,255,255,0.06); border-radius: 6px;
|
||||
}
|
||||
.em-skel::after {
|
||||
content: ''; position: absolute; inset: 0;
|
||||
background: linear-gradient(90deg, transparent, rgba(255,255,255,0.10), transparent);
|
||||
transform: translateX(-100%); animation: em-shimmer 1.3s infinite;
|
||||
}
|
||||
@keyframes em-shimmer { 100% { transform: translateX(100%); } }
|
||||
.em-skel-line { height: 11px; }
|
||||
.em-skel-bar { height: 7px; margin: 12px 0; border-radius: 999px; }
|
||||
.em-skel-card { display: flex; flex-direction: column; }
|
||||
.em-skel-row .em-row-img { background: rgba(255,255,255,0.06); }
|
||||
.em-skel-row { pointer-events: none; }
|
||||
|
||||
@media (prefers-reduced-motion: reduce) {
|
||||
.enrichment-manager-modal.em-in,
|
||||
.em-icon-btn.em-spinning,
|
||||
.em-skel::after { animation: none; }
|
||||
.em-bar-fill, .em-rail-cov-fill { transition: none; }
|
||||
}
|
||||
|
||||
/* --- Narrow screens: rail becomes a horizontal strip --- */
|
||||
@media (max-width: 760px) {
|
||||
.enrichment-manager-modal { height: 90vh; }
|
||||
.em-body { flex-direction: column; }
|
||||
.em-rail { flex: 0 0 auto; flex-direction: row; overflow-x: auto; border-right: none; border-bottom: 1px solid rgba(255,255,255,0.07); }
|
||||
.em-worker-row { flex: 0 0 auto; }
|
||||
.em-worker-meta { display: none; }
|
||||
}
|
||||
|
||||
/* ===== Panel polish: hero header, accent theming, segmented stats ===== */
|
||||
/* The panel sets --em-accent / --em-accent-rgb to the selected worker colour. */
|
||||
.em-section-label {
|
||||
font-size: 11px; font-weight: 700; text-transform: uppercase; letter-spacing: 0.8px;
|
||||
color: rgba(255,255,255,0.4); flex: 0 0 auto;
|
||||
}
|
||||
.em-section-label--inline { display: flex; align-items: center; gap: 8px; }
|
||||
.em-count {
|
||||
display: inline-flex; align-items: center; justify-content: center;
|
||||
min-width: 22px; height: 19px; padding: 0 7px; border-radius: 999px;
|
||||
background: rgba(var(--em-accent-rgb, 99,102,241), 0.2);
|
||||
color: rgb(var(--em-accent-rgb, 129,140,248));
|
||||
font-size: 11px; font-weight: 800; letter-spacing: 0;
|
||||
}
|
||||
|
||||
/* Hero header */
|
||||
.em-hero {
|
||||
position: relative; overflow: hidden;
|
||||
display: flex; align-items: center; gap: 16px;
|
||||
padding: 18px 20px;
|
||||
border-radius: 16px;
|
||||
background:
|
||||
linear-gradient(135deg, rgba(var(--em-accent-rgb, 99,102,241), 0.16), rgba(var(--em-accent-rgb, 99,102,241), 0.03) 60%),
|
||||
rgba(255,255,255,0.03);
|
||||
border: 1px solid rgba(var(--em-accent-rgb, 99,102,241), 0.22);
|
||||
}
|
||||
.em-hero-glow {
|
||||
position: absolute; top: -60%; right: -10%; width: 300px; height: 300px;
|
||||
background: radial-gradient(circle, rgba(var(--em-accent-rgb, 99,102,241), 0.22), transparent 70%);
|
||||
pointer-events: none;
|
||||
}
|
||||
/* Gentle 'alive' pulse while the worker is actively running. */
|
||||
.em-hero--live .em-hero-glow { animation: em-glow-pulse 3.2s ease-in-out infinite; }
|
||||
@keyframes em-glow-pulse { 0%,100% { opacity: 0.55; } 50% { opacity: 1; } }
|
||||
@media (prefers-reduced-motion: reduce) { .em-hero--live .em-hero-glow { animation: none; } }
|
||||
.em-hero .em-icon { width: 56px; height: 56px; }
|
||||
.em-hero .em-ph-titles { flex: 1 1 auto; min-width: 0; z-index: 1; }
|
||||
.em-ph-nameline { display: flex; align-items: center; gap: 11px; flex-wrap: wrap; }
|
||||
.em-ph-name-sub { font-size: 14px; font-weight: 500; color: rgba(255,255,255,0.45); }
|
||||
.em-hero .em-ph-actions { z-index: 1; }
|
||||
|
||||
/* Combined coverage + processing-order cards (click to enrich a group first) */
|
||||
.em-section-sub { font-weight: 500; text-transform: none; letter-spacing: 0; color: rgba(255,255,255,0.35); font-size: 11px; }
|
||||
.em-cards { display: grid; grid-template-columns: repeat(auto-fit, minmax(215px, 1fr)); gap: 12px; flex: 0 0 auto; }
|
||||
.em-card {
|
||||
display: flex; flex-direction: column; gap: 10px; text-align: left; cursor: pointer;
|
||||
padding: 14px; border-radius: 14px; position: relative; overflow: hidden;
|
||||
background: linear-gradient(160deg, rgba(255,255,255,0.06), rgba(255,255,255,0.02));
|
||||
border: 1.5px solid rgba(255,255,255,0.08);
|
||||
transition: transform 0.2s ease, border-color 0.2s ease, background 0.2s ease;
|
||||
}
|
||||
.em-card:hover {
|
||||
transform: translateY(-2px);
|
||||
border-color: rgba(var(--em-accent-rgb, 99,102,241), 0.45);
|
||||
background: linear-gradient(160deg, rgba(255,255,255,0.09), rgba(255,255,255,0.03));
|
||||
}
|
||||
.em-card-top { display: flex; align-items: center; gap: 8px; }
|
||||
.em-card-glyph { font-size: 15px; }
|
||||
.em-card-title { font-size: 12.5px; font-weight: 800; color: #fff; text-transform: uppercase; letter-spacing: 0.5px; }
|
||||
.em-card-pct { margin-left: auto; font-size: 21px; font-weight: 800; color: #fff; line-height: 1; }
|
||||
.em-card-badge { font-size: 9.5px; font-weight: 800; text-transform: uppercase; letter-spacing: 0.5px; padding: 2px 7px; border-radius: 999px; background: rgba(255,255,255,0.1); color: rgba(255,255,255,0.55); white-space: nowrap; }
|
||||
.em-card-badge--pin { background: rgba(var(--em-accent-rgb, 99,102,241), 0.28); color: #fff; }
|
||||
.em-card-badge--now { background: rgba(74,222,128,0.2); color: #4ade80; }
|
||||
.em-card-badge--done { background: rgba(74,222,128,0.14); color: rgba(74,222,128,0.85); }
|
||||
.em-card .em-stat-legend { gap: 12px; margin-top: 0; }
|
||||
.em-card .em-leg { font-size: 11.5px; color: rgba(255,255,255,0.7); }
|
||||
/* States */
|
||||
.em-card--pinned {
|
||||
border-color: rgba(var(--em-accent-rgb, 99,102,241), 0.65);
|
||||
background: linear-gradient(160deg, rgba(var(--em-accent-rgb, 99,102,241), 0.16), rgba(255,255,255,0.02));
|
||||
box-shadow: 0 0 0 1px rgba(var(--em-accent-rgb, 99,102,241), 0.25), 0 8px 24px rgba(var(--em-accent-rgb, 99,102,241), 0.15);
|
||||
}
|
||||
.em-card--current { border-color: rgba(74,222,128,0.4); }
|
||||
.em-card--current .em-card-badge--now { animation: em-glow-pulse 1.8s ease-in-out infinite; }
|
||||
.em-card--done { opacity: 0.74; }
|
||||
@media (prefers-reduced-motion: reduce) { .em-card--current .em-card-badge--now { animation: none; } }
|
||||
|
||||
/* Section label as a row (coverage % relocated here from the hero) */
|
||||
.em-section-label--row { display: flex; align-items: baseline; justify-content: space-between; gap: 12px; }
|
||||
.em-coverage-overall { font-size: 12.5px; font-weight: 500; letter-spacing: 0; text-transform: none; color: rgba(255,255,255,0.55); }
|
||||
.em-coverage-overall strong { color: rgb(var(--em-accent-rgb, 129,140,248)); font-weight: 800; }
|
||||
/* Accent-themed buttons / active states inside the panel */
|
||||
.em-panel .em-btn { border-color: rgba(var(--em-accent-rgb, 99,102,241), 0.45); background: rgba(var(--em-accent-rgb, 99,102,241), 0.16); }
|
||||
.em-panel .em-btn:hover:not(:disabled) { background: rgba(var(--em-accent-rgb, 99,102,241), 0.3); }
|
||||
|
||||
/* Entity glyph in stat-card titles */
|
||||
.em-stat-title { display: inline-flex; align-items: center; gap: 7px; }
|
||||
.em-stat-ico { font-size: 14px; filter: saturate(0.9); }
|
||||
|
||||
/* Segmented matched/not-found/pending bar */
|
||||
.em-seg { display: flex; height: 9px; border-radius: 999px; overflow: hidden; background: rgba(255,255,255,0.07); }
|
||||
.em-seg-fill { height: 100%; transition: width 0.6s cubic-bezier(0.16,1,0.3,1); }
|
||||
.em-seg--matched { background: linear-gradient(90deg, rgba(var(--em-accent-rgb, 74,222,128),0.85), rgb(var(--em-accent-rgb, 74,222,128))); }
|
||||
.em-seg--nf { background: #e0586b; }
|
||||
.em-seg--pend { background: rgba(240,192,96,0.85); }
|
||||
.em-stat-legend .em-leg { display: inline-flex; align-items: center; gap: 5px; }
|
||||
.em-stat-legend .em-leg i { width: 8px; height: 8px; border-radius: 50%; display: inline-block; }
|
||||
.em-leg--matched i { background: rgb(var(--em-accent-rgb, 74,222,128)); }
|
||||
.em-leg--nf i { background: #e0586b; }
|
||||
.em-leg--pend i { background: rgba(240,192,96,0.9); }
|
||||
|
||||
/* Unmatched toolbar */
|
||||
.em-unmatched-bar { display: flex; align-items: center; justify-content: space-between; gap: 14px; flex-wrap: wrap; flex: 0 0 auto; }
|
||||
.em-seg-tabs { display: inline-flex; padding: 3px; gap: 2px; background: rgba(255,255,255,0.05); border-radius: 10px; border: 1px solid rgba(255,255,255,0.07); }
|
||||
.em-seg-tab {
|
||||
padding: 6px 14px; border-radius: 8px; border: none; background: transparent;
|
||||
color: rgba(255,255,255,0.6); font-size: 12.5px; font-weight: 600; cursor: pointer;
|
||||
transition: all 0.18s ease;
|
||||
}
|
||||
.em-seg-tab:hover { color: #fff; }
|
||||
.em-seg-tab.active {
|
||||
background: rgba(var(--em-accent-rgb, 99,102,241), 0.9);
|
||||
color: #fff; box-shadow: 0 2px 8px rgba(var(--em-accent-rgb, 99,102,241), 0.4);
|
||||
}
|
||||
.em-search-wrap { position: relative; display: inline-flex; align-items: center; }
|
||||
.em-search-ico { position: absolute; left: 11px; color: rgba(255,255,255,0.4); font-size: 15px; pointer-events: none; }
|
||||
.em-search-wrap .em-search { padding-left: 30px; }
|
||||
.em-search:focus, .em-select:focus { outline: none; border-color: rgba(var(--em-accent-rgb, 99,102,241), 0.6); }
|
||||
.em-row:hover { background: rgba(var(--em-accent-rgb, 255,255,255), 0.07); border-color: rgba(var(--em-accent-rgb, 255,255,255), 0.14); }
|
||||
|
||||
/* Disabled / rate-limit banner */
|
||||
.em-banner { flex: 0 0 auto; padding: 10px 14px; border-radius: 10px; font-size: 12.5px; line-height: 1.45;
|
||||
background: rgba(255,255,255,0.05); border: 1px solid rgba(255,255,255,0.1); color: rgba(255,255,255,0.75); }
|
||||
.em-banner--warn { background: rgba(240,176,80,0.12); border-color: rgba(240,176,80,0.35); color: #f3cd86; }
|
||||
|
||||
/* Row checkbox + parent context (#3 / #4) */
|
||||
.em-row-check { width: 16px; height: 16px; flex: 0 0 auto; cursor: pointer; accent-color: rgb(var(--em-accent-rgb, 99,102,241)); }
|
||||
.em-row-parent { font-size: 12px; font-weight: 500; color: rgba(255,255,255,0.4); }
|
||||
|
||||
/* Bulk action bar */
|
||||
.em-bulk-bar { display: flex; align-items: center; gap: 10px; flex: 0 0 auto; padding: 8px 12px;
|
||||
background: rgba(var(--em-accent-rgb, 99,102,241), 0.12); border: 1px solid rgba(var(--em-accent-rgb, 99,102,241), 0.3);
|
||||
border-radius: 10px; }
|
||||
.em-bulk-count { font-size: 13px; font-weight: 700; color: #fff; margin-right: auto; }
|
||||
|
||||
/* Bulk retry-all + helper hint */
|
||||
.em-retry-all { margin-left: 4px; }
|
||||
.em-hint { font-size: 11px; color: rgba(255,255,255,0.38); flex: 0 0 auto; margin-top: -2px; }
|
||||
.em-hint :is(b, strong) { color: rgba(255,255,255,0.6); }
|
||||
|
|
|
|||
Loading…
Reference in a new issue