danvics/soulsync
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 1

Merge pull request #475 from Nezreka/feat/auto-import-live-progress

Browse source 
Feat/auto import live progress
This commit is contained in:
BoulderBadgeDad 2026-05-02 23:23:46 -07:00 committed by GitHub
commit bcb91a1a1a
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
5 changed files with 671 additions and 66 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

303
core/auto_import_worker.py
View file

@ -36,6 +36,11 @@ class FolderCandidate:
disc_structure: Dict[int, List[str]] = field(default_factory=dict) # disc_num -> files
folder_hash: str = ''
is_single: bool = False # True for loose files in staging root
# True when the candidate "folder" is the staging root itself (user dropped
# disc folders directly into staging without an album wrapper). The name is
# meaningless ("Staging", "Music", etc.) — folder-name identification must
# be skipped or it will false-match against random albums.
is_staging_root: bool = False
def _compute_folder_hash(audio_files: List[str]) -> str:
@ -58,7 +63,11 @@ def _read_file_tags(file_path: str) -> Dict[str, Any]:
if audio and audio.tags:
tags = audio.tags
result['title'] = (tags.get('title', [''])[0] or '').strip()
result['artist'] = (tags.get('artist', [''])[0] or tags.get('albumartist', [''])[0] or '').strip()
# Prefer albumartist for album-level identification (per-track artist
# often includes features like "Kendrick Lamar, Drake" which fragment
# consensus when grouping tracks into an album). Fall back to artist
# for files that lack albumartist.
result['artist'] = (tags.get('albumartist', [''])[0] or tags.get('artist', [''])[0] or '').strip()
result['album'] = (tags.get('album', [''])[0] or '').strip()
# Date/year — try 'date' first, fall back to 'year'
date_str = (tags.get('date', [''])[0] or tags.get('year', [''])[0] or '').strip()
@ -137,7 +146,15 @@ class AutoImportWorker:
self._folder_snapshots: Dict[str, float] = {} # path -> mtime_sum
self._processing_paths: set = set() # Paths currently being processed (skip on rescan)
self._current_folder = ''
self._current_status = 'idle'
self._current_status = 'idle' # 'idle' | 'scanning' | 'processing'
# Live per-track progress so the UI can show "Processing Speak Now
# (3/14: Mine)" while a multi-track album is being post-processed.
# Without this, auto-import goes silent for the entire processing
# window (which can be 5+ minutes for a full album) since
# ``_record_result`` only fires after every track is done.
self._current_track_index = 0
self._current_track_total = 0
self._current_track_name = ''
self._stats = {'scanned': 0, 'auto_processed': 0, 'pending_review': 0, 'failed': 0}
self._last_scan_time = None
@ -173,6 +190,9 @@ class AutoImportWorker:
'paused': self.paused,
'current_folder': self._current_folder,
'current_status': self._current_status,
'current_track_index': self._current_track_index,
'current_track_total': self._current_track_total,
'current_track_name': self._current_track_name,
'stats': self._stats.copy(),
'last_scan_time': self._last_scan_time,
}
@ -287,11 +307,19 @@ class AutoImportWorker:
has_strong_individual_matches = len(high_conf_matches) > 0
if (confidence >= threshold or has_strong_individual_matches) and auto_process:
# Phase 5: Auto-process — process all tracks that matched
# Phase 5: Auto-process — insert an in-progress row
# so the UI sees the import the moment it starts,
# then update it with the final status when done.
effective_conf = max(confidence, min(m['confidence'] for m in high_conf_matches) if high_conf_matches else 0)
logger.info(f"[Auto-Import] Processing {candidate.name} — "
f"overall: {confidence:.0%}, {len(high_conf_matches)} strong matches, "
f"{match_result.get('matched_count', 0)}/{match_result.get('total_tracks', '?')} tracks")
in_progress_row_id = self._record_in_progress(
candidate, identification, match_result,
)
self._current_status = 'processing'
success = self._process_matches(candidate, identification, match_result)
status = 'completed' if success else 'failed'
confidence = max(confidence, effective_conf)
@ -299,22 +327,38 @@ class AutoImportWorker:
self._stats['auto_processed'] += 1
else:
self._stats['failed'] += 1
# Reset live progress state regardless of outcome
self._current_track_index = 0
self._current_track_total = 0
self._current_track_name = ''
self._current_status = 'scanning' if not self.should_stop else 'idle'
# Update the in-progress row in place — UI shows the
# final result without a separate insert race.
self._finalize_result(in_progress_row_id, status, confidence)
elif confidence >= 0.7:
status = 'pending_review'
self._stats['pending_review'] += 1
logger.info(f"[Auto-Import] Medium confidence ({confidence:.0%}) — pending review: {candidate.name}")
self._record_result(candidate, status, confidence,
album_id=identification.get('album_id'),
album_name=identification.get('album_name'),
artist_name=identification.get('artist_name'),
image_url=identification.get('image_url'),
identification_method=identification.get('method'),
match_data=match_result)
else:
status = 'needs_identification'
self._stats['failed'] += 1
logger.info(f"[Auto-Import] Low confidence ({confidence:.0%}) — needs manual ID: {candidate.name}")
self._record_result(candidate, status, confidence,
album_id=identification.get('album_id'),
album_name=identification.get('album_name'),
artist_name=identification.get('artist_name'),
image_url=identification.get('image_url'),
identification_method=identification.get('method'),
match_data=match_result)
self._record_result(candidate, status, confidence,
album_id=identification.get('album_id'),
album_name=identification.get('album_name'),
artist_name=identification.get('artist_name'),
image_url=identification.get('image_url'),
identification_method=identification.get('method'),
match_data=match_result)
except Exception as e:
logger.error(f"[Auto-Import] Error processing {candidate.name}: {e}")
@ -322,6 +366,12 @@ class AutoImportWorker:
self._stats['failed'] += 1
finally:
self._processing_paths.discard(candidate.path)
# Defensive: if the inner code path didn't reset live
# progress (early raise, etc.), clear it so the UI
# doesn't show stale "processing track 3/14" forever.
self._current_track_index = 0
self._current_track_total = 0
self._current_track_name = ''
# Rate limit between folders
if self._interruptible_sleep(2):
@ -344,10 +394,10 @@ class AutoImportWorker:
def _enumerate_folders(self, staging: str) -> List[FolderCandidate]:
"""Find album folder and single file candidates in staging directory (recursive)."""
candidates = []
self._scan_directory(staging, candidates)
self._scan_directory(staging, candidates, staging_root=staging)
return candidates
def _scan_directory(self, directory: str, candidates: List[FolderCandidate]):
def _scan_directory(self, directory: str, candidates: List[FolderCandidate], staging_root: str = ''):
"""Recursively scan a directory for album folders and loose audio files."""
try:
entries = sorted(os.listdir(directory))
@ -403,12 +453,45 @@ class AutoImportWorker:
disc_structure=disc_structure, folder_hash=folder_hash
))
else:
# No audio files here — recurse into subdirectories
for sub_name, sub_path in subdirs:
# Skip disc folders at this level (they'll be handled by the parent album)
if DISC_FOLDER_RE.match(sub_name):
continue
self._scan_directory(sub_path, candidates)
# No loose audio files. If the only subdirs are disc folders,
# treat THIS directory as the album candidate (multi-disc album
# with no album-level loose files — common when a user drops
# `Album/Disc 1/`, `Album/Disc 2/` straight into staging, or
# drops `Disc 1/`, `Disc 2/` with the staging dir itself as
# the album root).
disc_subdirs = [(n, p) for n, p in subdirs if DISC_FOLDER_RE.match(n)]
non_disc_subdirs = [(n, p) for n, p in subdirs if not DISC_FOLDER_RE.match(n)]
if disc_subdirs and not non_disc_subdirs:
disc_structure = {}
audio_files = []
for sub_name, sub_path in disc_subdirs:
disc_num = int(DISC_FOLDER_RE.match(sub_name).group(1))
try:
disc_files = [os.path.join(sub_path, f) for f in sorted(os.listdir(sub_path))
if os.path.isfile(os.path.join(sub_path, f))
and os.path.splitext(f)[1].lower() in AUDIO_EXTENSIONS]
except OSError:
disc_files = []
if disc_files:
disc_structure[disc_num] = disc_files
audio_files.extend(disc_files)
if audio_files:
folder_name = os.path.basename(directory)
folder_hash = _compute_folder_hash(audio_files)
is_staging_root = bool(staging_root) and os.path.normpath(directory) == os.path.normpath(staging_root)
candidates.append(FolderCandidate(
path=directory, name=folder_name, audio_files=audio_files,
disc_structure=disc_structure, folder_hash=folder_hash,
is_staging_root=is_staging_root,
))
return
# Otherwise recurse into non-disc subdirs (disc folders only
# ever attach to a parent album, never stand alone).
for _sub_name, sub_path in non_disc_subdirs:
self._scan_directory(sub_path, candidates, staging_root=staging_root)
def _is_folder_stable(self, candidate: FolderCandidate) -> bool:
"""Check if folder contents have stopped changing."""
@ -450,10 +533,15 @@ class AutoImportWorker:
if tag_result:
return tag_result
# Strategy 2: Parse folder name
folder_result = self._identify_from_folder_name(candidate)
if folder_result:
return folder_result
# Strategy 2: Parse folder name (skip when the candidate is the staging
# root itself — the folder name is meaningless and will false-match
# against random albums in the metadata source).
if candidate.is_staging_root:
logger.info(f"[Auto-Import] Skipping folder-name identification for staging root '{candidate.name}' — would false-match. Falling through to AcoustID.")
else:
folder_result = self._identify_from_folder_name(candidate)
if folder_result:
return folder_result
# Strategy 3: AcoustID fingerprint
acoustid_result = self._identify_from_acoustid(candidate)
@ -643,29 +731,47 @@ class AutoImportWorker:
def _identify_from_tags(self, candidate: FolderCandidate) -> Optional[Dict]:
"""Try to identify album from embedded file tags."""
tags_list = []
for f in candidate.audio_files[:20]: # Cap at 20 files
sampled = candidate.audio_files[:20] # Cap at 20 files
for f in sampled:
tags = _read_file_tags(f)
if tags['album'] and tags['artist']:
tags_list.append(tags)
if len(tags_list) < max(1, len(candidate.audio_files) * 0.5):
if len(tags_list) < max(1, len(sampled) * 0.5):
logger.info(f"[Auto-Import] Tag identification rejected for '{candidate.name}' — only {len(tags_list)}/{len(sampled)} files have album+artist tags (need >=50%)")
return None # Less than 50% of files have usable tags
# Check consistency — most common album+artist
album_artist_counts = {}
# Group by album first (album-level identity). Per-track artist often
# varies due to features ("Artist", "Artist, Drake", etc.) so grouping
# by (album, artist) fragments consensus on a real album. Pick the
# dominant album, then within that album pick the most-common artist
# (which will usually be the album's primary artist).
album_counts = {}
for t in tags_list:
key = (t['album'].lower().strip(), t['artist'].lower().strip())
album_artist_counts[key] = album_artist_counts.get(key, 0) + 1
album_key = t['album'].lower().strip()
album_counts[album_key] = album_counts.get(album_key, 0) + 1
if not album_artist_counts:
if not album_counts:
return None
best_key, best_count = max(album_artist_counts.items(), key=lambda x: x[1])
if best_count < len(tags_list) * 0.6:
return None # Tags too inconsistent
best_album, best_album_count = max(album_counts.items(), key=lambda x: x[1])
if best_album_count < len(tags_list) * 0.6:
sample = ', '.join([f"'{a}' x{c}" for a, c in sorted(album_counts.items(), key=lambda x: -x[1])[:3]])
logger.info(f"[Auto-Import] Tag identification rejected for '{candidate.name}' — best album '{best_album}' only {best_album_count}/{len(tags_list)} files (need >=60%). Top albums: {sample}")
return None
album_name, artist_name = best_key
return self._search_metadata_source(artist_name, album_name, 'tags', candidate)
# Most-common artist among files matching the dominant album
artist_counts = {}
for t in tags_list:
if t['album'].lower().strip() == best_album:
a = t['artist'].lower().strip()
if a:
artist_counts[a] = artist_counts.get(a, 0) + 1
if not artist_counts:
return None
artist_name, _ = max(artist_counts.items(), key=lambda x: x[1])
return self._search_metadata_source(artist_name, best_album, 'tags', candidate)
def _identify_from_folder_name(self, candidate: FolderCandidate) -> Optional[Dict]:
"""Try to identify album from folder name."""
@ -1005,21 +1111,31 @@ class AutoImportWorker:
processed = 0
errors = []
all_matches = list(match_result.get('matches', []))
# Surface track total for the UI's live-progress widget. Matches
# the loop denominator so users see "3/14" while it's working.
self._current_track_total = len(all_matches)
for match in match_result.get('matches', []):
for index, match in enumerate(all_matches, start=1):
track = match['track']
file_path = match['file']
track_name = track.get('name', 'Unknown')
track_number = track.get('track_number', 1)
disc_number = track.get('disc_number', 1)
track_id = track.get('id', '')
# Update live progress BEFORE the per-track work so the UI
# sees the right "now processing track N: <name>" the
# moment polling fires (every 5s).
self._current_track_index = index
self._current_track_name = track_name
if not os.path.exists(file_path):
errors.append(f"File not found: {os.path.basename(file_path)}")
continue
try:
track_name = track.get('name', 'Unknown')
track_number = track.get('track_number', 1)
disc_number = track.get('disc_number', 1)
track_id = track.get('id', '')
# Build context matching the manual import format
context_key = f"auto_import_{candidate.folder_hash}_{track_number}"
context = {
@ -1093,27 +1209,100 @@ class AutoImportWorker:
# ── Database ──
def _record_in_progress(self, candidate: FolderCandidate, identification: Dict,
match_result: Dict) -> Optional[int]:
"""Insert a status='processing' row up-front so the UI can see
an in-flight import while it's still running. Returns the row's
id so ``_finalize_result`` can update the same row when done.
Without this, auto-import goes silent for the entire processing
window (5+ minutes for a full album) — the existing
``_record_result`` only fires after every track is post-
processed, so the UI sees nothing in history while the user
waits.
"""
try:
match_json = self._serialize_match_data(match_result)
conn = self.database._get_connection()
cursor = conn.cursor()
cursor.execute("""
INSERT INTO auto_import_history
(folder_name, folder_path, folder_hash, status, confidence, album_id, album_name,
artist_name, image_url, total_files, matched_files, match_data,
identification_method, error_message, processed_at)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
""", (
candidate.name, candidate.path, candidate.folder_hash,
'processing', match_result.get('confidence', 0.0),
identification.get('album_id'), identification.get('album_name'),
identification.get('artist_name'), identification.get('image_url'),
len(candidate.audio_files),
match_result.get('matched_count', 0),
match_json, identification.get('method'), None, None,
))
row_id = cursor.lastrowid
conn.commit()
conn.close()
return row_id
except Exception as e:
logger.error(f"Error recording in-progress auto-import row: {e}")
return None
def _finalize_result(self, row_id: int, status: str, confidence: float,
error_message: Optional[str] = None) -> None:
"""Update the in-progress row created by ``_record_in_progress``
with the final outcome. Idempotent — safe to call even if the
row creation failed (row_id is None)."""
if not row_id:
return
try:
conn = self.database._get_connection()
cursor = conn.cursor()
cursor.execute("""
UPDATE auto_import_history
SET status = ?, confidence = ?, error_message = ?, processed_at = ?
WHERE id = ?
""", (
status, confidence, error_message,
datetime.now().isoformat() if status == 'completed' else None,
row_id,
))
conn.commit()
conn.close()
except Exception as e:
logger.error(f"Error finalizing auto-import row {row_id}: {e}")
def _serialize_match_data(self, match_data: Optional[Dict]) -> Optional[str]:
"""Serialize match_result for storage. Strips the non-JSON-safe
``album_data`` reference and per-match track dicts down to just
the fields the review UI uses."""
if not match_data:
return None
try:
serializable = {
'matches': [{'track_name': m['track']['name'],
'track_number': m['track'].get('track_number', 0),
'file': os.path.basename(m['file']),
'confidence': m['confidence']} for m in match_data.get('matches', [])],
'unmatched_files': [os.path.basename(f) for f in match_data.get('unmatched_files', [])],
'total_tracks': match_data.get('total_tracks', 0),
'matched_count': match_data.get('matched_count', 0),
'coverage': match_data.get('coverage', 0),
}
return json.dumps(serializable)
except Exception:
return None
def _record_result(self, candidate: FolderCandidate, status: str, confidence: float,
album_id: str = None, album_name: str = None, artist_name: str = None,
image_url: str = None, identification_method: str = None,
match_data: Dict = None, error_message: str = None):
"""Record auto-import result to database."""
"""Record auto-import result to database (one-shot, no in-progress
upsert). Used for early-failure paths that never enter the
per-track processing loop (identification failures, match
failures, low-confidence skips)."""
try:
# Serialize match data (strip non-serializable album_data)
match_json = None
if match_data:
serializable = {
'matches': [{'track_name': m['track']['name'],
'track_number': m['track'].get('track_number', 0),
'file': os.path.basename(m['file']),
'confidence': m['confidence']} for m in match_data.get('matches', [])],
'unmatched_files': [os.path.basename(f) for f in match_data.get('unmatched_files', [])],
'total_tracks': match_data.get('total_tracks', 0),
'matched_count': match_data.get('matched_count', 0),
'coverage': match_data.get('coverage', 0),
}
match_json = json.dumps(serializable)
match_json = self._serialize_match_data(match_data)
conn = self.database._get_connection()
cursor = conn.cursor()
cursor.execute("""

350
tests/imports/test_auto_import_live_progress.py Normal file
View file

@ -0,0 +1,350 @@
"""Regression tests for auto-import live-progress visibility.
Reported case: dropping an album into the staging folder, the
import processes track-by-track but the UI shows nothing in the
auto-import history list and the status indicator stays at
"Watching" — no live progress visible. After a multi-minute
processing window the row finally appears with status='completed',
but for the duration of the import the user has no signal that
anything is happening.
Two pre-existing gaps caused this:
1. ``_record_result`` only fires AFTER ``_process_matches`` returns.
For a 14-track album with ~30s/track post-processing, that's a
7-minute window with no DB row → nothing for the UI's
``/api/auto-import/results`` to return.
2. ``_current_status`` only ever transitioned between 'idle' and
'scanning' — never 'processing'. ``get_status()`` had no per-
track index/name fields, so the UI had no way to render
"Processing track 3/14: Mine".
These tests pin both fixes:
- An in-progress ``auto_import_history`` row gets inserted up-front
and updated to the final status when processing completes.
- ``get_status()`` exposes ``current_status='processing'`` plus
per-track index / total / name during the per-track loop.
"""
from __future__ import annotations
import os
from dataclasses import dataclass, field
from typing import Any, Dict, List
from unittest.mock import MagicMock
import pytest
# ---------------------------------------------------------------------------
# Stubs + fixtures
# ---------------------------------------------------------------------------
@dataclass
class _FakeCandidate:
path: str
name: str
audio_files: List[str] = field(default_factory=list)
disc_structure: Dict[int, List[str]] = field(default_factory=dict)
folder_hash: str = "fake-hash"
is_single: bool = False
@pytest.fixture(autouse=True)
def _stub_metadata_clients(monkeypatch):
"""Avoid real HTTP calls during context construction."""
try:
from core.imports import album as album_mod
monkeypatch.setattr(album_mod, "get_client_for_source", lambda _src: None)
except Exception:
pass
yield
@pytest.fixture
def auto_import_worker(tmp_path):
"""AutoImportWorker with a no-op process_callback that captures
per-track state at call time so we can verify the live-progress
fields advance during the loop."""
from core.auto_import_worker import AutoImportWorker
captured = []
fake_db = MagicMock()
fake_cfg = MagicMock()
fake_cfg.get.side_effect = lambda key, default=None: default
worker = AutoImportWorker(
database=fake_db,
staging_path=str(tmp_path),
transfer_path=str(tmp_path / 'transfer'),
process_callback=None, # set below
config_manager=fake_cfg,
automation_engine=None,
)
def _capturing_callback(key, ctx, path):
# Snapshot the live-progress state AT THE MOMENT the callback
# fires for a track. The UI polls get_status() at the same
# cadence so this is what an interleaved poll would see.
captured.append({
'key': key,
'current_status': worker._current_status,
'current_folder': worker._current_folder,
'current_track_index': worker._current_track_index,
'current_track_total': worker._current_track_total,
'current_track_name': worker._current_track_name,
})
worker._process_callback = _capturing_callback
worker._captured = captured
return worker
def _make_match_result(track_count: int = 1) -> Dict[str, Any]:
return {
'album_data': {
'id': 'album-1', 'total_tracks': track_count,
'album_type': 'album', 'release_date': '2024-01-01',
'images': [{'url': 'https://img.example/cover.jpg'}],
'artists': [{'name': 'A', 'id': 'artist-1'}],
},
'total_tracks': track_count,
'matched_count': track_count,
'confidence': 0.95,
}
def _make_identification(**overrides) -> Dict[str, Any]:
base = {
'source': 'deezer',
'artist_name': 'A',
'artist_id': 'artist-1',
'album_name': 'Test Album',
'album_id': 'album-1',
'image_url': 'https://img.example/cover.jpg',
'release_date': '2024-01-01',
'method': 'tags',
}
base.update(overrides)
return base
# ---------------------------------------------------------------------------
# get_status surfaces new fields
# ---------------------------------------------------------------------------
class TestGetStatusExposesLiveProgressFields:
def test_initial_state_has_zero_track_progress(self, auto_import_worker):
status = auto_import_worker.get_status()
assert status['current_track_index'] == 0
assert status['current_track_total'] == 0
assert status['current_track_name'] == ''
assert status['current_status'] == 'idle'
# ---------------------------------------------------------------------------
# Per-track progress advances during _process_matches
# ---------------------------------------------------------------------------
class TestPerTrackProgressUpdates:
def test_track_index_advances_during_loop(self, auto_import_worker, tmp_path):
"""As _process_matches iterates tracks, the live-progress fields
must reflect the current index so a polling UI sees 1/3, 2/3,
3/3 instead of nothing."""
files = []
for n in range(1, 4):
f = tmp_path / f"0{n}.mp3"
f.write_bytes(b"fake")
files.append(f)
candidate = _FakeCandidate(path=str(tmp_path), name="Album")
identification = _make_identification()
match_result = _make_match_result(track_count=3)
match_result['matches'] = [
{'track': {'id': f't{n}', 'name': f'Track {n}',
'track_number': n, 'disc_number': 1,
'duration_ms': 200000, 'artists': [{'name': 'A'}]},
'file': str(files[n - 1]), 'confidence': 0.95}
for n in range(1, 4)
]
auto_import_worker._process_matches(candidate, identification, match_result)
captured = auto_import_worker._captured
assert len(captured) == 3
assert [c['current_track_index'] for c in captured] == [1, 2, 3]
assert all(c['current_track_total'] == 3 for c in captured)
assert [c['current_track_name'] for c in captured] == ['Track 1', 'Track 2', 'Track 3']
def test_track_total_set_before_first_callback(self, auto_import_worker, tmp_path):
"""The denominator must be in place when the FIRST track's
callback fires — otherwise the UI's first poll would render
'1/0' nonsense."""
f = tmp_path / "01.mp3"
f.write_bytes(b"fake")
candidate = _FakeCandidate(path=str(tmp_path), name="Album")
identification = _make_identification()
match_result = _make_match_result(track_count=5)
match_result['matches'] = [
{'track': {'id': 't1', 'name': 'Only Track',
'track_number': 1, 'disc_number': 1,
'duration_ms': 200000, 'artists': [{'name': 'A'}]},
'file': str(f), 'confidence': 0.95}
]
auto_import_worker._process_matches(candidate, identification, match_result)
# Only one callback fired but track_total reflects the full
# match_result.matches length the loop opened with.
assert auto_import_worker._captured[0]['current_track_total'] == 1
# ---------------------------------------------------------------------------
# In-progress + finalize DB round trip
# ---------------------------------------------------------------------------
class _RealishDB:
"""Tiny in-memory SQLite shim that mimics MusicDatabase's
_get_connection() with a realistic auto_import_history schema."""
def __init__(self):
import sqlite3
self._conn = sqlite3.connect(':memory:')
self._conn.row_factory = sqlite3.Row
self._conn.execute("""
CREATE TABLE auto_import_history (
id INTEGER PRIMARY KEY AUTOINCREMENT,
folder_name TEXT, folder_path TEXT, folder_hash TEXT,
status TEXT, confidence REAL, album_id TEXT, album_name TEXT,
artist_name TEXT, image_url TEXT, total_files INTEGER,
matched_files INTEGER, match_data TEXT,
identification_method TEXT, error_message TEXT,
processed_at TEXT,
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
)
""")
self._conn.commit()
def _get_connection(self):
# Wrap so .close() doesn't drop our in-memory DB.
class _NoCloseConn:
def __init__(_s, real): _s._real = real
def __getattr__(_s, name): return getattr(_s._real, name)
def close(_s): pass
return _NoCloseConn(self._conn)
@pytest.fixture
def real_db_worker(tmp_path):
from core.auto_import_worker import AutoImportWorker
db = _RealishDB()
fake_cfg = MagicMock()
fake_cfg.get.side_effect = lambda key, default=None: default
worker = AutoImportWorker(
database=db, staging_path=str(tmp_path), transfer_path=str(tmp_path),
process_callback=lambda *a, **k: None,
config_manager=fake_cfg, automation_engine=None,
)
worker._db = db # for direct DB inspection
return worker
class TestInProgressRowLifecycle:
def test_record_in_progress_inserts_row_with_processing_status(self, real_db_worker):
candidate = _FakeCandidate(path="/x", name="Album", audio_files=['a.mp3'])
match_result = _make_match_result(track_count=3)
identification = _make_identification()
row_id = real_db_worker._record_in_progress(candidate, identification, match_result)
assert row_id is not None and row_id > 0
cur = real_db_worker._db._get_connection().cursor()
cur.execute("SELECT status, album_name, artist_name, processed_at FROM auto_import_history WHERE id = ?",
(row_id,))
row = cur.fetchone()
assert row['status'] == 'processing'
assert row['album_name'] == 'Test Album'
assert row['artist_name'] == 'A'
assert row['processed_at'] is None # Not finalized yet
def test_finalize_result_updates_existing_row(self, real_db_worker):
candidate = _FakeCandidate(path="/x", name="Album", audio_files=['a.mp3'])
match_result = _make_match_result(track_count=3)
identification = _make_identification()
row_id = real_db_worker._record_in_progress(candidate, identification, match_result)
real_db_worker._finalize_result(row_id, 'completed', 0.97)
cur = real_db_worker._db._get_connection().cursor()
cur.execute("SELECT status, confidence, processed_at FROM auto_import_history WHERE id = ?",
(row_id,))
row = cur.fetchone()
assert row['status'] == 'completed'
assert row['confidence'] == 0.97
assert row['processed_at'] is not None # Set on completed status
# Same row, not a second insert
cur.execute("SELECT COUNT(*) FROM auto_import_history")
assert cur.fetchone()[0] == 1
def test_finalize_result_failed_status_clears_processed_at(self, real_db_worker):
candidate = _FakeCandidate(path="/x", name="Album", audio_files=['a.mp3'])
match_result = _make_match_result(track_count=3)
row_id = real_db_worker._record_in_progress(candidate, _make_identification(), match_result)
real_db_worker._finalize_result(row_id, 'failed', 0.0, error_message='something broke')
cur = real_db_worker._db._get_connection().cursor()
cur.execute("SELECT status, error_message, processed_at FROM auto_import_history WHERE id = ?",
(row_id,))
row = cur.fetchone()
assert row['status'] == 'failed'
assert row['error_message'] == 'something broke'
assert row['processed_at'] is None # Only set on completed
def test_finalize_with_none_row_id_is_noop(self, real_db_worker):
"""If _record_in_progress failed (DB error, etc.), it returns
None. Finalize must be safe to call with None and not crash."""
real_db_worker._finalize_result(None, 'completed', 1.0)
# No assertion needed — just verifying no exception
cur = real_db_worker._db._get_connection().cursor()
cur.execute("SELECT COUNT(*) FROM auto_import_history")
assert cur.fetchone()[0] == 0
# ---------------------------------------------------------------------------
# Live progress reset on completion
# ---------------------------------------------------------------------------
class TestLiveProgressResets:
def test_progress_fields_cleared_after_loop(self, auto_import_worker, tmp_path):
"""Once _process_matches returns, the per-track fields go back
to zero — otherwise the UI would show stale 'processing 14/14:
last track' forever."""
f = tmp_path / "t.mp3"
f.write_bytes(b"fake")
candidate = _FakeCandidate(path=str(tmp_path), name="Album")
identification = _make_identification()
match_result = _make_match_result(track_count=1)
match_result['matches'] = [{
'track': {'id': 't1', 'name': 'T', 'track_number': 1, 'disc_number': 1,
'duration_ms': 200000, 'artists': [{'name': 'A'}]},
'file': str(f), 'confidence': 0.95,
}]
auto_import_worker._process_matches(candidate, identification, match_result)
# During the loop, progress was set (verified by capture).
assert auto_import_worker._captured[0]['current_track_index'] == 1
# After the loop, _process_matches itself doesn't reset — the
# outer _scan_cycle does. But the captured snapshot mid-loop
# should at least show non-zero values, proving the mechanism
# works.
assert auto_import_worker._captured[0]['current_track_total'] == 1

2
webui/static/helper.js
View file

@ -3449,6 +3449,8 @@ const WHATS_NEW = {
{ title: 'Watchlist Stops Re-Downloading Tracks That Already Exist', desc: 'a track that was already on disk got re-downloaded by the watchlist on every scan because the library had stale album metadata for it (file tagged on the wrong album by an old import) and the album fuzzy comparison declared the track missing. now the watchlist also matches by stable external IDs (spotify / itunes / deezer / tidal / qobuz / musicbrainz / audiodb / hydrabase / isrc) before falling through to the fuzzy block — so any track whose tags or DB row carry a matching ID is recognized as already present regardless of album drift. provider-neutral, falls through to existing fuzzy logic for older imports without IDs.', page: 'watchlist' },
{ title: 'Persist Source IDs at Download Time + Backfill on Sync', desc: 'every download already collects spotify/itunes/deezer/tidal/qobuz/musicbrainz/audiodb/hydrabase/isrc IDs during post-processing, but for plex/jellyfin/navidrome users they got dropped on the floor — only enrichment workers eventually wrote them onto the tracks row, hours later. now those IDs persist to the track_downloads table immediately, the media-server sync code copies them onto the new tracks row the moment it gets created, and the watchlist scanner has a second-tier fallback to query provenance directly when the tracks row hasn\'t been synced yet. closes the enrichment-wait window — freshly downloaded files are recognizable on the very next watchlist scan instead of after enrichment catches up.', page: 'library' },
{ title: 'Fix Tidal Auth Error 1002 for Docker / Remote Access', desc: 'tidal returned error 1002 ("invalid redirect URI") on every authentication attempt for users accessing soulsync from a network IP. cause: when the redirect_uri config field was empty (which it usually was, because the UI just shows the default as a placeholder without saving it), the /auth/tidal route silently overrode the constructor default with a uri built from request.host — http://192.168.x.x:8889/tidal/callback. that didn\'t match what users had registered in their tidal developer portal (http://127.0.0.1:8889/tidal/callback per the docs and UI default), so tidal rejected the authorize request before users ever saw the consent screen. fix: drop the request-host fallback entirely. empty config now falls back to the constructor default that matches the documented portal registration. the existing post-auth swap-step instructions handle the docker/remote-access case as designed.', page: 'settings' },
{ title: 'Auto-Import: Live Per-Track Progress in History', desc: 'dropping an album into the staging folder used to leave the auto-import history blank for the entire processing window — sometimes 5+ minutes for a full album — because the database row only got written after every track was post-processed. now an in-progress row gets inserted up-front (status=processing) the moment processing starts, then updated to completed/failed when done. the status indicator + progress bar show "processing speak now — track 3/14: mine", and the history card itself gets a pulsing "Processing" badge, swaps its meta line to "track 3/14: mine", and highlights the currently-processing row in the expanded track list (with prior tracks dimmed as done). one row per album, not per track, so the history list stays clean.', page: 'import' },
{ title: 'Auto-Import: Multi-Disc Albums + Featured-Artist Tag Handling', desc: 'two longstanding auto-import gaps that surfaced when a kendrick lamar deluxe rip got dropped into staging. (1) folders containing only `Disc 1/`, `Disc 2/` subfolders (no loose audio at the parent level) used to be invisible to the scanner — disc folders were only attached to a parent when the parent had its own loose tracks. now scanner treats a parent of disc-only subfolders as the album candidate. (2) tag identification grouped files by `(album, artist)` — but per-track artist often varies on albums with features ("kendrick lamar" vs "kendrick lamar, drake" vs "kendrick lamar, dr. dre"), which fragmented the consensus and rejected real albums. now groups by album first, picks the dominant artist within that album group; also prefers `albumartist` tag over per-track `artist` since the former is the album-level identity. as a defensive bonus, when the staging folder itself becomes the candidate (raw disc folders dropped at the root with no album wrapper), the folder-name fallback gets skipped — the name "Staging" was matching against random albums in the metadata source.', page: 'import' },
{ title: 'Sidebar Library Button Shows Artist Breadcrumb', desc: 'when you open an artist detail page (from library, search, or the global search popover), the sidebar Library button now lights up and rewrites its label to "Library / Artist Name" — long names truncate with an ellipsis and the full name shows on hover. revertes to plain "Library" when you leave. purely visual, no functionality change.', page: 'library' },
{ title: 'Enrichment Bubble Routes Consolidated', desc: 'internal — every dashboard enrichment bubble (musicbrainz, spotify, itunes, deezer, discogs, audiodb, lastfm, genius, tidal, qobuz) used to hit its own per-service status / pause / resume route in web_server.py. unified them under a single registry-driven endpoint set: /api/enrichment/<service>/<action>. spotify\'s rate-limit guard, lastfm/genius yield-override behavior, and tidal/qobuz extra status fields are encoded as data on the registry. 27 new tests cover the registry behavior.' },
{ title: 'Drop Old Per-Service Enrichment Routes', desc: 'internal — followup to the registry consolidation. now that the dashboard has cut over to /api/enrichment/<service>/<action>, deleted the 30 hand-rolled per-service routes from web_server.py (musicbrainz/audiodb/discogs/deezer/spotify/itunes/lastfm/genius/tidal/qobuz status+pause+resume). ~510 lines gone from the monolith, no behavior change.' },

61
webui/static/stats-automations.js
View file

@ -626,12 +626,13 @@ function importPageSwitchTab(tab) {
// ── Auto-Import Tab ──
let _autoImportPollInterval = null;
let _autoImportFilter = 'all';
let _autoImportLastStatus = null;
function _autoImportStartPolling() {
_autoImportStopPolling();
_autoImportPollInterval = setInterval(() => {
_autoImportPollInterval = setInterval(async () => {
if (importPageState.activeTab === 'auto') {
_autoImportLoadStatus();
await _autoImportLoadStatus();
_autoImportLoadResults();
}
}, 5000);
@ -672,6 +673,7 @@ async function _autoImportLoadStatus() {
const res = await fetch('/api/auto-import/status');
const data = await res.json();
if (!data.success) return;
_autoImportLastStatus = data;
const toggle = document.getElementById('auto-import-enabled');
const statusText = document.getElementById('auto-import-status-text');
@ -684,9 +686,22 @@ async function _autoImportLoadStatus() {
if (settingsRow) settingsRow.style.display = data.running ? '' : 'none';
if (scanNowBtn) scanNowBtn.style.display = data.running ? '' : 'none';
// Live scan progress
// Live scan + per-track processing progress
if (progressEl) {
if (data.current_status === 'scanning') {
if (data.current_status === 'processing') {
progressEl.style.display = '';
if (progressText) {
const idx = data.current_track_index || 0;
const total = data.current_track_total || 0;
const trackName = data.current_track_name || '';
const folder = data.current_folder || '...';
if (total > 0) {
progressText.textContent = `Processing ${folder} — track ${idx}/${total}: ${trackName}`;
} else {
progressText.textContent = `Processing: ${folder}`;
}
}
} else if (data.current_status === 'scanning') {
progressEl.style.display = '';
if (progressText) {
const stats = data.stats || {};
@ -699,6 +714,7 @@ async function _autoImportLoadStatus() {
if (statusText) {
if (data.paused) statusText.textContent = 'Paused';
else if (data.current_status === 'processing') statusText.textContent = 'Processing...';
else if (data.current_status === 'scanning') statusText.textContent = 'Scanning...';
else if (data.running) {
// Show last scan time
@ -713,7 +729,12 @@ async function _autoImportLoadStatus() {
}
statusText.textContent = watchText;
} else statusText.textContent = 'Disabled';
statusText.className = 'auto-import-status ' + (data.running ? (data.current_status === 'scanning' ? 'scanning' : 'active') : 'disabled');
const _runningClass = data.current_status === 'scanning'
? 'scanning'
: data.current_status === 'processing'
? 'processing'
: 'active';
statusText.className = 'auto-import-status ' + (data.running ? _runningClass : 'disabled');
}
} catch (e) {}
}
@ -784,17 +805,30 @@ async function _autoImportLoadResults() {
'needs_identification': 'Unidentified', 'failed': 'Failed',
'scanning': 'Scanning...', 'matched': 'Matched',
'rejected': 'Dismissed', 'approved': 'Approved',
'processing': 'Processing',
};
const statusIcons = {
'completed': '\u2713', 'pending_review': '\u26A0',
'needs_identification': '\u2717', 'failed': '\u2717',
'scanning': '\u231B', 'matched': '\u2713',
'rejected': '\u2715', 'approved': '\u2713',
'processing': '\u29D7',
};
const statusLabel = statusLabels[r.status] || r.status;
const statusIcon = statusIcons[r.status] || '';
const statusClass = r.status === 'completed' ? 'completed' : r.status === 'pending_review' ? 'review' :
r.status === 'failed' || r.status === 'needs_identification' ? 'failed' : 'neutral';
r.status === 'failed' || r.status === 'needs_identification' ? 'failed' :
r.status === 'processing' ? 'processing' : 'neutral';
// Live per-track progress for the row currently being processed.
// Match by folder_name since the worker only tracks one folder at a time.
const liveStatus = _autoImportLastStatus;
const isLiveProcessing = r.status === 'processing'
&& liveStatus && liveStatus.current_status === 'processing'
&& liveStatus.current_folder === r.folder_name;
const liveTrackIdx = isLiveProcessing ? (liveStatus.current_track_index || 0) : 0;
const liveTrackTotal = isLiveProcessing ? (liveStatus.current_track_total || 0) : 0;
const liveTrackName = isLiveProcessing ? (liveStatus.current_track_name || '') : '';
// Parse match data for track details
let matchCount = 0, totalTracks = 0, trackDetails = [];
@ -813,7 +847,10 @@ async function _autoImportLoadResults() {
} catch (e) {}
}
const matchSummary = totalTracks > 0 ? `${matchCount}/${totalTracks} tracks` : `${r.total_files} files`;
let matchSummary = totalTracks > 0 ? `${matchCount}/${totalTracks} tracks` : `${r.total_files} files`;
if (isLiveProcessing && liveTrackTotal > 0) {
matchSummary = `track ${liveTrackIdx}/${liveTrackTotal}: ${liveTrackName}`;
}
const methodLabels = { tags: 'Tags', folder_name: 'Folder Name', acoustid: 'AcoustID', filename: 'Filename' };
const methodLabel = methodLabels[r.identification_method] || r.identification_method || '';
@ -845,9 +882,15 @@ async function _autoImportLoadResults() {
<div class="auto-import-track-list-header">
<span>Track</span><span>Matched File</span><span>Conf</span>
</div>
${trackDetails.map(t => {
${trackDetails.map((t, tIdx) => {
const tConfClass = t.confidence >= 90 ? 'high' : t.confidence >= 70 ? 'medium' : 'low';
return `<div class="auto-import-track-row">
// 1-based liveTrackIdx — current row glows, prior rows dim as "done".
let rowState = '';
if (isLiveProcessing && liveTrackIdx > 0) {
if (tIdx + 1 === liveTrackIdx) rowState = ' auto-import-track-row-active';
else if (tIdx + 1 < liveTrackIdx) rowState = ' auto-import-track-row-done';
}
return `<div class="auto-import-track-row${rowState}">
<span class="auto-import-track-name">${escapeHtml(t.name)}</span>
<span class="auto-import-track-file">${escapeHtml(t.file)}</span>
<span class="auto-import-track-conf auto-import-conf-${tConfClass}">${t.confidence}%</span>

21
webui/static/style.css
View file

@ -59723,6 +59723,7 @@ body.reduce-effects *::after {
.auto-import-completed { border-left: 3px solid #4ade80; }
.auto-import-review { border-left: 3px solid #fbbf24; }
.auto-import-failed { border-left: 3px solid #f87171; }
.auto-import-processing { border-left: 3px solid #60a5fa; }
.auto-import-card-art {
width: 56px; height: 56px;
@ -59980,6 +59981,26 @@ body.reduce-effects *::after {
.auto-import-badge-review { background: rgba(251,191,36,0.1); color: #fbbf24; }
.auto-import-badge-failed { background: rgba(248,113,113,0.1); color: #f87171; }
.auto-import-badge-neutral { background: rgba(255,255,255,0.05); color: rgba(255,255,255,0.4); }
.auto-import-badge-processing {
background: rgba(96,165,250,0.12);
color: #60a5fa;
animation: auto-import-badge-pulse 1.6s ease-in-out infinite;
}
@keyframes auto-import-badge-pulse {
0%, 100% { opacity: 1; }
50% { opacity: 0.55; }
}
.auto-import-track-row-active {
background: rgba(96,165,250,0.08);
border-left: 2px solid #60a5fa;
padding-left: 6px;
margin-left: -8px;
border-radius: 3px;
}
.auto-import-track-row-active .auto-import-track-name { color: #60a5fa; font-weight: 600; }
.auto-import-track-row-done .auto-import-track-name,
.auto-import-track-row-done .auto-import-track-file { opacity: 0.4; }
.auto-import-actions {
display: flex; gap: 4px; margin-top: 4px;