From 0e14ff03eec3e0a85b3180fe06b3439cb04bb5a0 Mon Sep 17 00:00:00 2001 From: dev Date: Mon, 15 Jun 2026 15:39:35 +0200 Subject: [PATCH] fix(quality-scan): walk music folders on disk instead of resolving DB paths MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The quality job kept resolving 0/N because DB-path resolution failed in the deployed environment for reasons the logs wouldn't surface. Switch to the mechanism the WORKING file tools use: os.walk the real music folders (transfer + download + configured library paths, abspath'd) exactly like orphan_file_detector and fake_lossless_detector — those reliably see files because they never touch the DB's stored relative paths. - Walk all existing music dirs, collect audio files (dedup by realpath), probe each with the same probe_audio_quality the import guard uses, check quality_meets_profile (strict). Below-profile files become findings. - Match each walked file back to its DB track via a path-suffix index (last 1-3 components) for real title/artist/album + track id; fall back to the file's own tags when no DB row matches (finding filed as 'file'). - Loud diagnostics: logs the folders walked and the audio-file count, and warns clearly when no music folder exists to walk. The fix handler already works with the now-absolute file_path and an optional entity_id (deletes the file by real path; DB row only when known). Co-Authored-By: Claude Sonnet 4.6 --- core/repair_jobs/quality_upgrade_scanner.py | 265 +++++++++++--------- 1 file changed, 143 insertions(+), 122 deletions(-) diff --git a/core/repair_jobs/quality_upgrade_scanner.py b/core/repair_jobs/quality_upgrade_scanner.py index 5a0c5da8..b6f16435 100644 --- a/core/repair_jobs/quality_upgrade_scanner.py +++ b/core/repair_jobs/quality_upgrade_scanner.py @@ -1,19 +1,22 @@ """Quality Upgrade Scanner Job — flags library tracks below the user's profile. -Scans the whole library (not just Transfer) by resolving each track's DB file -path to a real file on disk, probing its ACTUAL measured audio quality (bit -depth / sample rate / bitrate via mutagen — the SAME `probe_audio_quality` the -download import guard uses), and checking it against the user's v3 ranked -targets with `quality_meets_profile` (strict; fallback ignored — that's a -download-time concession, not a definition of "good enough"). +Walks the music folders ON DISK (transfer + download + every configured library +path) exactly like the Orphan / Fake-Lossless detectors — those reliably "see" +files because they os.walk real directories instead of trying to resolve the +DB's stored (often relative) paths. For each audio file it probes the ACTUAL +measured audio quality (bit depth / sample rate / bitrate via the same +`probe_audio_quality` the download import guard uses) and checks it against the +user's v3 ranked targets with `quality_meets_profile` (strict — fallback +ignored, that's a download-time concession, not a definition of "good enough"). -Every track that satisfies none of the targets becomes a finding the user can: +Every file that satisfies none of the targets becomes a finding the user can: - 'redownload': add the track to the wishlist and delete the low-quality file - - 'delete': remove the low-quality file and its DB record + - 'delete': remove the low-quality file (+ DB row when known) - 'ignore': dismiss the finding (handled in the UI via the dismiss endpoint) -This unifies the library quality check onto the exact same pipeline as the -download quality gate — no more extension-only tier guessing. +Each walked file is matched back to its DB track (by path suffix) so the finding +carries the real title/artist/album + track id; when no DB row matches, the +file's own tags are used and the finding is filed as a loose 'file'. """ import os @@ -24,6 +27,8 @@ from utils.logging_config import get_logger logger = get_logger("repair_job.quality_upgrade") +AUDIO_EXTENSIONS = {'.mp3', '.flac', '.ogg', '.opus', '.m4a', '.aac', '.wav', '.wma', '.aiff', '.aif'} + @register_job class QualityUpgradeScannerJob(RepairJob): @@ -31,18 +36,17 @@ class QualityUpgradeScannerJob(RepairJob): display_name = 'Quality Upgrade Scanner' description = 'Flags library tracks below your quality profile' help_text = ( - 'Scans your music library and checks every track\'s REAL audio quality ' - '(bit depth, sample rate, bitrate — read from the file itself, not just ' - 'the extension) against your configured quality profile. This is the same ' - 'check the download pipeline runs, so a track flagged here is one the ' - 'downloader would also reject.\n\n' + 'Scans your music folders on disk and checks every track\'s REAL audio ' + 'quality (bit depth, sample rate, bitrate — read from the file itself, ' + 'not just the extension) against your configured quality profile. This is ' + 'the same check the download pipeline runs, so a track flagged here is one ' + 'the downloader would also reject.\n\n' 'Each below-profile track is reported as a finding. You can:\n' '• Re-download — add the track to your wishlist and remove the low-quality file\n' - '• Delete — remove the low-quality file entirely\n' + '• Delete — remove the low-quality file\n' '• Ignore — dismiss the finding and keep the file\n\n' 'The scan only reports — it never deletes or re-downloads on its own. ' - 'Profile targets and fallback come straight from Settings → Quality, so ' - 'adjusting your profile changes what counts as "below quality" here.' + 'Profile targets and fallback come straight from Settings → Quality.' ) icon = 'repair-icon-lossless' default_enabled = False @@ -72,41 +76,52 @@ class QualityUpgradeScannerJob(RepairJob): from core.imports.file_ops import probe_audio_quality - db_tracks = self._load_db_tracks(context) - if not db_tracks: - logger.info("No library tracks with file paths found") + # --- Collect the music folders to walk (real dirs, abspath'd) --- + base_dirs = self._collect_music_dirs(context) + if not base_dirs: + logger.warning( + "[QualityScan] No existing music folder to walk (transfer=%r, cwd=%r). " + "Set soulseek.transfer_path to the real mount or add your library under " + "Settings → Library → Music Paths.", + context.transfer_folder, os.getcwd()) return result + logger.info("[QualityScan] Walking %d folder(s): %r", len(base_dirs), base_dirs) - track_list = sorted(db_tracks.items(), key=lambda x: str(x[0])) - total = len(track_list) + # --- Gather audio files (dedup by real path) --- + audio_files = [] + seen = set() + for base in base_dirs: + for root, _dirs, files in os.walk(base): + if context.check_stop(): + return result + for fname in files: + if os.path.splitext(fname)[1].lower() in AUDIO_EXTENSIONS: + fpath = os.path.join(root, fname) + rp = os.path.realpath(fpath) + if rp in seen: + continue + seen.add(rp) + audio_files.append(fpath) + + total = len(audio_files) + logger.info("[QualityScan] Found %d audio file(s) to check", total) if context.report_progress: - context.report_progress(phase=f'Scanning {total} library tracks...', total=total) + context.report_progress(phase=f'Checking {total} files...', total=total) if context.update_progress: context.update_progress(0, total) + # --- DB suffix index so a walked file maps back to its track row --- + db_index = self._build_db_suffix_index(context) + probe_failed = 0 - _diag_logged = False - for i, (track_id, info) in enumerate(track_list): + for i, fpath in enumerate(audio_files): if context.check_stop(): return result if i % 20 == 0 and context.wait_if_paused(): return result - raw_fp = info.get('file_path', '') - resolved = self._resolve_path(raw_fp, context) - if not resolved: - # One-shot diagnostic on the first unresolved track — logs EXACTLY - # what the resolver tried (base dirs, cwd) so a path/mount mismatch - # is diagnosable instead of a silent "all skipped". - if not _diag_logged: - _diag_logged = True - self._log_resolve_diag(raw_fp, context) - result.skipped += 1 - probe_failed += 1 - continue - result.scanned += 1 - fname = os.path.basename(resolved) + fname = os.path.basename(fpath) if context.report_progress and i % 25 == 0: context.report_progress( scanned=i + 1, total=total, @@ -116,29 +131,30 @@ class QualityUpgradeScannerJob(RepairJob): ) try: - aq = probe_audio_quality(resolved) + aq = probe_audio_quality(fpath) except Exception as e: logger.debug("Probe failed for %s: %s", fname, e) aq = None - if aq is None: - # File couldn't be read — can't judge quality, leave unflagged. probe_failed += 1 result.skipped += 1 continue - # Strict profile check — identical to the download guard. if quality_meets_profile(aq, targets): if context.update_progress and (i + 1) % 25 == 0: context.update_progress(i + 1, total) continue - # Below profile → create a finding. + # Below profile → resolve metadata (DB row preferred, file tags fallback). + meta = self._lookup_meta(fpath, db_index) current_label = aq.label() target_labels = [t.label for t in targets] + disp_title = meta.get('title') or os.path.splitext(fname)[0] + disp_artist = meta.get('artist') or 'Unknown' + if context.report_progress: context.report_progress( - log_line=f'Below quality: {fname} — {current_label}', + log_line=f'Below quality: {disp_title} — {current_label}', log_type='error', ) if context.create_finding: @@ -146,13 +162,12 @@ class QualityUpgradeScannerJob(RepairJob): job_id=self.job_id, finding_type='quality_upgrade', severity='info', - entity_type='track', - entity_id=str(track_id), - file_path=resolved, - title=f'Below quality: {info.get("title") or fname} ({current_label})', + entity_type='track' if meta.get('track_id') else 'file', + entity_id=str(meta['track_id']) if meta.get('track_id') else None, + file_path=fpath, + title=f'Below quality: {disp_title} ({current_label})', description=( - f'"{info.get("title") or fname}" by ' - f'{info.get("artist") or "Unknown"} is {current_label}, ' + f'"{disp_title}" by {disp_artist} is {current_label}, ' f'which does not meet your quality profile ' f'({", ".join(target_labels[:3])}' f'{"…" if len(target_labels) > 3 else ""}).' @@ -164,12 +179,12 @@ class QualityUpgradeScannerJob(RepairJob): 'current_sample_rate': aq.sample_rate, 'current_bit_depth': aq.bit_depth, 'target_qualities': target_labels, - 'expected_title': info.get('title', ''), - 'expected_artist': info.get('artist', ''), - 'album_title': info.get('album_title', ''), - 'track_number': info.get('track_number'), - 'album_thumb_url': info.get('album_thumb_url'), - 'artist_thumb_url': info.get('artist_thumb_url'), + 'expected_title': disp_title, + 'expected_artist': disp_artist, + 'album_title': meta.get('album', ''), + 'track_number': meta.get('track_number'), + 'album_thumb_url': meta.get('album_thumb_url'), + 'artist_thumb_url': meta.get('artist_thumb_url'), }, ) if inserted: @@ -184,17 +199,42 @@ class QualityUpgradeScannerJob(RepairJob): context.update_progress(total, total) if probe_failed: - logger.warning( - "Quality upgrade scan: %d/%d tracks could not be read/resolved and " - "were left unflagged (their quality couldn't be verified).", - probe_failed, total) - logger.info("Quality upgrade scan: %d scanned, %d below profile, %d skipped", + logger.warning("[QualityScan] %d/%d files could not be probed (unreadable)", + probe_failed, total) + logger.info("Quality upgrade scan: %d checked, %d below profile, %d skipped", result.scanned, result.findings_created, result.skipped) return result - def _load_db_tracks(self, context: JobContext) -> dict: - """Load all library tracks keyed by ID (mirrors AcoustIDScannerJob).""" - tracks = {} + def _collect_music_dirs(self, context: JobContext) -> list: + """All existing music directories to walk, as absolute paths (dedup).""" + cm = context.config_manager + raw = [context.transfer_folder] + if cm: + try: + raw.append(cm.get('soulseek.transfer_path', './Transfer')) + raw.append(cm.get('soulseek.download_path', './downloads')) + mp = cm.get('library.music_paths', []) or [] + if isinstance(mp, list): + raw.extend([p for p in mp if isinstance(p, str) and p.strip()]) + except Exception as e: + logger.debug("music dir config read failed: %s", e) + out, seen = [], set() + for d in raw: + if not d: + continue + ad = os.path.abspath(d) + if ad in seen: + continue + seen.add(ad) + if os.path.isdir(ad): + out.append(ad) + return out + + def _build_db_suffix_index(self, context: JobContext) -> dict: + """Map normalized path suffixes (last 1-3 components, lowercased) → + track metadata, so a walked absolute file can be matched to its DB row + even when the DB stores a different (relative) path prefix.""" + index = {} conn = None try: conn = context.db._get_connection() @@ -210,75 +250,56 @@ class QualityUpgradeScannerJob(RepairJob): WHERE t.file_path IS NOT NULL AND t.file_path != '' """) for row in cursor.fetchall(): - track_id = row[0] - if track_id is None: + fp = (row[3] or '').replace('\\', '/') + if not fp: continue - tracks[str(track_id)] = { + parts = fp.split('/') + meta = { + 'track_id': row[0], 'title': row[1] or '', 'artist': row[2] or '', - 'file_path': row[3] or '', 'track_number': row[4], - 'album_title': row[5] or '', + 'album': row[5] or '', 'album_thumb_url': row[6] or None, 'artist_thumb_url': row[7] or None, } + for depth in range(1, min(4, len(parts) + 1)): + suffix = '/'.join(parts[-depth:]).lower() + index.setdefault(suffix, meta) except Exception as e: - logger.error("Error loading tracks from DB: %s", e) + logger.error("Error building DB suffix index: %s", e) finally: if conn: conn.close() - return tracks + return index - def _resolve_path(self, file_path, context): - """Resolve a DB file path to an actual file on disk via the shared - resolver (picks up library.music_paths + Plex locations too).""" - if not file_path: - return None - if os.path.exists(file_path): - return file_path - from core.library.path_resolver import resolve_library_file_path - return resolve_library_file_path( - file_path, - transfer_folder=context.transfer_folder, - config_manager=context.config_manager, - ) - - def _log_resolve_diag(self, file_path, context): - """Log a detailed diagnostic for the first track whose path can't be - resolved — the only reliable way to tell apart a CWD problem, a wrong - transfer mount, or genuinely-missing files in this container.""" - from core.library.path_resolver import resolve_library_file_path_with_diagnostic + def _lookup_meta(self, fpath: str, db_index: dict) -> dict: + """Match a walked file to a DB track via path suffix; fall back to the + file's own tags for title/artist/album when nothing matches.""" + parts = fpath.replace('\\', '/').split('/') + for depth in range(min(3, len(parts)), 0, -1): + suffix = '/'.join(parts[-depth:]).lower() + hit = db_index.get(suffix) + if hit: + return hit + # No DB row — read the file's own tags. + meta = {'track_id': None} try: - _, attempt = resolve_library_file_path_with_diagnostic( - file_path, - transfer_folder=context.transfer_folder, - config_manager=context.config_manager, - ) - tf = context.transfer_folder - abs_tf = os.path.abspath(tf) if tf else None - logger.warning( - "[QualityResolve] unresolved db_path=%r | cwd=%r | transfer_folder=%r " - "(abspath=%r, isdir=%s) | base_dirs_tried=%r | abs_join_exists=%s", - file_path, os.getcwd(), tf, abs_tf, - os.path.isdir(abs_tf) if abs_tf else None, - attempt.base_dirs_tried, - os.path.exists(os.path.join(abs_tf, file_path)) if abs_tf else None, - ) + from mutagen import File as MutagenFile + audio = MutagenFile(fpath, easy=True) + if audio: + meta['title'] = (audio.get('title') or [None])[0] or '' + meta['artist'] = (audio.get('artist') or audio.get('albumartist') or [None])[0] or '' + meta['album'] = (audio.get('album') or [None])[0] or '' except Exception as e: - logger.warning("[QualityResolve] diagnostic failed: %s", e) + logger.debug("tag read failed for %s: %s", os.path.basename(fpath), e) + return meta def estimate_scope(self, context: JobContext) -> int: - conn = None - try: - conn = context.db._get_connection() - cursor = conn.cursor() - cursor.execute(""" - SELECT COUNT(*) FROM tracks - WHERE file_path IS NOT NULL AND file_path != '' - """) - return cursor.fetchone()[0] - except Exception: - return 0 - finally: - if conn: - conn.close() + count = 0 + for base in self._collect_music_dirs(context): + for _root, _dirs, files in os.walk(base): + for fname in files: + if os.path.splitext(fname)[1].lower() in AUDIO_EXTENSIONS: + count += 1 + return count