From d1698180438d029904ec4f461efa682f097ba7a3 Mon Sep 17 00:00:00 2001 From: nick2000713 Date: Tue, 23 Jun 2026 12:43:02 +0200 Subject: [PATCH] feat(quality): upgrade Finder to v3 quality + clarify the two quality jobs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The two library quality jobs overlapped confusingly. Keep both, but make each one's role obvious and put them on the same v3 quality definition. Quality Upgrade Finder (quality_upgrade) — the ACTIVE job: - Quality decision moved from v2 (extension + DB bitrate) to v3: probes the REAL file with mutagen (measured bit depth / sample rate / bitrate) and checks it against the profile's ranked targets — same as the import guard. - New optional `deep_audio_verify` setting (default OFF): also run the ffmpeg decode guard (truncation + silence); a broken file is proposed for replacement. - Renamed to "Quality Upgrade Finder (active — proposes a replacement)" + help text spells out it actively searches a better version and queues it. - v3 helpers imported at module level so they stay monkeypatchable in tests. Quality Check (quality_upgrade_scanner) — the FLAG-ONLY job: - `deep_audio_verify` default flipped ON->OFF (the ffmpeg decode is the CPU-heavy step; matches the download pipeline's default). - Renamed to "Quality Check (flag only — you decide per finding)" + help text contrasts it with the active Finder. UI: deep_audio_verify setting label now shows "(ffmpeg decode — CPU heavy)". Tests: scan() tests stub the v3 probe path (probe_audio_quality / quality_meets_profile / resolve_library_file_path) since they use fake paths. The v2 pure-function helpers stay (still unit-tested). Co-Authored-By: Claude Opus 4.8 --- core/repair_jobs/quality_upgrade.py | 106 +++++++++++++++----- core/repair_jobs/quality_upgrade_scanner.py | 47 +++++---- tests/repair_jobs/test_quality_upgrade.py | 20 +++- webui/static/enrichment.js | 6 ++ 4 files changed, 129 insertions(+), 50 deletions(-) diff --git a/core/repair_jobs/quality_upgrade.py b/core/repair_jobs/quality_upgrade.py index f0ad8c53..b1b610ab 100644 --- a/core/repair_jobs/quality_upgrade.py +++ b/core/repair_jobs/quality_upgrade.py @@ -39,6 +39,11 @@ from core.discovery.quality_scanner import ( ) from core.library.file_tags import read_embedded_tags from core.library.path_resolver import resolve_library_file_path +# v3 quality: probe the real file + check it against the ranked profile targets, +# the SAME definition the download import guard uses. Module-level so they're +# monkeypatchable in tests. +from core.imports.file_ops import probe_audio_quality +from core.quality.selection import targets_from_profile, quality_meets_profile from utils.logging_config import get_logger logger = get_logger("repair_jobs.quality_upgrade") @@ -436,45 +441,58 @@ def _find_best_match(engine: Any, source_priority: List[str], title: str, artist @register_job class QualityUpgradeJob(RepairJob): job_id = 'quality_upgrade' - display_name = 'Quality Upgrade Finder' - description = 'Finds library tracks below your preferred quality and proposes a better version' + display_name = 'Quality Upgrade Finder (active — proposes a replacement)' + description = 'Finds library tracks below your quality profile and actively searches a better version to add to the wishlist' help_text = ( - 'Scans your library (or just your watchlist artists) and compares each ' - "track against your Quality Profile using BOTH the file format and its " - 'bitrate — so a 128 kbps MP3 is no longer treated the same as a 320 kbps ' - 'one, and enabling MP3-320/256 in your profile actually counts.\n\n' - 'For every track below your preferred quality it resolves the exact better ' - 'version using the most precise identity available, in order: the source ' - "track ID enrichment wrote into the file → the file's ISRC → the album's " - 'tracklist (by stored album ID or album search) → a name/artist search. The ' - 'fuzzy steps also reject candidates whose length is off (wrong live/edit cut). ' - 'It skips tracks it already proposed, so re-runs are cheap. Nothing is queued ' - 'automatically: applying a finding adds that matched track — with its album ' - 'context — to the wishlist, the same as any other download.\n\n' + 'ACTIVE quality job. For every library track below your quality profile it ' + 'goes one step further than the Quality Check (flag-only) scanner: it ' + 'actively SEARCHES your metadata source for the exact better version and ' + 'attaches it to the finding, so Apply adds that track straight to your ' + 'wishlist.\n\n' + 'Quality is judged the SAME way as the download/import pipeline — it reads ' + 'the REAL file with mutagen (measured bit depth / sample rate / bitrate) and ' + 'checks it against your v3 quality profile targets (strict: fallback is ' + 'ignored, that\'s a download-time concession, not "good enough" for an ' + 'upgrade). So a 128 kbps MP3, a 16-bit FLAC where you want 24-bit, etc. are ' + 'all caught accurately.\n\n' + 'For every below-profile track it resolves the better version by the most ' + 'precise identity available, in order: the source track ID enrichment wrote ' + "into the file → the file's ISRC → the album's tracklist (by stored album ID " + 'or album search) → a name/artist search (with a duration guard against wrong ' + 'live/edit cuts). It skips tracks it already proposed, so re-runs are cheap. ' + 'Nothing is queued automatically — applying a finding adds the matched track ' + '(with album context) to the wishlist, like any other download.\n\n' 'Settings:\n' '- Scope: "watchlist" (watchlisted artists only) or "all" (whole library)\n' - '- Min confidence: minimum match confidence (0-1) to surface a finding\n\n' - 'Note: detecting fake/transcoded lossless files is handled by the separate ' - 'Fake Lossless Detector job.' + '- Min confidence: minimum match confidence (0-1) to surface a finding\n' + '- Deep audio verify (default OFF): also run the ffmpeg decode guard ' + '(truncation + silence) per track — catches broken/incomplete files the ' + 'header hides, but decodes every file (seconds per track, CPU-heavy).\n\n' + 'Sibling job: "Quality Check (flag only)" finds the same below-profile tracks ' + 'but only flags them for you to decide per finding (re-download / delete / ' + 'ignore) instead of searching a replacement. Fake/transcoded lossless ' + 'detection is the separate Fake Lossless Detector job.' ) icon = 'repair-icon-lossy' default_enabled = False default_interval_hours = 168 - default_settings = {'scope': 'watchlist', 'min_confidence': 0.7} - setting_options = {'scope': ['watchlist', 'all']} + default_settings = {'scope': 'watchlist', 'min_confidence': 0.7, 'deep_audio_verify': False} + setting_options = {'scope': ['watchlist', 'all'], 'deep_audio_verify': [True, False]} auto_fix = False def _get_settings(self, context: JobContext) -> Dict[str, Any]: cfg = context.config_manager scope = 'watchlist' min_conf = 0.7 + deep_verify = False if cfg: scope = cfg.get(self.get_config_key('settings.scope'), 'watchlist') or 'watchlist' try: min_conf = float(cfg.get(self.get_config_key('settings.min_confidence'), 0.7)) except (TypeError, ValueError): min_conf = 0.7 - return {'scope': scope, 'min_confidence': min_conf} + deep_verify = cfg.get(self.get_config_key('settings.deep_audio_verify'), False) is True + return {'scope': scope, 'min_confidence': min_conf, 'deep_audio_verify': deep_verify} def _load_tracks(self, db: Any, scope: str) -> List[dict]: conn = db._get_connection() @@ -530,11 +548,18 @@ class QualityUpgradeJob(RepairJob): settings = self._get_settings(context) scope = settings['scope'] min_conf = settings['min_confidence'] + deep_verify = settings['deep_audio_verify'] + # v3 quality: judge the REAL file (mutagen-measured bit depth / sample rate + # / bitrate) against the profile's ranked targets — the SAME definition the + # download import guard uses. Strict: fallback is ignored (a download-time + # concession, not "good enough" for an upgrade). targets_from_profile / + # quality_meets_profile / probe_audio_quality are imported at module level. db = context.db quality_profile = db.get_quality_profile() - if preferred_quality_floor(quality_profile) is None: - logger.info("[Quality Upgrade] No quality buckets enabled in profile — nothing to flag") + targets, _fallback = targets_from_profile(quality_profile) + if not targets: + logger.info("[Quality Upgrade] No quality targets in profile — nothing to flag") return result try: @@ -583,13 +608,38 @@ class QualityUpgradeJob(RepairJob): result.findings_skipped_dedup += 1 continue - if meets_preferred_quality(file_path, bitrate, quality_profile): + # v3 quality decision — probe the REAL file. Resolve the library path + # first (the DB stores a possibly-relative path). + resolved_path = resolve_library_file_path(file_path) if file_path else None + if not resolved_path and file_path and os.path.isfile(file_path): + resolved_path = file_path + + measured_aq = probe_audio_quality(resolved_path) if resolved_path else None + + # Optional ffmpeg deep verify (default off): a truncated/silent file is + # treated as "needs a replacement" just like a below-profile one. + broken_reason = None + if deep_verify and resolved_path: + try: + from core.imports.silence import detect_broken_audio + broken_reason = detect_broken_audio(resolved_path) + except Exception as e: + logger.debug("[Quality Upgrade] deep verify failed for %s: %s", file_path, e) + + if measured_aq is None and not broken_reason: + # Can't read the file → can't judge it; leave it alone. result.skipped += 1 if context.update_progress and (i + 1) % 25 == 0: context.update_progress(i + 1, total) continue - # Below preferred quality — find a better version to propose. + if not broken_reason and measured_aq is not None and quality_meets_profile(measured_aq, targets): + result.skipped += 1 + if context.update_progress and (i + 1) % 25 == 0: + context.update_progress(i + 1, total) + continue + + # Below profile (or broken) — find a better version to propose. if engine is None: from core.matching_engine import MusicMatchingEngine engine = MusicMatchingEngine() @@ -602,12 +652,14 @@ class QualityUpgradeJob(RepairJob): logger.info("[Quality Upgrade] Spotify rate-limited — stopping scan early") return result - current_rank = classify_track_quality(file_path, bitrate) - current_label = _rank_label(current_rank) + current_label = measured_aq.label() if measured_aq is not None else 'broken/unreadable' + if broken_reason: + current_label = f'{current_label} (broken: {broken_reason})' if measured_aq is not None else f'broken ({broken_reason})' if context.report_progress: + _why = 'broken audio' if broken_reason else 'low quality' context.report_progress( scanned=i + 1, total=total, - log_line=f'Low quality ({current_label}): {artist_name} - {title}', + log_line=f'{_why} ({current_label}): {artist_name} - {title}', log_type='info') # Read the identifiers enrichment embedded in the file once (ISRC + diff --git a/core/repair_jobs/quality_upgrade_scanner.py b/core/repair_jobs/quality_upgrade_scanner.py index 8dee87ac..6df5f547 100644 --- a/core/repair_jobs/quality_upgrade_scanner.py +++ b/core/repair_jobs/quality_upgrade_scanner.py @@ -33,29 +33,28 @@ AUDIO_EXTENSIONS = {'.mp3', '.flac', '.ogg', '.opus', '.m4a', '.aac', '.wav', '. @register_job class QualityUpgradeScannerJob(RepairJob): job_id = 'quality_upgrade_scanner' - display_name = 'Quality Upgrade Scanner' - description = 'Flags library tracks below your quality profile' + display_name = 'Quality Check (flag only — you decide per finding)' + description = 'Flags library tracks below your quality profile; you choose re-download / delete / ignore per finding' help_text = ( - 'Scans your music library folder and verifies every track with the SAME ' - 'two-stage check the download/import pipeline runs:\n' - '1. Real-audio guard — ffmpeg actually decodes the file (truncation + ' + 'FLAG-ONLY quality job. Walks your music library folder on disk (so it also ' + 'catches loose files not in the DB) and checks every track against your v3 ' + 'quality profile — then just FLAGS what is below profile. Unlike the active ' + '"Quality Upgrade Finder", it does NOT search a replacement; you decide what ' + 'to do per finding: Re-download / Delete / Ignore.\n\n' + 'Two-stage check (same as the download/import pipeline):\n' + '1. Real-audio guard (optional, ffmpeg) — decodes the file (truncation + ' 'silence detection) to catch broken/incomplete audio the header hides.\n' '2. Quality gate — measured bit depth / sample rate / bitrate vs your ' - 'configured quality profile.\n' - 'A track flagged here is one the downloader would also reject. Because it ' - 'decodes each file, a full scan takes real time (seconds per track), not ' - 'milliseconds — that\'s the deep verification doing its job.\n\n' + 'profile targets.\n\n' 'Settings:\n' - '- deep_audio_verify (default on): run the ffmpeg decode guard. Turn off ' - 'for a fast header-only pass.\n' + '- Deep audio verify (default OFF): run the ffmpeg decode guard. Off = fast ' + 'header-only quality pass (milliseconds/track). On = full decode ' + '(seconds/track, CPU-heavy) but catches broken/silent audio.\n' '- library_tracks_only (default off): only check files matched to a ' - 'library DB track.\n\n' - 'Each below-profile track is reported as a finding. You can:\n' - '• Re-download — add the track to your wishlist and remove the low-quality file\n' - '• Delete — remove the low-quality file\n' - '• Ignore — dismiss the finding and keep the file\n\n' + 'library DB track (skip loose/orphan files).\n\n' 'The scan only reports — it never deletes or re-downloads on its own. ' - 'Profile targets and fallback come straight from Settings → Quality.' + 'Use the sibling "Quality Upgrade Finder" instead if you want it to actively ' + 'find and queue a better version for you.' ) icon = 'repair-icon-lossless' default_enabled = False @@ -65,7 +64,11 @@ class QualityUpgradeScannerJob(RepairJob): # audio file in the Music Library output folder, which is what users expect # ("check my library folder"). DB matching after a reset is unreliable and # would wrongly skip everything. Turn ON to ignore non-DB files. - default_settings = {'library_tracks_only': False, 'deep_audio_verify': True} + # + # deep_audio_verify default OFF: the ffmpeg decode is the CPU-heavy step. Most + # users want the fast header-only quality pass; turn it on for a deep scan that + # also catches broken/silent audio. (Matches the download pipeline's default.) + default_settings = {'library_tracks_only': False, 'deep_audio_verify': False} setting_options = {'library_tracks_only': [True, False], 'deep_audio_verify': [True, False]} auto_fix = False # User chooses fix action per finding @@ -140,10 +143,10 @@ class QualityUpgradeScannerJob(RepairJob): _settings = self._get_settings(context) library_only = _settings.get('library_tracks_only', False) # Deep verify = run the ffmpeg AudioGuard (real decode) per file, exactly - # like the download pipeline. Slower than a header read but it's the - # whole point: it verifies the REAL audio, not just the metadata. On by - # default; can be turned off for a fast header-only pass. - deep_verify = _settings.get('deep_audio_verify', True) + # like the download pipeline. Slower than a header read (seconds vs ms) but + # it verifies the REAL audio, not just the metadata. OFF by default (the + # decode is the CPU-heavy step); turn on for a deep scan. + deep_verify = _settings.get('deep_audio_verify', False) probe_failed = 0 not_in_library = 0 diff --git a/tests/repair_jobs/test_quality_upgrade.py b/tests/repair_jobs/test_quality_upgrade.py index d1ff35fa..9764c2af 100644 --- a/tests/repair_jobs/test_quality_upgrade.py +++ b/tests/repair_jobs/test_quality_upgrade.py @@ -149,7 +149,24 @@ def _row(track_id=1, title='Song One', path='/music/a.mp3', bitrate=128, duratio return (track_id, title, path, bitrate, duration, artist, album, album_id, track_number) +def _stub_quality(monkeypatch, *, meets: bool): + """Stub the v3 quality path so scan() works on fake (non-existent) file paths. + + The job now probes the REAL file (mutagen) and checks it against the v3 + ranked targets. Tests use fake paths, so we resolve the path to itself, + return a dummy measured quality, and force the meets/below verdict. + """ + from core.quality.model import AudioQuality + monkeypatch.setattr(qu, 'targets_from_profile', lambda profile: (['target'], False)) + monkeypatch.setattr(qu, 'resolve_library_file_path', lambda p: p) + monkeypatch.setattr(qu, 'probe_audio_quality', + lambda p: AudioQuality(format='mp3', bitrate=128)) + monkeypatch.setattr(qu, 'quality_meets_profile', lambda aq, targets: meets) + + def _stub_engine(monkeypatch): + # Below-profile by default — the finding-creating tests want a flagged track. + _stub_quality(monkeypatch, meets=False) monkeypatch.setattr(qu, 'get_primary_source', lambda: 'spotify') monkeypatch.setattr(qu, 'get_source_priority', lambda src: ['spotify']) monkeypatch.setattr( @@ -346,8 +363,9 @@ def test_find_track_in_album_exact_title_with_track_number(monkeypatch): def test_scan_skips_tracks_meeting_quality(monkeypatch): - # A 320 kbps MP3 meets the balanced profile → no finding, no metadata calls. + # A track that meets the profile → no finding, no metadata calls. db = _FakeDB([_row(track_id=2, title='Good Song', bitrate=320)], BALANCED) + _stub_quality(monkeypatch, meets=True) def _boom(*a, **k): # must never be called for an acceptable track raise AssertionError("matching should not run for an acceptable track") diff --git a/webui/static/enrichment.js b/webui/static/enrichment.js index 92a96598..e43cfe17 100644 --- a/webui/static/enrichment.js +++ b/webui/static/enrichment.js @@ -1698,6 +1698,12 @@ function switchRepairTab(tab) { // Turn a snake_case setting key into a human label. Handles acronym fix-ups // (EP, ID, URL, MB, AC, OS) that the naive Title-Case would otherwise botch. function _prettifyRepairSettingKey(key) { + // Full-key label overrides — for settings whose plain prettified name + // doesn't convey an important cost/behaviour (e.g. that it runs ffmpeg). + const fullKeyLabels = { + 'deep_audio_verify': 'Deep Audio Verify (ffmpeg decode — CPU heavy)', + }; + if (fullKeyLabels[key]) return fullKeyLabels[key]; const words = key.replace(/^_+/, '').split('_'); const acronyms = { 'eps': 'EPs', 'id': 'ID', 'url': 'URL', 'mb': 'MB', 'ac': 'AC', 'os': 'OS', 'api': 'API', 'mp3': 'MP3',