danvics/soulsync
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 1

fix(quality-scan): walk music folders on disk instead of resolving DB paths

Browse source 
The quality job kept resolving 0/N because DB-path resolution failed in the
deployed environment for reasons the logs wouldn't surface. Switch to the
mechanism the WORKING file tools use: os.walk the real music folders
(transfer + download + configured library paths, abspath'd) exactly like
orphan_file_detector and fake_lossless_detector — those reliably see files
because they never touch the DB's stored relative paths.

- Walk all existing music dirs, collect audio files (dedup by realpath),
  probe each with the same probe_audio_quality the import guard uses, check
  quality_meets_profile (strict). Below-profile files become findings.
- Match each walked file back to its DB track via a path-suffix index (last
  1-3 components) for real title/artist/album + track id; fall back to the
  file's own tags when no DB row matches (finding filed as 'file').
- Loud diagnostics: logs the folders walked and the audio-file count, and
  warns clearly when no music folder exists to walk.

The fix handler already works with the now-absolute file_path and an optional
entity_id (deletes the file by real path; DB row only when known).

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
dev 2026-06-15 15:39:35 +02:00
parent 8bb749de9c
commit 0e14ff03ee
1 changed files with 143 additions and 122 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

265
core/repair_jobs/quality_upgrade_scanner.py
View file

@ -1,19 +1,22 @@
"""Quality Upgrade Scanner Job — flags library tracks below the user's profile.
Scans the whole library (not just Transfer) by resolving each track's DB file
path to a real file on disk, probing its ACTUAL measured audio quality (bit
depth / sample rate / bitrate via mutagen — the SAME `probe_audio_quality` the
download import guard uses), and checking it against the user's v3 ranked
targets with `quality_meets_profile` (strict; fallback ignored — that's a
download-time concession, not a definition of "good enough").
Walks the music folders ON DISK (transfer + download + every configured library
path) exactly like the Orphan / Fake-Lossless detectors — those reliably "see"
files because they os.walk real directories instead of trying to resolve the
DB's stored (often relative) paths. For each audio file it probes the ACTUAL
measured audio quality (bit depth / sample rate / bitrate via the same
`probe_audio_quality` the download import guard uses) and checks it against the
user's v3 ranked targets with `quality_meets_profile` (strict — fallback
ignored, that's a download-time concession, not a definition of "good enough").
Every track that satisfies none of the targets becomes a finding the user can:
Every file that satisfies none of the targets becomes a finding the user can:
- 'redownload': add the track to the wishlist and delete the low-quality file
- 'delete': remove the low-quality file and its DB record
- 'delete': remove the low-quality file (+ DB row when known)
- 'ignore': dismiss the finding (handled in the UI via the dismiss endpoint)
This unifies the library quality check onto the exact same pipeline as the
download quality gate — no more extension-only tier guessing.
Each walked file is matched back to its DB track (by path suffix) so the finding
carries the real title/artist/album + track id; when no DB row matches, the
file's own tags are used and the finding is filed as a loose 'file'.
"""
import os
@ -24,6 +27,8 @@ from utils.logging_config import get_logger
logger = get_logger("repair_job.quality_upgrade")
AUDIO_EXTENSIONS = {'.mp3', '.flac', '.ogg', '.opus', '.m4a', '.aac', '.wav', '.wma', '.aiff', '.aif'}
@register_job
class QualityUpgradeScannerJob(RepairJob):
@ -31,18 +36,17 @@ class QualityUpgradeScannerJob(RepairJob):
display_name = 'Quality Upgrade Scanner'
description = 'Flags library tracks below your quality profile'
help_text = (
'Scans your music library and checks every track\'s REAL audio quality '
'(bit depth, sample rate, bitrate — read from the file itself, not just '
'the extension) against your configured quality profile. This is the same '
'check the download pipeline runs, so a track flagged here is one the '
'downloader would also reject.\n\n'
'Scans your music folders on disk and checks every track\'s REAL audio '
'quality (bit depth, sample rate, bitrate — read from the file itself, '
'not just the extension) against your configured quality profile. This is '
'the same check the download pipeline runs, so a track flagged here is one '
'the downloader would also reject.\n\n'
'Each below-profile track is reported as a finding. You can:\n'
'• Re-download — add the track to your wishlist and remove the low-quality file\n'
'• Delete — remove the low-quality file entirely\n'
'• Delete — remove the low-quality file\n'
'• Ignore — dismiss the finding and keep the file\n\n'
'The scan only reports — it never deletes or re-downloads on its own. '
'Profile targets and fallback come straight from Settings → Quality, so '
'adjusting your profile changes what counts as "below quality" here.'
'Profile targets and fallback come straight from Settings → Quality.'
)
icon = 'repair-icon-lossless'
default_enabled = False
@ -72,41 +76,52 @@ class QualityUpgradeScannerJob(RepairJob):
from core.imports.file_ops import probe_audio_quality
db_tracks = self._load_db_tracks(context)
if not db_tracks:
logger.info("No library tracks with file paths found")
# --- Collect the music folders to walk (real dirs, abspath'd) ---
base_dirs = self._collect_music_dirs(context)
if not base_dirs:
logger.warning(
"[QualityScan] No existing music folder to walk (transfer=%r, cwd=%r). "
"Set soulseek.transfer_path to the real mount or add your library under "
"Settings → Library → Music Paths.",
context.transfer_folder, os.getcwd())
return result
logger.info("[QualityScan] Walking %d folder(s): %r", len(base_dirs), base_dirs)
track_list = sorted(db_tracks.items(), key=lambda x: str(x[0]))
total = len(track_list)
# --- Gather audio files (dedup by real path) ---
audio_files = []
seen = set()
for base in base_dirs:
for root, _dirs, files in os.walk(base):
if context.check_stop():
return result
for fname in files:
if os.path.splitext(fname)[1].lower() in AUDIO_EXTENSIONS:
fpath = os.path.join(root, fname)
rp = os.path.realpath(fpath)
if rp in seen:
continue
seen.add(rp)
audio_files.append(fpath)
total = len(audio_files)
logger.info("[QualityScan] Found %d audio file(s) to check", total)
if context.report_progress:
context.report_progress(phase=f'Scanning {total} library tracks...', total=total)
context.report_progress(phase=f'Checking {total} files...', total=total)
if context.update_progress:
context.update_progress(0, total)
# --- DB suffix index so a walked file maps back to its track row ---
db_index = self._build_db_suffix_index(context)
probe_failed = 0
_diag_logged = False
for i, (track_id, info) in enumerate(track_list):
for i, fpath in enumerate(audio_files):
if context.check_stop():
return result
if i % 20 == 0 and context.wait_if_paused():
return result
raw_fp = info.get('file_path', '')
resolved = self._resolve_path(raw_fp, context)
if not resolved:
# One-shot diagnostic on the first unresolved track — logs EXACTLY
# what the resolver tried (base dirs, cwd) so a path/mount mismatch
# is diagnosable instead of a silent "all skipped".
if not _diag_logged:
_diag_logged = True
self._log_resolve_diag(raw_fp, context)
result.skipped += 1
probe_failed += 1
continue
result.scanned += 1
fname = os.path.basename(resolved)
fname = os.path.basename(fpath)
if context.report_progress and i % 25 == 0:
context.report_progress(
scanned=i + 1, total=total,
@ -116,29 +131,30 @@ class QualityUpgradeScannerJob(RepairJob):
)
try:
aq = probe_audio_quality(resolved)
aq = probe_audio_quality(fpath)
except Exception as e:
logger.debug("Probe failed for %s: %s", fname, e)
aq = None
if aq is None:
# File couldn't be read — can't judge quality, leave unflagged.
probe_failed += 1
result.skipped += 1
continue
# Strict profile check — identical to the download guard.
if quality_meets_profile(aq, targets):
if context.update_progress and (i + 1) % 25 == 0:
context.update_progress(i + 1, total)
continue
# Below profile → create a finding.
# Below profile → resolve metadata (DB row preferred, file tags fallback).
meta = self._lookup_meta(fpath, db_index)
current_label = aq.label()
target_labels = [t.label for t in targets]
disp_title = meta.get('title') or os.path.splitext(fname)[0]
disp_artist = meta.get('artist') or 'Unknown'
if context.report_progress:
context.report_progress(
log_line=f'Below quality: {fname} — {current_label}',
log_line=f'Below quality: {disp_title} — {current_label}',
log_type='error',
)
if context.create_finding:
@ -146,13 +162,12 @@ class QualityUpgradeScannerJob(RepairJob):
job_id=self.job_id,
finding_type='quality_upgrade',
severity='info',
entity_type='track',
entity_id=str(track_id),
file_path=resolved,
title=f'Below quality: {info.get("title") or fname} ({current_label})',
entity_type='track' if meta.get('track_id') else 'file',
entity_id=str(meta['track_id']) if meta.get('track_id') else None,
file_path=fpath,
title=f'Below quality: {disp_title} ({current_label})',
description=(
f'"{info.get("title") or fname}" by '
f'{info.get("artist") or "Unknown"} is {current_label}, '
f'"{disp_title}" by {disp_artist} is {current_label}, '
f'which does not meet your quality profile '
f'({", ".join(target_labels[:3])}'
f'{"…" if len(target_labels) > 3 else ""}).'
@ -164,12 +179,12 @@ class QualityUpgradeScannerJob(RepairJob):
'current_sample_rate': aq.sample_rate,
'current_bit_depth': aq.bit_depth,
'target_qualities': target_labels,
'expected_title': info.get('title', ''),
'expected_artist': info.get('artist', ''),
'album_title': info.get('album_title', ''),
'track_number': info.get('track_number'),
'album_thumb_url': info.get('album_thumb_url'),
'artist_thumb_url': info.get('artist_thumb_url'),
'expected_title': disp_title,
'expected_artist': disp_artist,
'album_title': meta.get('album', ''),
'track_number': meta.get('track_number'),
'album_thumb_url': meta.get('album_thumb_url'),
'artist_thumb_url': meta.get('artist_thumb_url'),
},
)
if inserted:
@ -184,17 +199,42 @@ class QualityUpgradeScannerJob(RepairJob):
context.update_progress(total, total)
if probe_failed:
logger.warning(
"Quality upgrade scan: %d/%d tracks could not be read/resolved and "
"were left unflagged (their quality couldn't be verified).",
probe_failed, total)
logger.info("Quality upgrade scan: %d scanned, %d below profile, %d skipped",
logger.warning("[QualityScan] %d/%d files could not be probed (unreadable)",
probe_failed, total)
logger.info("Quality upgrade scan: %d checked, %d below profile, %d skipped",
result.scanned, result.findings_created, result.skipped)
return result
def _load_db_tracks(self, context: JobContext) -> dict:
"""Load all library tracks keyed by ID (mirrors AcoustIDScannerJob)."""
tracks = {}
def _collect_music_dirs(self, context: JobContext) -> list:
"""All existing music directories to walk, as absolute paths (dedup)."""
cm = context.config_manager
raw = [context.transfer_folder]
if cm:
try:
raw.append(cm.get('soulseek.transfer_path', './Transfer'))
raw.append(cm.get('soulseek.download_path', './downloads'))
mp = cm.get('library.music_paths', []) or []
if isinstance(mp, list):
raw.extend([p for p in mp if isinstance(p, str) and p.strip()])
except Exception as e:
logger.debug("music dir config read failed: %s", e)
out, seen = [], set()
for d in raw:
if not d:
continue
ad = os.path.abspath(d)
if ad in seen:
continue
seen.add(ad)
if os.path.isdir(ad):
out.append(ad)
return out
def _build_db_suffix_index(self, context: JobContext) -> dict:
"""Map normalized path suffixes (last 1-3 components, lowercased) →
track metadata, so a walked absolute file can be matched to its DB row
even when the DB stores a different (relative) path prefix."""
index = {}
conn = None
try:
conn = context.db._get_connection()
@ -210,75 +250,56 @@ class QualityUpgradeScannerJob(RepairJob):
WHERE t.file_path IS NOT NULL AND t.file_path != ''
""")
for row in cursor.fetchall():
track_id = row[0]
if track_id is None:
fp = (row[3] or '').replace('\\', '/')
if not fp:
continue
tracks[str(track_id)] = {
parts = fp.split('/')
meta = {
'track_id': row[0],
'title': row[1] or '',
'artist': row[2] or '',
'file_path': row[3] or '',
'track_number': row[4],
'album_title': row[5] or '',
'album': row[5] or '',
'album_thumb_url': row[6] or None,
'artist_thumb_url': row[7] or None,
}
for depth in range(1, min(4, len(parts) + 1)):
suffix = '/'.join(parts[-depth:]).lower()
index.setdefault(suffix, meta)
except Exception as e:
logger.error("Error loading tracks from DB: %s", e)
logger.error("Error building DB suffix index: %s", e)
finally:
if conn:
conn.close()
return tracks
return index
def _resolve_path(self, file_path, context):
"""Resolve a DB file path to an actual file on disk via the shared
resolver (picks up library.music_paths + Plex locations too)."""
if not file_path:
return None
if os.path.exists(file_path):
return file_path
from core.library.path_resolver import resolve_library_file_path
return resolve_library_file_path(
file_path,
transfer_folder=context.transfer_folder,
config_manager=context.config_manager,
)
def _log_resolve_diag(self, file_path, context):
"""Log a detailed diagnostic for the first track whose path can't be
resolved — the only reliable way to tell apart a CWD problem, a wrong
transfer mount, or genuinely-missing files in this container."""
from core.library.path_resolver import resolve_library_file_path_with_diagnostic
def _lookup_meta(self, fpath: str, db_index: dict) -> dict:
"""Match a walked file to a DB track via path suffix; fall back to the
file's own tags for title/artist/album when nothing matches."""
parts = fpath.replace('\\', '/').split('/')
for depth in range(min(3, len(parts)), 0, -1):
suffix = '/'.join(parts[-depth:]).lower()
hit = db_index.get(suffix)
if hit:
return hit
# No DB row — read the file's own tags.
meta = {'track_id': None}
try:
_, attempt = resolve_library_file_path_with_diagnostic(
file_path,
transfer_folder=context.transfer_folder,
config_manager=context.config_manager,
)
tf = context.transfer_folder
abs_tf = os.path.abspath(tf) if tf else None
logger.warning(
"[QualityResolve] unresolved db_path=%r | cwd=%r | transfer_folder=%r "
"(abspath=%r, isdir=%s) | base_dirs_tried=%r | abs_join_exists=%s",
file_path, os.getcwd(), tf, abs_tf,
os.path.isdir(abs_tf) if abs_tf else None,
attempt.base_dirs_tried,
os.path.exists(os.path.join(abs_tf, file_path)) if abs_tf else None,
)
from mutagen import File as MutagenFile
audio = MutagenFile(fpath, easy=True)
if audio:
meta['title'] = (audio.get('title') or [None])[0] or ''
meta['artist'] = (audio.get('artist') or audio.get('albumartist') or [None])[0] or ''
meta['album'] = (audio.get('album') or [None])[0] or ''
except Exception as e:
logger.warning("[QualityResolve] diagnostic failed: %s", e)
logger.debug("tag read failed for %s: %s", os.path.basename(fpath), e)
return meta
def estimate_scope(self, context: JobContext) -> int:
conn = None
try:
conn = context.db._get_connection()
cursor = conn.cursor()
cursor.execute("""
SELECT COUNT(*) FROM tracks
WHERE file_path IS NOT NULL AND file_path != ''
""")
return cursor.fetchone()[0]
except Exception:
return 0
finally:
if conn:
conn.close()
count = 0
for base in self._collect_music_dirs(context):
for _root, _dirs, files in os.walk(base):
for fname in files:
if os.path.splitext(fname)[1].lower() in AUDIO_EXTENSIONS:
count += 1
return count