update soulsync repair worker with new jobs

This commit is contained in:
Broque Thomas 2026-03-15 10:18:50 -07:00
parent 6fa3b490ea
commit dab94ce65d
8 changed files with 1034 additions and 7 deletions

View file

@ -36,6 +36,7 @@ _JOB_MODULES = [
'core.repair_jobs.metadata_gap_filler',
'core.repair_jobs.album_completeness',
'core.repair_jobs.fake_lossless_detector',
'core.repair_jobs.library_reorganize',
]

View file

@ -35,6 +35,7 @@ class JobContext:
should_stop: Optional[Callable[[], bool]] = None
is_paused: Optional[Callable[[], bool]] = None
update_progress: Optional[Callable[[int, int], None]] = None
report_progress: Optional[Callable] = None # Rich progress: (phase, log_line, log_type, scanned, total)
def check_stop(self) -> bool:
"""Return True if the worker should stop."""

View file

@ -0,0 +1,572 @@
"""Library Reorganize Job — moves files to match the current file organization template.
Safety design:
- Dry run mode is ON by default. The job only creates findings (reports) showing
what WOULD move. The user must explicitly disable dry_run in job settings.
- The job is disabled by default (default_enabled=False) so it never runs
automatically unless the user explicitly enables it.
- Case-insensitive path comparison on Windows prevents false moves.
- Destination collision check prevents overwriting existing files.
- Files without usable tags are skipped, not guessed at.
- Moves are always within the transfer folder; cannot escape to parent dirs.
"""
import os
import re
import shutil
import sys
import time
from core.repair_jobs import register_job
from core.repair_jobs.base import JobContext, JobResult, RepairJob
from utils.logging_config import get_logger
logger = get_logger("repair_job.library_reorganize")
AUDIO_EXTENSIONS = {'.mp3', '.flac', '.ogg', '.opus', '.m4a', '.aac', '.wav', '.wma', '.aiff', '.aif'}
SIDECAR_EXTENSIONS = {'.lrc', '.jpg', '.jpeg', '.png', '.nfo', '.txt', '.cue'}
# Windows and macOS use case-insensitive filesystems by default
_CASE_INSENSITIVE = sys.platform in ('win32', 'darwin')
def _paths_equivalent(path_a: str, path_b: str) -> bool:
"""Compare two paths, case-insensitive on Windows/macOS."""
a = os.path.normpath(path_a)
b = os.path.normpath(path_b)
if _CASE_INSENSITIVE:
return a.lower() == b.lower()
return a == b
def _sanitize_filename(filename: str) -> str:
"""Sanitize filename for file system compatibility."""
sanitized = re.sub(r'[<>:"/\\|?*]', '_', filename)
sanitized = re.sub(r'\s+', ' ', sanitized).strip()
sanitized = sanitized.rstrip('. ') or '_'
if re.match(r'^(CON|PRN|AUX|NUL|COM[1-9]|LPT[1-9])(\.|$)', sanitized, re.IGNORECASE):
sanitized = '_' + sanitized
return sanitized[:200]
def _sanitize_context_values(context: dict) -> dict:
"""Sanitize all string values for path safety."""
sanitized = {}
for key, value in context.items():
if isinstance(value, str):
sanitized[key] = _sanitize_filename(value)
else:
sanitized[key] = value
return sanitized
def _apply_path_template(template: str, context: dict) -> str:
"""Apply template variables to build a path string."""
clean = _sanitize_context_values(context)
result = template
result = result.replace('$albumartist', clean.get('albumartist', clean.get('artist', 'Unknown Artist')))
result = result.replace('$albumtype', clean.get('albumtype', 'Album'))
result = result.replace('$playlist', clean.get('playlist_name', ''))
result = result.replace('$artistletter', (clean.get('artist', 'U') or 'U')[0].upper())
result = result.replace('$artist', clean.get('artist', 'Unknown Artist'))
result = result.replace('$album', clean.get('album', 'Unknown Album'))
result = result.replace('$title', clean.get('title', 'Unknown Track'))
result = result.replace('$track', f"{clean.get('track_number', 1):02d}")
result = result.replace('$year', str(clean.get('year', '')))
result = re.sub(r'\s+', ' ', result)
result = re.sub(r'\s*-\s*-\s*', ' - ', result)
return result.strip()
def _build_path_from_template(template: str, context: dict) -> tuple:
"""Build (folder_path, filename_base) from a template string and context."""
full_path = _apply_path_template(template, context)
quality_value = context.get('quality', '')
disc_value = f"{context.get('disc_number', 1):02d}"
path_parts = full_path.split('/')
if len(path_parts) > 1:
folder_parts = path_parts[:-1]
filename_base = path_parts[-1]
cleaned_folders = []
for part in folder_parts:
part = part.replace('$quality', '')
part = part.replace('$disc', '')
part = re.sub(r'\s*\[\s*\]', '', part)
part = re.sub(r'\s*\(\s*\)', '', part)
part = re.sub(r'\s*\{\s*\}', '', part)
part = re.sub(r'\s*-\s*$', '', part)
part = re.sub(r'^\s*-\s*', '', part)
part = re.sub(r'\s+', ' ', part).strip()
if part:
cleaned_folders.append(part)
filename_base = filename_base.replace('$quality', quality_value)
filename_base = filename_base.replace('$disc', disc_value)
filename_base = re.sub(r'\s*\[\s*\]', '', filename_base)
filename_base = re.sub(r'\s*\(\s*\)', '', filename_base)
filename_base = re.sub(r'\s*\{\s*\}', '', filename_base)
filename_base = re.sub(r'\s*-\s*$', '', filename_base)
filename_base = re.sub(r'\s+', ' ', filename_base).strip()
sanitized_folders = [_sanitize_filename(p) for p in cleaned_folders]
folder_path = os.path.join(*sanitized_folders) if sanitized_folders else ''
return folder_path, _sanitize_filename(filename_base)
else:
full_path = full_path.replace('$quality', quality_value)
full_path = full_path.replace('$disc', disc_value)
full_path = re.sub(r'\s*\[\s*\]', '', full_path)
full_path = re.sub(r'\s*\(\s*\)', '', full_path)
full_path = re.sub(r'\s*\{\s*\}', '', full_path)
full_path = re.sub(r'\s*-\s*$', '', full_path)
full_path = re.sub(r'\s+', ' ', full_path).strip()
return '', _sanitize_filename(full_path)
def _get_audio_quality(file_path: str) -> str:
"""Read audio file and return a quality descriptor string."""
try:
ext = os.path.splitext(file_path)[1].lower()
if ext == '.flac':
from mutagen.flac import FLAC
audio = FLAC(file_path)
bits = audio.info.bits_per_sample
return f"FLAC {bits}bit"
elif ext == '.mp3':
from mutagen.mp3 import MP3, BitrateMode
audio = MP3(file_path)
kbps = audio.info.bitrate // 1000
if audio.info.bitrate_mode == BitrateMode.VBR:
return "MP3-VBR"
return f"MP3-{kbps}"
elif ext in ('.m4a', '.aac', '.mp4'):
from mutagen.mp4 import MP4
audio = MP4(file_path)
kbps = audio.info.bitrate // 1000
return f"M4A-{kbps}"
elif ext == '.ogg':
from mutagen.oggvorbis import OggVorbis
audio = OggVorbis(file_path)
kbps = audio.info.bitrate // 1000
return f"OGG-{kbps}"
elif ext == '.opus':
from mutagen.oggopus import OggOpus
audio = OggOpus(file_path)
kbps = audio.info.bitrate // 1000
return f"OPUS-{kbps}"
return ''
except Exception:
return ''
def _read_tag_metadata(file_path: str) -> dict:
"""Read artist, album, title, track_number, disc_number, year from file tags."""
try:
from mutagen import File as MutagenFile
audio = MutagenFile(file_path, easy=True)
if audio is None:
return {}
def first(tag_list):
if isinstance(tag_list, list) and tag_list:
return str(tag_list[0])
if isinstance(tag_list, str):
return tag_list
return ''
meta = {}
meta['artist'] = first(audio.get('artist', ['']))
meta['albumartist'] = first(audio.get('albumartist', [''])) or meta['artist']
meta['album'] = first(audio.get('album', ['']))
meta['title'] = first(audio.get('title', ['']))
# Track number: may be "3/12" format
raw_track = first(audio.get('tracknumber', ['1']))
try:
meta['track_number'] = int(raw_track.split('/')[0])
except (ValueError, IndexError):
meta['track_number'] = 1
# Disc number: may be "1/2" format
raw_disc = first(audio.get('discnumber', ['1']))
try:
meta['disc_number'] = int(raw_disc.split('/')[0])
except (ValueError, IndexError):
meta['disc_number'] = 1
# Year
raw_date = first(audio.get('date', ['']))
meta['year'] = raw_date[:4] if raw_date and len(raw_date) >= 4 else ''
return meta
except Exception as e:
logger.debug("Failed to read tags from %s: %s", file_path, e)
return {}
def _remove_empty_dirs(directory: str, root: str):
"""Remove empty directories up to root. Never removes root itself."""
directory = os.path.normpath(directory)
root = os.path.normpath(root)
while directory != root and len(directory) > len(root) and directory.startswith(root):
try:
if os.path.isdir(directory) and not os.listdir(directory):
os.rmdir(directory)
directory = os.path.dirname(directory)
else:
break
except OSError:
break
@register_job
class LibraryReorganizeJob(RepairJob):
job_id = 'library_reorganize'
display_name = 'Library Reorganize'
description = 'Moves files to match the current file organization template (dry run by default)'
icon = 'repair-icon-reorganize'
default_enabled = False
default_interval_hours = 168 # Weekly — but disabled by default so won't auto-run
default_settings = {
'dry_run': True,
'move_sidecars': True,
}
auto_fix = True
def scan(self, context: JobContext) -> JobResult:
result = JobResult()
transfer = context.transfer_folder
if not os.path.isdir(transfer):
logger.warning("Transfer folder does not exist: %s", transfer)
return result
# Get template config
cm = context.config_manager
if not cm:
logger.error("No config manager available")
return result
if not cm.get('file_organization.enabled', True):
logger.info("File organization is disabled — skipping reorganize")
if context.report_progress:
context.report_progress(phase='Skipped — file organization disabled',
log_line='File organization is disabled in settings',
log_type='skip')
return result
templates = cm.get('file_organization.templates', {})
album_template = templates.get('album_path', '$albumartist/$albumartist - $album/$track - $title')
single_template = templates.get('single_path', '$artist/$artist - $title/$title')
disc_label = cm.get('file_organization.disc_label', 'Disc')
dry_run = self._get_setting(context, 'dry_run', True)
move_sidecars = self._get_setting(context, 'move_sidecars', True)
if context.report_progress:
mode_label = 'DRY RUN' if dry_run else 'LIVE'
context.report_progress(phase=f'Scanning files ({mode_label})...',
log_line=f'Mode: {mode_label} — Scanning {transfer}',
log_type='info')
# Collect all audio files
audio_files = []
for root_dir, _dirs, files in os.walk(transfer):
if context.check_stop():
return result
for fname in files:
ext = os.path.splitext(fname)[1].lower()
if ext in AUDIO_EXTENSIONS:
audio_files.append(os.path.join(root_dir, fname))
total = len(audio_files)
if total == 0:
logger.info("No audio files found in transfer folder")
if context.report_progress:
context.report_progress(phase='No files found', log_line='No audio files in transfer folder',
log_type='info')
return result
if context.report_progress:
context.report_progress(phase=f'Reading tags from {total} files...',
log_line=f'Found {total} audio files',
log_type='info', scanned=0, total=total)
# Pre-read all tags and group by album for multi-disc detection
file_tags = {} # fpath -> tags dict
album_groups = {} # (albumartist, album) -> [fpath, ...]
for fpath in audio_files:
tags = _read_tag_metadata(fpath)
file_tags[fpath] = tags
key = (tags.get('albumartist', '') or tags.get('artist', ''),
tags.get('album', ''))
if key not in album_groups:
album_groups[key] = []
album_groups[key].append(fpath)
# Compute total_discs per album group
album_total_discs = {}
for key, fpaths in album_groups.items():
max_disc = max((file_tags[fp].get('disc_number', 1) for fp in fpaths), default=1)
album_total_discs[key] = max_disc
# Track claimed destinations to detect in-batch collisions
claimed_destinations = set()
for i, fpath in enumerate(audio_files):
if context.check_stop():
return result
if i % 50 == 0 and context.wait_if_paused():
return result
result.scanned += 1
fname = os.path.basename(fpath)
file_ext = os.path.splitext(fname)[1]
tags = file_tags.get(fpath, {})
# Skip files without minimum usable tags
title = tags.get('title', '') or ''
artist = tags.get('artist', '') or ''
if not title and not artist:
result.skipped += 1
if context.report_progress and i % 20 == 0:
context.report_progress(scanned=i + 1, total=total,
phase=f'Processing ({i+1}/{total})...')
continue
# Use defaults only when tags exist but are empty
artist = artist or 'Unknown Artist'
albumartist = tags.get('albumartist', '') or artist
album = tags.get('album', '') or ''
title = title or 'Unknown Track'
track_number = tags.get('track_number', 1) or 1
disc_number = tags.get('disc_number', 1) or 1
year = tags.get('year', '')
# Read quality for $quality template variable
quality = _get_audio_quality(fpath)
# Determine template type: album or single
album_key = (albumartist, album)
group_size = len(album_groups.get(album_key, []))
is_album = bool(album) and group_size > 1
total_discs = album_total_discs.get(album_key, 1)
template_context = {
'artist': artist,
'albumartist': albumartist,
'album': album or title,
'title': title,
'track_number': track_number,
'disc_number': disc_number,
'year': year,
'quality': quality,
'albumtype': 'Album',
}
if is_album:
template = album_template
user_controls_disc = '$disc' in template
folder_path, filename_base = _build_path_from_template(template, template_context)
if folder_path and filename_base:
if total_discs > 1 and not user_controls_disc:
disc_folder = f"{disc_label} {disc_number}"
expected = os.path.join(transfer, folder_path, disc_folder, filename_base + file_ext)
else:
expected = os.path.join(transfer, folder_path, filename_base + file_ext)
else:
result.skipped += 1
continue
else:
template = single_template
folder_path, filename_base = _build_path_from_template(template, template_context)
if folder_path and filename_base:
expected = os.path.join(transfer, folder_path, filename_base + file_ext)
else:
result.skipped += 1
continue
# Safety: verify destination is still inside transfer folder
expected_norm = os.path.normpath(expected)
transfer_norm = os.path.normpath(transfer)
if not expected_norm.startswith(transfer_norm + os.sep) and expected_norm != transfer_norm:
logger.warning("Computed path escapes transfer folder, skipping: %s", expected_norm)
result.skipped += 1
continue
actual_norm = os.path.normpath(fpath)
# Case-insensitive comparison on Windows/macOS
if _paths_equivalent(actual_norm, expected_norm):
if context.report_progress and i % 20 == 0:
context.report_progress(scanned=i + 1, total=total,
phase=f'Processing ({i+1}/{total})...')
continue
# Check for in-batch destination collision
dest_key = expected_norm.lower() if _CASE_INSENSITIVE else expected_norm
if dest_key in claimed_destinations:
result.skipped += 1
if context.report_progress:
context.report_progress(
scanned=i + 1, total=total,
log_line=f'SKIP (duplicate dest): {os.path.basename(fpath)}',
log_type='skip'
)
continue
claimed_destinations.add(dest_key)
# File needs to move
if dry_run:
rel_actual = os.path.relpath(actual_norm, transfer)
rel_expected = os.path.relpath(expected_norm, transfer)
if context.create_finding:
context.create_finding(
job_id=self.job_id,
finding_type='path_mismatch',
severity='info',
entity_type='file',
entity_id=None,
file_path=fpath,
title=f'Would move: {os.path.basename(fpath)}',
description=f'From: {rel_actual}\nTo: {rel_expected}',
details={'from': rel_actual, 'to': rel_expected}
)
result.findings_created += 1
if context.report_progress:
context.report_progress(
scanned=i + 1, total=total,
phase=f'Dry run ({i+1}/{total})...',
log_line=f'[DRY] {os.path.basename(fpath)} -> {os.path.relpath(expected_norm, transfer)}',
log_type='info'
)
else:
# Actually move the file
try:
dest_dir = os.path.dirname(expected_norm)
os.makedirs(dest_dir, exist_ok=True)
# Collision: skip if destination already exists and is a different file
if os.path.exists(expected_norm):
# On case-insensitive FS, check if it's the same file (case rename)
try:
same_file = os.path.samefile(actual_norm, expected_norm)
except (OSError, ValueError):
same_file = False
if not same_file:
result.skipped += 1
if context.report_progress:
context.report_progress(
scanned=i + 1, total=total,
log_line=f'SKIP (exists): {os.path.basename(fpath)}',
log_type='skip'
)
continue
# Same file, different case — use two-step rename to avoid
# OS refusing rename to "same" path on case-insensitive FS
if _CASE_INSENSITIVE:
tmp_path = expected_norm + '.tmp_rename'
shutil.move(actual_norm, tmp_path)
shutil.move(tmp_path, expected_norm)
else:
shutil.move(actual_norm, expected_norm)
else:
shutil.move(actual_norm, expected_norm)
result.auto_fixed += 1
# Move sidecar files (LRC, cover art, etc.)
if move_sidecars:
stem = os.path.splitext(os.path.basename(actual_norm))[0]
src_dir = os.path.dirname(actual_norm)
for sidecar_ext in SIDECAR_EXTENSIONS:
sidecar_src = os.path.join(src_dir, stem + sidecar_ext)
if os.path.isfile(sidecar_src):
new_stem = os.path.splitext(os.path.basename(expected_norm))[0]
sidecar_dst = os.path.join(dest_dir, new_stem + sidecar_ext)
try:
shutil.move(sidecar_src, sidecar_dst)
except Exception as se:
logger.debug("Failed to move sidecar %s: %s", sidecar_src, se)
# Update DB file_path if there's a matching track
self._update_db_path(context.db, actual_norm, expected_norm)
# Clean up empty source directories
_remove_empty_dirs(os.path.dirname(actual_norm), transfer)
if context.report_progress:
context.report_progress(
scanned=i + 1, total=total,
phase=f'Moving ({i+1}/{total})...',
log_line=f'Moved: {os.path.basename(fpath)}',
log_type='success'
)
except Exception as e:
logger.error("Failed to move %s -> %s: %s", fpath, expected_norm, e)
result.errors += 1
if context.report_progress:
context.report_progress(
scanned=i + 1, total=total,
log_line=f'ERROR: {os.path.basename(fpath)} -- {e}',
log_type='error'
)
if context.update_progress and (i + 1) % 10 == 0:
context.update_progress(i + 1, total)
if context.update_progress:
context.update_progress(total, total)
mode_text = 'Dry run' if dry_run else 'Reorganize'
summary = f"{mode_text} complete: {result.scanned} scanned, {result.auto_fixed} moved, {result.findings_created} findings, {result.skipped} skipped, {result.errors} errors"
logger.info(summary)
if context.report_progress:
context.report_progress(
phase='Complete',
log_line=summary,
log_type='success',
scanned=total, total=total
)
return result
def estimate_scope(self, context: JobContext) -> int:
transfer = context.transfer_folder
if not os.path.isdir(transfer):
return 0
count = 0
for _root, _dirs, files in os.walk(transfer):
for fname in files:
if os.path.splitext(fname)[1].lower() in AUDIO_EXTENSIONS:
count += 1
return count
def _get_setting(self, context: JobContext, key: str, default):
"""Read a job-specific setting from config."""
if context.config_manager:
return context.config_manager.get(f'repair.jobs.{self.job_id}.settings.{key}', default)
return default
def _update_db_path(self, db, old_path: str, new_path: str):
"""Update file_path in the tracks table when a file is moved."""
conn = None
try:
conn = db._get_connection()
cursor = conn.cursor()
# Try exact match first
cursor.execute("UPDATE tracks SET file_path = ? WHERE file_path = ?",
(new_path, old_path))
if cursor.rowcount == 0:
# Try normalized path match
cursor.execute("UPDATE tracks SET file_path = ? WHERE file_path = ?",
(new_path, os.path.normpath(old_path)))
conn.commit()
except Exception as e:
logger.debug("DB path update failed for %s: %s", old_path, e)
finally:
if conn:
conn.close()

View file

@ -29,13 +29,14 @@ _PLACEHOLDER_IDS = {
class TrackNumberRepairJob(RepairJob):
job_id = 'track_number_repair'
display_name = 'Track Number Repair'
description = 'Fixes mismatched track numbers using API lookups'
description = 'Detects mismatched track numbers using API lookups (dry run by default)'
icon = 'repair-icon-tracknumber'
default_enabled = True
default_interval_hours = 24
default_settings = {
'anomaly_threshold': 3,
'title_similarity': 0.80,
'dry_run': True,
}
auto_fix = True
@ -44,11 +45,13 @@ class TrackNumberRepairJob(RepairJob):
settings = self._get_settings(context)
anomaly_threshold = settings.get('anomaly_threshold', 3)
title_similarity = settings.get('title_similarity', 0.80)
dry_run = settings.get('dry_run', True)
# Thread-local state to avoid race conditions with concurrent scan_folders()
scan_state = {
'album_tracks_cache': {},
'title_similarity': title_similarity,
'dry_run': dry_run,
}
transfer = context.transfer_folder
@ -162,14 +165,32 @@ class TrackNumberRepairJob(RepairJob):
# Process each file
title_sim = scan_state.get('title_similarity', 0.80)
dry_run = scan_state.get('dry_run', True)
for fpath, fname, _ in file_track_data:
if context.check_stop():
return result
result.scanned += 1
try:
if _repair_single_track(fpath, fname, api_tracks, len(api_tracks), title_sim, context):
result.auto_fixed += 1
if dry_run:
finding = _check_single_track(fpath, fname, api_tracks, len(api_tracks), title_sim)
if finding:
if context.create_finding:
context.create_finding(
job_id=self.job_id,
finding_type='track_number_mismatch',
severity='warning',
entity_type='file',
entity_id=None,
file_path=fpath,
title=f'Track number fix: {os.path.basename(fpath)}',
description=finding['description'],
details=finding['details']
)
result.findings_created += 1
else:
if _repair_single_track(fpath, fname, api_tracks, len(api_tracks), title_sim, context):
result.auto_fixed += 1
except Exception as e:
logger.error("Error repairing %s: %s", fpath, e, exc_info=True)
result.errors += 1
@ -309,6 +330,7 @@ class TrackNumberRepairJob(RepairJob):
scan_state = {
'album_tracks_cache': {},
'title_similarity': settings.get('title_similarity', 0.80),
'dry_run': settings.get('dry_run', True),
}
for folder_path in folders:
@ -770,6 +792,65 @@ def _lookup_album_ids_from_db(file_track_data: List[Tuple[str, str, Any]],
return None, None, None
def _check_single_track(file_path: str, filename: str, api_tracks: List[Dict],
total_tracks: int, title_similarity: float) -> Optional[Dict]:
"""Check if a track needs repair and return finding info (dry run mode).
Returns a dict with 'description' and 'details' if repair is needed, else None.
"""
from mutagen import File as MutagenFile
audio = MutagenFile(file_path)
if audio is None:
return None
file_title = _read_title_tag(audio)
matched_track = None
if file_title:
matched_track = _match_title_to_api_track(file_title, api_tracks, title_similarity)
if not matched_track:
basename = os.path.splitext(filename)[0]
clean_name = re.sub(r'^\d{1,3}[\s.\-_]*', '', basename).strip()
if clean_name:
matched_track = _match_title_to_api_track(clean_name, api_tracks, title_similarity)
if not matched_track:
return None
correct_num = matched_track.get('track_number')
if correct_num is None:
return None
current_num, current_total = _read_track_number_tag(audio)
if current_num == correct_num and current_total == total_tracks:
return None # Already correct
changes = []
if current_num != correct_num:
changes.append(f'Track number: {current_num} -> {correct_num}')
if current_total != total_tracks:
changes.append(f'Total tracks: {current_total} -> {total_tracks}')
# Check if filename would change
basename_noext = os.path.splitext(filename)[0]
new_basename = re.sub(r'^\d{1,3}', f'{correct_num:02d}', basename_noext)
if new_basename != basename_noext:
changes.append(f'Filename: {filename} -> {new_basename}{os.path.splitext(filename)[1]}')
return {
'description': f'Matched to: "{matched_track.get("name", "?")}"\n' + '\n'.join(changes),
'details': {
'current_track_num': current_num,
'correct_track_num': correct_num,
'matched_title': matched_track.get('name', ''),
'file_title': file_title or filename,
'changes': changes,
}
}
def _repair_single_track(file_path: str, filename: str, api_tracks: List[Dict],
total_tracks: int, title_similarity: float,
context: JobContext) -> bool:

View file

@ -68,6 +68,11 @@ class RepairWorker:
# Config manager (set externally after init)
self._config_manager = None
# Rich progress callbacks (set by web_server.py)
self._on_job_start = None # (job_id, display_name) -> None
self._on_job_progress = None # (job_id, **kwargs) -> None
self._on_job_finish = None # (job_id, status, result) -> None
# Lazy client accessors
self._spotify_client = None
self._itunes_client = None
@ -80,12 +85,24 @@ class RepairWorker:
# ------------------------------------------------------------------
# Config manager
# ------------------------------------------------------------------
def register_progress_callbacks(self, on_start, on_progress, on_finish):
"""Register callbacks for rich per-job progress reporting.
Args:
on_start: (job_id, display_name) called when a job begins
on_progress: (job_id, **kwargs) called for incremental updates
on_finish: (job_id, status, result) called when a job ends
"""
self._on_job_start = on_start
self._on_job_progress = on_progress
self._on_job_finish = on_finish
def set_config_manager(self, config_manager):
"""Set the config manager for persisting job settings."""
self._config_manager = config_manager
# Load master enabled state
if config_manager:
self.enabled = config_manager.get('repair.master_enabled', False)
self.enabled = config_manager.get('repair.master_enabled', True)
# ------------------------------------------------------------------
# Lazy client accessors
@ -453,9 +470,24 @@ class RepairWorker:
raw = self._get_transfer_path_from_db()
self.transfer_folder = self._resolve_path(raw)
# Notify rich progress system
if self._on_job_start:
try:
self._on_job_start(job_id, job.display_name)
except Exception:
pass
# Record job start
run_id = self._record_job_start(job_id)
# Build report_progress callback for this job
def _report_progress(**kwargs):
if self._on_job_progress:
try:
self._on_job_progress(job_id, **kwargs)
except Exception:
pass
# Build context
context = JobContext(
db=self.db,
@ -470,6 +502,7 @@ class RepairWorker:
should_stop=lambda: self.should_stop,
is_paused=lambda: not self.enabled,
update_progress=self._update_progress,
report_progress=_report_progress,
)
start_time = time.time()
@ -492,6 +525,14 @@ class RepairWorker:
# Record job completion
self._record_job_finish(run_id, job_id, result, duration)
# Notify rich progress system of completion
if self._on_job_finish:
try:
status = 'error' if result.errors > 0 and result.auto_fixed == 0 else 'finished'
self._on_job_finish(job_id, status, result)
except Exception:
pass
logger.info(
"Job %s complete: scanned=%d fixed=%d findings=%d errors=%d (%.1fs)",
job_id, result.scanned, result.auto_fixed,

View file

@ -37451,6 +37451,63 @@ try:
transfer_path = docker_resolve_path(config_manager.get('soulseek.transfer_path', './Transfer'))
repair_worker = RepairWorker(database=repair_db, transfer_folder=transfer_path)
repair_worker.set_config_manager(config_manager)
# --- Repair Job Progress Tracking (live progress like automation cards) ---
repair_job_progress_states = {} # job_id (str) -> state dict
repair_job_progress_lock = threading.Lock()
def _repair_job_start(job_id, display_name):
with repair_job_progress_lock:
repair_job_progress_states[job_id] = {
'status': 'running',
'display_name': display_name,
'progress': 0, 'phase': 'Starting...', 'current_item': '',
'processed': 0, 'total': 0,
'log': [{'type': 'info', 'text': f'Starting {display_name}'}],
'started_at': datetime.now(timezone.utc).isoformat(),
'finished_at': None,
}
def _repair_job_progress(job_id, **kwargs):
with repair_job_progress_lock:
state = repair_job_progress_states.get(job_id)
if not state:
return
for k, v in kwargs.items():
if k == 'log_line':
state['log'].append({'type': kwargs.get('log_type', 'info'), 'text': v})
if len(state['log']) > 50:
state['log'] = state['log'][-50:]
elif k == 'scanned':
state['processed'] = v
if state.get('total', 0) > 0:
state['progress'] = round(v / state['total'] * 100)
elif k == 'total':
state['total'] = v
if state.get('processed', 0) > 0 and v > 0:
state['progress'] = round(state['processed'] / v * 100)
elif k != 'log_type':
state[k] = v
def _repair_job_finish(job_id, status, result):
with repair_job_progress_lock:
state = repair_job_progress_states.get(job_id)
if not state:
return
state['status'] = status
state['progress'] = 100
state['finished_at'] = datetime.now(timezone.utc).isoformat()
summary = f'Done: {result.scanned} scanned, {result.auto_fixed} fixed, {result.findings_created} findings, {result.errors} errors'
state['log'].append({'type': 'success' if status == 'finished' else 'error', 'text': summary})
try:
socketio.emit('repair:progress', {job_id: dict(state)})
except Exception:
pass
repair_worker.register_progress_callbacks(_repair_job_start, _repair_job_progress, _repair_job_finish)
# Store refs for WebSocket push loop
repair_worker._progress_lock_ref = repair_job_progress_lock
repair_worker._progress_states_ref = repair_job_progress_states
repair_worker.start()
print("✅ Repair worker initialized and started")
except Exception as e:
@ -37681,6 +37738,27 @@ def repair_history():
logger.error(f"Error getting repair history: {e}")
return jsonify({'error': str(e)}), 500
@app.route('/api/repair/progress', methods=['GET'])
def repair_job_progress():
"""Get current repair job progress states (for initial page load)"""
try:
if repair_worker is None:
return jsonify({}), 200
lock = getattr(repair_worker, '_progress_lock_ref', None)
states = getattr(repair_worker, '_progress_states_ref', None)
if lock is None or states is None:
return jsonify({}), 200
with lock:
result = {}
for jid, state in states.items():
cp = dict(state)
cp['log'] = list(state['log'])
result[jid] = cp
return jsonify(result), 200
except Exception as e:
logger.error(f"Error getting repair progress: {e}")
return jsonify({'error': str(e)}), 500
# ================================================================================================
# END LIBRARY REPAIR WORKER
# ================================================================================================
@ -38999,6 +39077,46 @@ def _emit_automation_progress_loop():
except Exception as e:
logger.debug(f"Error emitting automation progress: {e}")
def _emit_repair_progress_loop():
"""Push repair:progress events every 1 second for running repair jobs."""
while True:
socketio.sleep(1)
try:
if repair_worker is None:
continue
# Access the progress states set up during repair worker init
lock = getattr(repair_worker, '_progress_lock_ref', None)
states = getattr(repair_worker, '_progress_states_ref', None)
if lock is None or states is None:
continue
with lock:
active = {}
stale = []
now = datetime.now()
for jid, state in states.items():
if state['status'] == 'running':
cp = dict(state)
cp['log'] = list(state['log'])
active[jid] = cp
elif state['status'] in ('finished', 'error') and state.get('finished_at'):
try:
finished_time = datetime.fromisoformat(state['finished_at'])
if (now - finished_time).total_seconds() > 60:
stale.append(jid)
except (ValueError, TypeError):
stale.append(jid)
# Still include recently finished states so frontend sees final state
if jid not in stale:
cp = dict(state)
cp['log'] = list(state['log'])
active[jid] = cp
for jid in stale:
del states[jid]
if active:
socketio.emit('repair:progress', active)
except Exception as e:
logger.debug(f"Error emitting repair progress: {e}")
# ================================================================================================
# END WEBSOCKET HANDLERS
# ================================================================================================
@ -39087,6 +39205,8 @@ if __name__ == '__main__':
socketio.start_background_task(_emit_scan_status_loop)
# Phase 6: Automation progress
socketio.start_background_task(_emit_automation_progress_loop)
print("✅ WebSocket emitters started (Phase 1-6: global/dashboard/enrichment/tools/sync/automations)")
# Phase 7: Repair job progress
socketio.start_background_task(_emit_repair_progress_loop)
print("✅ WebSocket emitters started (Phase 1-7: global/dashboard/enrichment/tools/sync/automations/repair)")
socketio.run(app, host='0.0.0.0', port=8008, debug=False, allow_unsafe_werkzeug=True)

View file

@ -257,6 +257,7 @@ function initializeWebSocket() {
socket.on('enrichment:qobuz-enrichment', (data) => updateQobuzEnrichmentStatusFromData(data));
socket.on('enrichment:hydrabase', (data) => updateHydrabaseStatusFromData(data));
socket.on('enrichment:repair', (data) => updateRepairStatusFromData(data));
socket.on('repair:progress', (data) => updateRepairJobProgressFromData(data));
// Phase 4 event listeners (tool progress)
socket.on('tool:stream', (data) => updateStreamStatusFromData(data));
@ -49860,7 +49861,7 @@ const REPAIR_FINDINGS_PAGE_SIZE = 30;
/**
* Open the Library Maintenance modal
*/
function openRepairModal() {
async function openRepairModal() {
const modal = document.getElementById('repair-modal');
if (!modal) return;
modal.style.display = 'flex';
@ -49869,6 +49870,17 @@ function openRepairModal() {
switchRepairTab('jobs');
// Load master toggle state
updateRepairStatus();
// Load any active job progress
try {
const resp = await fetch('/api/repair/progress');
if (resp.ok) {
const data = await resp.json();
if (Object.keys(data).length > 0) {
// Brief delay so job cards are rendered first
setTimeout(() => updateRepairJobProgressFromData(data), 300);
}
}
} catch (e) { /* ignore */ }
}
function closeRepairModal() {
@ -49974,10 +49986,11 @@ async function loadRepairJobs() {
</div>`;
}
return `<div class="repair-job-card ${statusClass}">
return `<div class="repair-job-card ${statusClass}" data-job-id="${job.job_id}">
<div class="repair-job-main">
<div class="repair-job-info">
<div class="repair-job-title">
<span class="repair-job-icon"><span class="${job.icon || 'repair-icon-default'}"></span></span>
<span class="repair-job-name">${job.display_name}</span>
<span class="repair-job-status-pill ${statusClass}">${statusText}</span>
${job.auto_fix ? '<span class="repair-job-autofix-pill">Auto-fix</span>' : ''}
@ -50070,6 +50083,94 @@ async function runRepairJobNow(jobId) {
}
}
// ── Repair Job Live Progress ──
const _repairProgressLogCounts = {};
const _repairProgressHideTimers = {};
function updateRepairJobProgressFromData(data) {
for (const [jobId, state] of Object.entries(data)) {
const card = document.querySelector(`.repair-job-card[data-job-id="${jobId}"]`);
if (!card) continue;
// Update status pill
const pill = card.querySelector('.repair-job-status-pill');
if (pill) {
pill.className = 'repair-job-status-pill ' + state.status;
if (state.status === 'running') pill.textContent = 'Running';
else if (state.status === 'finished') pill.textContent = 'Complete';
else if (state.status === 'error') pill.textContent = 'Error';
}
// Add/update card running class
card.classList.toggle('running', state.status === 'running');
// Create or find progress panel
let panel = card.querySelector('.repair-job-progress');
if (!panel) {
panel = document.createElement('div');
panel.className = 'repair-job-progress';
panel.innerHTML = `
<div class="repair-progress-phase"></div>
<div class="repair-progress-bar-wrap">
<div class="repair-progress-bar" style="width:0%"></div>
</div>
<div class="repair-progress-log"></div>
`;
card.appendChild(panel);
}
// Show panel
panel.classList.add('visible');
panel.classList.toggle('finished', state.status === 'finished');
panel.classList.toggle('error', state.status === 'error');
// Update phase
const phaseEl = panel.querySelector('.repair-progress-phase');
if (phaseEl && state.phase) phaseEl.textContent = state.phase;
// Update progress bar
const bar = panel.querySelector('.repair-progress-bar');
if (bar) bar.style.width = (state.progress || 0) + '%';
// Update log
const logEl = panel.querySelector('.repair-progress-log');
if (logEl && state.log) {
const prevCount = _repairProgressLogCounts[jobId] || 0;
if (state.log.length > prevCount) {
const newLines = state.log.slice(prevCount);
for (const line of newLines) {
const div = document.createElement('div');
div.className = 'repair-log-line ' + (line.type || 'info');
div.textContent = line.text;
logEl.appendChild(div);
}
logEl.scrollTop = logEl.scrollHeight;
}
_repairProgressLogCounts[jobId] = state.log.length;
}
// Auto-hide panel after completion
if (state.status === 'finished' || state.status === 'error') {
if (!_repairProgressHideTimers[jobId]) {
_repairProgressHideTimers[jobId] = setTimeout(() => {
panel.classList.remove('visible');
card.classList.remove('running');
delete _repairProgressHideTimers[jobId];
delete _repairProgressLogCounts[jobId];
// Reload to get updated stats
loadRepairJobs();
}, 30000);
}
} else {
// Clear any existing hide timer if job restarts
if (_repairProgressHideTimers[jobId]) {
clearTimeout(_repairProgressHideTimers[jobId]);
delete _repairProgressHideTimers[jobId];
}
}
}
}
async function loadRepairFindings() {
const container = document.getElementById('repair-findings-list');
if (!container) return;

View file

@ -40371,3 +40371,113 @@ tr.tag-diff-same {
align-self: flex-end;
}
}
/* ── Repair Job Live Progress Panel ── */
.repair-job-card.running {
border-color: rgba(var(--accent-rgb, 99, 102, 241), 0.35);
box-shadow: 0 0 12px rgba(var(--accent-rgb, 99, 102, 241), 0.1);
}
@keyframes repair-card-pulse {
0%, 100% { box-shadow: 0 0 8px rgba(var(--accent-rgb, 99, 102, 241), 0.08); }
50% { box-shadow: 0 0 16px rgba(var(--accent-rgb, 99, 102, 241), 0.18); }
}
.repair-job-card.running {
animation: repair-card-pulse 2s ease-in-out infinite;
}
.repair-job-status-pill.finished {
background: rgba(34, 197, 94, 0.12);
color: #22c55e;
}
.repair-job-status-pill.error {
background: rgba(239, 68, 68, 0.12);
color: #ef4444;
}
.repair-job-icon {
display: flex;
align-items: center;
font-size: 14px;
color: rgba(255, 255, 255, 0.4);
}
.repair-job-progress {
max-height: 0;
overflow: hidden;
transition: max-height 0.3s ease, padding 0.3s ease;
padding: 0 16px;
}
.repair-job-progress.visible {
max-height: 300px;
padding: 10px 16px 14px;
border-top: 1px solid rgba(255, 255, 255, 0.04);
}
.repair-progress-phase {
font-size: 11px;
color: rgba(255, 255, 255, 0.55);
margin-bottom: 6px;
font-weight: 500;
}
.repair-progress-bar-wrap {
width: 100%;
height: 4px;
background: rgba(255, 255, 255, 0.06);
border-radius: 2px;
overflow: hidden;
margin-bottom: 8px;
}
.repair-progress-bar {
height: 100%;
background: linear-gradient(90deg, rgb(var(--accent-rgb, 99, 102, 241)), rgb(var(--accent-light-rgb, 129, 140, 248)));
transition: width 0.5s ease;
border-radius: 2px;
}
.repair-job-progress.finished .repair-progress-bar {
background: #4ade80;
}
.repair-job-progress.error .repair-progress-bar {
background: #ef4444;
}
.repair-progress-log {
max-height: 160px;
overflow-y: auto;
font-family: 'SF Mono', 'Cascadia Code', 'Fira Code', monospace;
font-size: 10px;
line-height: 1.5;
}
.repair-progress-log::-webkit-scrollbar {
width: 4px;
}
.repair-progress-log::-webkit-scrollbar-thumb {
background: rgba(255, 255, 255, 0.1);
border-radius: 2px;
}
.repair-log-line {
padding: 1px 0;
color: rgba(255, 255, 255, 0.45);
}
.repair-log-line.success {
color: #4ade80;
}
.repair-log-line.error {
color: #f87171;
}
.repair-log-line.skip {
color: rgba(255, 255, 255, 0.25);
}