danvics/soulsync
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 1

tools: add Preview Clip Cleanup repair job (detect ~30s previews, re-fetch full track)

Browse source 
HiFi (and occasionally other) downloads sometimes deliver a ~30s preview clip instead of the
full song; it lands in the library looking real. new repair job scans short tracks (duration
<= 30s, configurable), looks up the EXPECTED length from the track's metadata source
(spotify/itunes/mb get_track_details), and flags any whose real length is much longer than the
file (default: >= 30s longer) as a preview clip.

approving the finding (repair_worker._fix_short_preview_track) deletes the preview file (path
resolved via _resolve_file_path like the other delete tools), drops the DB row so the track
goes missing, and re-adds it to the wishlist with the full payload (mirrors _fix_dead_file)
so the real version downloads. scan ONLY creates findings — nothing destructive without user
approval, like every other tool.

conservative: genuine short tracks (source agrees they're short) and tracks whose length can't
be verified are skipped, never flagged. registered the job + finding-type label/fix-button in
the UI. 5 tests (scan flag/skip/scope + fix delete+remove+wishlist); 89 repair tests green.
This commit is contained in:
BoulderBadgeDad 2026-06-27 16:07:41 -07:00
parent 319b6483a2
commit 116edeb477
5 changed files with 464 additions and 1 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
core/repair_jobs/__init__.py
View file

@ -52,6 +52,7 @@ _JOB_MODULES = [
'core.repair_jobs.canonical_version_resolve',
'core.repair_jobs.library_retag',
'core.repair_jobs.quality_upgrade',
'core.repair_jobs.short_preview_track',
]

209
core/repair_jobs/short_preview_track.py Normal file
View file

@ -0,0 +1,209 @@
"""Repair job: detect ~30s PREVIEW clips and re-fetch the full track.
The HiFi endpoint (and occasionally others) sometimes deliver a ~30-second preview/sample
instead of the full song. Those land in the library looking like real tracks. This job
finds short tracks, looks up the EXPECTED length from the track's metadata source, and —
when the source says the real track is meaningfully longer than the file — flags it as a
preview clip.
Approving the finding (in repair_worker._fix_short_preview_track) deletes the preview file,
drops the DB row so the track goes missing again, and re-adds it to the wishlist with a full
payload so the real version gets downloaded. The scan itself ONLY creates findings — nothing
is deleted, removed, or wishlisted without the user approving, exactly like the other tools.
Conservative by design (it deletes a file): a track is only flagged when the source confirms
it should be much longer. Genuine short tracks (intros, skits, interludes — where the source
agrees the track is short) are left alone, and tracks whose length can't be verified from a
source are skipped, never flagged.
"""
from __future__ import annotations
from typing import Any, Dict, Optional
from core.repair_jobs import register_job
from core.repair_jobs.base import JobContext, JobResult, RepairJob
from utils.logging_config import get_logger
logger = get_logger("repair_jobs.short_preview")
@register_job
class ShortPreviewTrackJob(RepairJob):
job_id = "short_preview_track"
display_name = "Preview Clip Cleanup"
description = (
"Finds ~30s preview clips that slipped in instead of the full song (common from the "
"HiFi endpoint) and re-fetches the real track."
)
help_text = (
"Some downloads — especially via the HiFi source — deliver a ~30-second preview clip "
"instead of the full song. They look like normal tracks in your library. This job scans "
"for short tracks, checks how long the track ACTUALLY is from its metadata source "
"(Spotify / iTunes / MusicBrainz), and flags any whose real length is much greater than "
"the file — i.e. a preview.\n\n"
"Approving a finding deletes the preview file, removes the track from the database (so it "
"shows as missing), and re-adds it to your Wishlist so the full version downloads.\n\n"
"It's conservative: genuine short tracks (intros, skits) where the source agrees the track "
"is short are left alone, and tracks whose length can't be verified are skipped.\n\n"
"Settings:\n"
" - max_duration_seconds: only tracks at or below this length are considered (default 30).\n"
" - min_expected_drift_seconds: the source must say the real track is at least this many "
"seconds longer than the file before it's flagged (default 30)."
)
icon = "scissors"
default_enabled = False
default_interval_hours = 168 # weekly
default_settings = {
"max_duration_seconds": 30,
"min_expected_drift_seconds": 30,
}
setting_options: Dict[str, list] = {}
auto_fix = False
def _setting_int(self, context: JobContext, key: str, default: int) -> int:
cm = getattr(context, "config_manager", None)
if cm is None:
return default
try:
return int(cm.get(self.get_config_key(key), default) or default)
except (TypeError, ValueError):
return default
def scan(self, context: JobContext) -> JobResult:
result = JobResult()
max_dur_s = self._setting_int(context, "max_duration_seconds", 30)
min_drift_s = self._setting_int(context, "min_expected_drift_seconds", 30)
max_dur_ms = max_dur_s * 1000
conn = context.db._get_connection()
try:
cursor = conn.cursor()
cursor.execute(
"""
SELECT t.id, t.title, t.duration, t.file_path,
t.spotify_track_id, t.itunes_track_id, t.musicbrainz_recording_id,
ar.name AS artist_name, ar.thumb_url AS artist_thumb,
al.title AS album_title, al.thumb_url AS album_thumb
FROM tracks t
LEFT JOIN artists ar ON ar.id = t.artist_id
LEFT JOIN albums al ON al.id = t.album_id
WHERE t.duration IS NOT NULL AND t.duration > 0 AND t.duration <= ?
AND t.file_path IS NOT NULL AND t.file_path != ''
""",
(max_dur_ms,),
)
rows = [dict(r) for r in cursor.fetchall()]
finally:
conn.close()
total = len(rows)
for i, row in enumerate(rows):
if context.check_stop() or context.wait_if_paused():
break
result.scanned += 1
file_dur_s = (row["duration"] or 0) / 1000.0
expected_dur_s = self._expected_duration_s(context, row)
# Can't verify the real length → never flag (a delete must be backed by evidence).
if expected_dur_s is None or expected_dur_s <= 0:
result.skipped += 1
self._tick(context, i, total)
continue
# Source agrees the track is short (genuine intro/skit) → leave it alone. Only a
# source that says the real track is MUCH longer than the file marks a preview.
if (expected_dur_s - file_dur_s) < min_drift_s:
result.skipped += 1
self._tick(context, i, total)
continue
if context.create_finding:
title = row["title"] or "Unknown"
artist = row["artist_name"] or "Unknown"
try:
inserted = context.create_finding(
job_id=self.job_id,
finding_type="short_preview_track",
severity="warning",
entity_type="track",
entity_id=str(row["id"]),
file_path=row["file_path"],
title=f"Preview clip: {artist} - {title}",
description=(
f'File is {file_dur_s:.0f}s but "{title}" by {artist} is '
f"{expected_dur_s:.0f}s at the source — looks like a preview clip. "
"Approve to delete it and re-download the full version."
),
details={
"track_id": row["id"],
"title": row["title"],
"artist": row["artist_name"],
"album": row["album_title"],
"album_thumb_url": row["album_thumb"],
"artist_thumb_url": row["artist_thumb"],
"file_duration_s": round(file_dur_s, 1),
"expected_duration_s": round(expected_dur_s, 1),
"original_path": row["file_path"],
},
)
if inserted:
result.findings_created += 1
else:
result.findings_skipped_dedup += 1
except Exception as exc:
logger.debug("create_finding failed for track %s: %s", row["id"], exc)
result.errors += 1
self._tick(context, i, total)
return result
def _tick(self, context: JobContext, i: int, total: int) -> None:
if context.update_progress and (i + 1) % 5 == 0:
try:
context.update_progress(i + 1, total)
except Exception: # noqa: S110 — progress reporting is best-effort, never fail a scan on it
pass
def _expected_duration_s(self, context: JobContext, row: Dict[str, Any]) -> Optional[float]:
"""Canonical track length (seconds) from the track's metadata source, or None when
no source id is usable / the lookup fails. Every metadata client exposes
get_track_details(id) -> {... 'duration_ms': N ...} (the metadata-service contract)."""
candidates = [
(row.get("spotify_track_id"), context.spotify_client,
context.is_spotify_rate_limited()),
(row.get("itunes_track_id"), context.itunes_client, False),
(row.get("musicbrainz_recording_id"), context.mb_client, False),
]
for source_id, client, rate_limited in candidates:
if not source_id or client is None or rate_limited:
continue
getter = getattr(client, "get_track_details", None)
if getter is None:
continue
try:
details = getter(str(source_id))
ms = (details or {}).get("duration_ms")
if ms and ms > 0:
return ms / 1000.0
except Exception as exc:
logger.debug("duration lookup failed for %s: %s", source_id, exc)
return None
def estimate_scope(self, context: JobContext) -> int:
try:
max_dur_ms = self._setting_int(context, "max_duration_seconds", 30) * 1000
conn = context.db._get_connection()
try:
cursor = conn.cursor()
cursor.execute(
"SELECT COUNT(*) FROM tracks WHERE duration > 0 AND duration <= ? "
"AND file_path IS NOT NULL AND file_path != ''",
(max_dur_ms,),
)
return (cursor.fetchone() or [0])[0]
finally:
conn.close()
except Exception:
return 0

111
core/repair_worker.py
View file

@ -997,6 +997,7 @@ class RepairWorker:
'quality_upgrade': self._fix_quality_upgrade,
'missing_discography_track': self._fix_discography_backfill,
'library_retag': self._fix_library_retag,
'short_preview_track': self._fix_short_preview_track,
}
handler = handlers.get(finding_type)
if not handler:
@ -1211,6 +1212,116 @@ class RepairWorker:
if conn:
conn.close()
def _fix_short_preview_track(self, entity_type, entity_id, file_path, details):
"""Approve a preview-clip finding: delete the ~30s preview file, drop its DB row, and
re-add the track to the wishlist (full payload) so the real version downloads. Mirrors
the dead-file 'redownload' payload + the acoustid-mismatch file delete. (Tools #937-adj)
"""
if not entity_id:
return {'success': False, 'error': 'No track ID associated with this finding'}
conn = None
try:
conn = self.db._get_connection()
cursor = conn.cursor()
cursor.execute("""
SELECT t.id, t.title, t.track_number, t.duration, t.bitrate,
t.spotify_track_id, t.itunes_track_id, t.deezer_id, t.isrc,
ar.name AS artist_name, ar.spotify_artist_id,
al.title AS album_title, al.spotify_album_id,
al.record_type, al.track_count, al.year, al.thumb_url AS album_thumb
FROM tracks t
LEFT JOIN artists ar ON ar.id = t.artist_id
LEFT JOIN albums al ON al.id = t.album_id
WHERE t.id = ?
""", (entity_id,))
row = cursor.fetchone()
if not row:
return {'success': False, 'error': 'Track not found in database'}
track_name = row['title'] or details.get('title', 'Unknown')
artist_name = row['artist_name'] or details.get('artist', 'Unknown Artist')
album_title = row['album_title'] or details.get('album', '')
wishlist_id = (row['spotify_track_id']
or row['itunes_track_id']
or row['deezer_id']
or f"preview_redl_{entity_id}")
album_images = []
album_thumb = row['album_thumb'] or details.get('album_thumb_url')
if album_thumb:
album_images = [{'url': album_thumb}]
spotify_track_data = {
'id': wishlist_id,
'name': track_name,
'artists': [{'name': artist_name}],
'album': {
'name': album_title or track_name,
'id': row['spotify_album_id'] or '',
'release_date': str(row['year']) if row['year'] else '',
'images': album_images,
'album_type': row['record_type'] or 'album',
'total_tracks': row['track_count'] or 0,
'artists': [{'name': artist_name}],
},
'duration_ms': int((details.get('expected_duration_s') or 0) * 1000) or (row['duration'] or 0),
'track_number': row['track_number'] or 1,
'disc_number': 1,
'explicit': False,
'external_urls': {},
'popularity': 0,
'preview_url': None,
'uri': f"spotify:track:{row['spotify_track_id']}" if row['spotify_track_id'] else '',
'is_local': False,
}
source_info = {
'original_path': file_path or details.get('original_path', ''),
'album_title': album_title,
'artist': artist_name,
'reason': 'preview_clip_redownload',
}
added = self.db.add_to_wishlist(
spotify_track_data,
failure_reason='Preview clip — re-downloading full track',
source_type='redownload',
source_info=source_info,
)
if not added:
return {'success': False, 'error': 'Failed to add to wishlist (may already exist or be blocklisted)'}
# Delete the preview file (path resolved like the other delete tools).
deleted_file = False
target_path = file_path or details.get('original_path')
if target_path:
download_folder = self._config_manager.get('soulseek.download_path', '') if self._config_manager else None
resolved = _resolve_file_path(target_path, self.transfer_folder,
download_folder=download_folder,
config_manager=self._config_manager)
if resolved and os.path.exists(resolved):
try:
os.remove(resolved)
deleted_file = True
except Exception as e:
logger.warning("Could not delete preview file %s: %s", resolved, e)
# Drop the DB row so the track shows as missing.
cursor.execute("DELETE FROM tracks WHERE id = ?", (entity_id,))
conn.commit()
return {'success': True, 'action': 'added_to_wishlist',
'message': (f'Deleted preview clip and re-wishlisted "{track_name}" for full download'
if deleted_file else
f'Re-wishlisted "{track_name}" (preview file already gone)')}
except Exception as e:
logger.error("Preview-clip fix failed for track %s: %s", entity_id, e)
return {'success': False, 'error': str(e)}
finally:
if conn:
conn.close()
def _fix_orphan_file(self, entity_type, entity_id, file_path, details):
"""Handle an orphan file — move to staging or delete based on user choice.

141
tests/repair_jobs/test_short_preview_track.py Normal file
View file

@ -0,0 +1,141 @@
"""Preview-clip cleanup job (#937-adjacent): flag ~30s preview clips whose source says the
real track is much longer, then on approval delete the file + drop the row + re-wishlist."""
from __future__ import annotations
from pathlib import Path
import pytest
from core.repair_jobs.base import JobContext
from core.repair_jobs.short_preview_track import ShortPreviewTrackJob
from core.repair_worker import RepairWorker
from database.music_database import MusicDatabase
def _seed(db: MusicDatabase):
conn = db._get_connection()
conn.execute("INSERT OR IGNORE INTO artists (id, name) VALUES ('ar1', 'A-ha')")
conn.execute("INSERT INTO albums (id, artist_id, title) VALUES ('al1', 'ar1', 'Hunting High and Low')")
conn.commit()
conn.close()
def _track(db, tid, duration_ms, path, spotify_id=None):
conn = db._get_connection()
conn.execute(
"INSERT INTO tracks (id, artist_id, album_id, title, duration, file_path, spotify_track_id) "
"VALUES (?, 'ar1', 'al1', ?, ?, ?, ?)",
(tid, f"Track {tid}", duration_ms, path, spotify_id),
)
conn.commit()
conn.close()
class _FakeSpotify:
"""get_track_details(id) -> {'duration_ms': N}. 'sp_long' is a full song; else short."""
def get_track_details(self, track_id, **_):
return {'duration_ms': 200_000} if track_id == 'sp_long' else {'duration_ms': 28_000}
def _ctx(db, findings, spotify=None):
return JobContext(
db=db, transfer_folder='/tmp', config_manager=None,
spotify_client=spotify,
create_finding=lambda **kw: findings.append(kw) or True,
should_stop=lambda: False, is_paused=lambda: False,
)
# ── scan ──
def test_scan_flags_preview_skips_genuine_short_and_unverifiable(tmp_path: Path):
db = MusicDatabase(str(tmp_path / 'm.db'))
_seed(db)
_track(db, 'preview', 28_000, '/m/p.flac', spotify_id='sp_long') # 28s file, source 200s → FLAG
_track(db, 'skit', 28_000, '/m/i.flac', spotify_id='sp_short') # 28s file, source 28s → skip (genuine)
_track(db, 'noid', 28_000, '/m/m.flac', spotify_id=None) # 28s, no source id → skip (unverifiable)
_track(db, 'longone', 200_000, '/m/l.flac', spotify_id='sp_long') # 200s → not scanned (>30s)
findings = []
result = ShortPreviewTrackJob().scan(_ctx(db, findings, _FakeSpotify()))
assert len(findings) == 1
f = findings[0]
assert f['finding_type'] == 'short_preview_track'
assert f['entity_id'] == 'preview'
assert f['entity_type'] == 'track'
assert f['details']['expected_duration_s'] == pytest.approx(200.0)
assert result.findings_created == 1
assert result.scanned == 3 # the 200s track is excluded by the query, not scanned
assert result.skipped == 2 # skit + noid
def test_scan_creates_no_finding_when_source_agrees_short(tmp_path: Path):
db = MusicDatabase(str(tmp_path / 'm.db'))
_seed(db)
_track(db, 'skit', 28_000, '/m/i.flac', spotify_id='sp_short') # source also says 28s
findings = []
ShortPreviewTrackJob().scan(_ctx(db, findings, _FakeSpotify()))
assert findings == []
def test_estimate_scope_counts_short_tracks(tmp_path: Path):
db = MusicDatabase(str(tmp_path / 'm.db'))
_seed(db)
_track(db, 'a', 28_000, '/m/a.flac', spotify_id='sp_long')
_track(db, 'b', 10_000, '/m/b.flac', spotify_id='sp_short')
_track(db, 'c', 200_000, '/m/c.flac', spotify_id='sp_long') # >30s, excluded
assert ShortPreviewTrackJob().estimate_scope(_ctx(db, [], _FakeSpotify())) == 2
# ── fix (approval) ──
def test_fix_deletes_file_removes_row_and_wishlists(tmp_path: Path):
db = MusicDatabase(str(tmp_path / 'm.db'))
_seed(db)
preview = tmp_path / 'preview.flac'
preview.write_bytes(b'fake audio bytes')
_track(db, 't1', 28_000, str(preview), spotify_id='sp1')
captured = {}
db.add_to_wishlist = lambda spotify_track_data, **kw: captured.update(
{'data': spotify_track_data, 'kw': kw}) or True
w = RepairWorker.__new__(RepairWorker)
w.db = db
w.transfer_folder = str(tmp_path)
w._config_manager = None
res = w._fix_short_preview_track(
'track', 't1', str(preview),
{'expected_duration_s': 225.0, 'original_path': str(preview)})
assert res['success'] is True
assert not preview.exists() # preview file deleted
assert captured['data']['name'] == 'Track t1' # re-wishlisted with payload
assert captured['data']['duration_ms'] == 225_000 # uses the real (expected) length
assert captured['kw'].get('source_type') == 'redownload'
conn = db._get_connection()
remaining = conn.execute("SELECT COUNT(*) FROM tracks WHERE id='t1'").fetchone()[0]
conn.close()
assert remaining == 0 # DB row dropped → track missing again
def test_fix_missing_file_still_wishlists_and_drops_row(tmp_path: Path):
"""If the preview file is already gone, still re-wishlist + drop the row (idempotent-ish)."""
db = MusicDatabase(str(tmp_path / 'm.db'))
_seed(db)
_track(db, 't2', 28_000, str(tmp_path / 'gone.flac'), spotify_id='sp2')
db.add_to_wishlist = lambda spotify_track_data, **kw: True
w = RepairWorker.__new__(RepairWorker)
w.db = db
w.transfer_folder = str(tmp_path)
w._config_manager = None
res = w._fix_short_preview_track('track', 't2', str(tmp_path / 'gone.flac'), {})
assert res['success'] is True
conn = db._get_connection()
assert conn.execute("SELECT COUNT(*) FROM tracks WHERE id='t2'").fetchone()[0] == 0
conn.close()

3
webui/static/enrichment.js
View file

@ -2749,7 +2749,7 @@ async function loadRepairFindings() {
missing_lyrics: 'Missing Lyrics', expired_download: 'Expired',
missing_replaygain: 'No ReplayGain', empty_folder: 'Empty Folder',
missing_lossy_copy: 'No Lossy Copy', library_retag: 'Re-tag',
quality_upgrade: 'Low Quality'
quality_upgrade: 'Low Quality', short_preview_track: 'Preview Clip'
};
// Finding types that have an automated fix action
@ -2770,6 +2770,7 @@ async function loadRepairFindings() {
quality_upgrade: 'Upgrade',
missing_discography_track: 'Add to Wishlist',
library_retag: 'Apply Tags',
short_preview_track: 'Re-download',
};
container.innerHTML = items.map(f => {