soulsync/core/imports/file_integrity.py
BoulderBadgeDad 20ca4bb981 Import: don't duration-quarantine manual imports against a re-resolved release (#804)
CubeComming #804: importing Coldplay "Yellow" (the 269s Parachutes album track,
correctly tagged) was quarantined — "Duration mismatch: file is 269.2s, expected
266.0s (drift 3.2s > tolerance 3.0s)". The expected 266s came from a re-resolved
*single* edition, not the file's actual album. The duration-agreement integrity
check exists to catch truncated/wrong slskd TRANSFERS — but a manual import is
the user's own already-tagged file being sorted, so checking it against a
re-resolved release just manufactures false quarantines.

Fix: both manual-import paths (singles + album) now mark the context
is_local_import; the integrity check skips the duration-agreement leg for local
imports via expected_duration_for_check() (new pure helper). The size +
mutagen-parse legs still run, so genuinely broken files are still caught — only
the release-vs-file duration comparison is skipped, and only for manual imports.
slskd downloads are completely unaffected.

This does NOT change the deeper matching (file still groups under Singles vs the
Parachutes album — the #767 canonical-version family); it stops the false
quarantine so the file imports.

Tests: 4 on the helper (local skips, download keeps, zero/None/garbage, string
coercion) + updated the routes context assertion. 557 import/integrity tests pass.
2026-06-07 23:02:34 -07:00

261 lines
10 KiB
Python

"""Audio file integrity checks for downloaded files.
slskd (and other download sources) sometimes ship broken files: truncated
transfers, corrupted FLAC frames, mp3s with bad headers, or wrong files
that share a name with the target. These slip past the slskd "completed"
status and only get caught later (often by Plex/Jellyfin failing to scan
the file, or by users hearing dead air).
Verification runs after the slskd transfer settles but before the heavy
post-processing work (tagging, copying, server sync). Failed files get
quarantined and the slot is freed for a retry from another candidate.
Three checks, in order from cheapest to most expensive:
1. **File-size sanity** — anything below ~10KB is almost certainly a
stub, broken transfer, or non-audio masquerading as audio.
2. **Mutagen parse** — catches truncated headers, corrupted streamheaders,
wrong-format files (mp3 with .flac extension, etc). If mutagen can't
parse the audio info block, the file won't import cleanly downstream.
3. **Duration agreement** — if the caller provides an expected duration
(Spotify/MusicBrainz `duration_ms`), the decoded length must agree
within tolerance. Catches truncated files whose headers parse fine
but whose audio is incomplete, and "wrong file" cases the slskd
transfer matched on a similarly-named track.
This is the "tier 1" integrity layer — universal across formats, no
external binary dep. A future tier could verify the FLAC STREAMINFO MD5
by actually decoding the audio (requires `flac` binary or libflac
wrapper); skipped for now since tier 1 catches the vast majority of
real-world corruption.
"""
from __future__ import annotations
from dataclasses import dataclass, field
from typing import Any, Dict, Optional
from utils.logging_config import get_logger
logger = get_logger("imports.file_integrity")
# Minimum plausible audio file size. A 1-second 64kbps mp3 is ~8KB; a
# 1-second FLAC is much larger. Anything under this is a broken stub.
_MIN_FILE_SIZE_BYTES = 10 * 1024
# Default tolerance for duration agreement. Most legitimate length
# variations (intro silence, encoder padding, live recording trims) sit
# inside 3 seconds. Goes up to 5s if the expected duration is itself
# long (>10 minutes) since absolute drift scales with length.
_DEFAULT_LENGTH_TOLERANCE_S = 3.0
_LENGTH_TOLERANCE_LONG_TRACK_S = 5.0
_LONG_TRACK_THRESHOLD_S = 600.0 # 10 minutes
# Upper bound for the user-configurable override. Anything past 60s
# means the check is effectively off — cap defends against accidental
# nonsense like 9999 making logs misleading. Users who genuinely want
# to disable the check can set 60.
_MAX_USER_TOLERANCE_S = 60.0
def resolve_duration_tolerance(value: Any) -> Optional[float]:
"""Coerce a user-configured tolerance value to a float override.
Returns:
- None when value is missing / 0 / negative / unparseable, so
callers fall back to the auto-scaled defaults (3s/5s).
- float in (0, _MAX_USER_TOLERANCE_S] when value is a positive
numeric string or float — clamped to the upper bound.
Pure helper. No I/O. Drives the `length_tolerance_s` override on
`check_audio_integrity`.
"""
if value is None:
return None
try:
parsed = float(value)
except (TypeError, ValueError):
return None
if parsed <= 0:
return None
if parsed > _MAX_USER_TOLERANCE_S:
return _MAX_USER_TOLERANCE_S
return parsed
def expected_duration_for_check(expected_ms: Any, is_local_import: bool) -> Optional[int]:
"""The expected duration (ms) to run the duration-agreement leg against,
or None to skip that leg.
The duration check exists to catch BROKEN slskd TRANSFERS (truncated /
wrong-file downloads). A local/manual import is the user's own already-
tagged file being sorted, not a transfer — duration-agreeing it against a
re-resolved release is meaningless and produces false quarantines (#804:
Coldplay "Yellow" album file, 269s, false-rejected against a *single*
edition's 266s). So for local imports we skip the duration leg; the
size + mutagen-parse legs still run and catch genuinely broken files.
"""
if is_local_import:
return None
try:
return int(expected_ms) or None
except (TypeError, ValueError):
return None
@dataclass
class IntegrityResult:
"""Outcome of an integrity check.
`ok` is the single bit the caller cares about. `reason` is the
human-readable explanation when `ok` is False (suitable for
quarantine sidecar / log lines / UI). `checks` carries the
per-check details — useful for debugging and tests.
"""
ok: bool
reason: str = ""
checks: Dict[str, Any] = field(default_factory=dict)
def check_audio_integrity(
file_path: str,
expected_duration_ms: Optional[int] = None,
*,
length_tolerance_s: Optional[float] = None,
min_file_size_bytes: int = _MIN_FILE_SIZE_BYTES,
) -> IntegrityResult:
"""Verify a downloaded audio file is not broken.
Args:
file_path: Path to the audio file on disk.
expected_duration_ms: Expected track length from the metadata
source (Spotify/MB/etc). If None, the duration check is
skipped and only the size + parse checks run.
length_tolerance_s: Override the default tolerance for the
duration check. None uses the auto-scaled default
(3s for normal tracks, 5s for >10min tracks).
min_file_size_bytes: Override the minimum size threshold.
Returns:
IntegrityResult with `ok`, `reason`, and per-check details.
Never raises — all errors become `ok=False` with an explanatory
reason, so callers can rely on a clean boolean.
"""
import os
checks: Dict[str, Any] = {}
# --- Check 1: file size ---
try:
size = os.path.getsize(file_path)
except OSError as exc:
return IntegrityResult(ok=False, reason=f"Cannot stat file: {exc}",
checks={"size": "stat_failed"})
checks["size_bytes"] = size
if size < min_file_size_bytes:
return IntegrityResult(
ok=False,
reason=f"File too small ({size} bytes, minimum {min_file_size_bytes}) — "
"likely truncated transfer or empty stub",
checks=checks,
)
# --- Check 2: mutagen parse ---
try:
from mutagen import File as MutagenFile
except ImportError:
# mutagen is a hard dep elsewhere in the codebase, but degrade
# gracefully if it's somehow missing — pass with a warning
# rather than failing every download.
logger.warning("[Integrity] mutagen unavailable — skipping parse check")
checks["mutagen_parse"] = "unavailable"
return IntegrityResult(ok=True, checks=checks)
try:
audio = MutagenFile(file_path)
except Exception as exc:
return IntegrityResult(
ok=False,
reason=f"Mutagen could not parse file: {exc}",
checks={**checks, "mutagen_parse": "exception"},
)
if audio is None:
return IntegrityResult(
ok=False,
reason="Mutagen could not identify file format — likely corrupted "
"or wrong file extension",
checks={**checks, "mutagen_parse": "unidentified"},
)
if audio.info is None:
return IntegrityResult(
ok=False,
reason="Mutagen parsed file but found no audio info block — "
"header damage suspected",
checks={**checks, "mutagen_parse": "no_info"},
)
actual_length_s = float(getattr(audio.info, "length", 0) or 0)
checks["actual_length_s"] = actual_length_s
if actual_length_s <= 0:
# Length 0 is NOT proof of corruption here: the file already passed the
# size gate, was identified as a real audio format, and has a valid
# info block. A genuinely empty/truncated/stub file fails one of those
# earlier checks instead. The real cause of a clean-but-zero-length
# parse is "length unknown" — fragmented / streamed FLAC carries
# total_samples=0 in its STREAMINFO even though every audio frame is
# present and the file plays fine. HiFi is the common trigger: it
# assembles FLAC from HLS segments and demuxes with `ffmpeg -c copy`,
# which preserves total_samples=0, so mutagen computes length 0 and the
# file was wrongly quarantined (#756). Treat it as unknown length:
# accept the file and skip the duration cross-check we can't perform
# without a length. mutagen never decoded/validated frame data anyway,
# so accepting here doesn't weaken real corruption detection.
logger.warning(
"[Integrity] %s parsed cleanly (%d bytes, format=%s) but reports "
"length 0 — treating as unknown length (likely streamed/fragmented "
"FLAC), not rejecting",
os.path.basename(file_path), size, type(audio).__name__,
)
return IntegrityResult(
ok=True,
checks={**checks, "mutagen_parse": "zero_length_unknown",
"length_check": "skipped_unknown_length"},
)
# --- Check 3: duration agreement (optional) ---
if expected_duration_ms is None or expected_duration_ms <= 0:
checks["length_check"] = "skipped"
return IntegrityResult(ok=True, checks=checks)
expected_length_s = expected_duration_ms / 1000.0
checks["expected_length_s"] = expected_length_s
if length_tolerance_s is None:
length_tolerance_s = (
_LENGTH_TOLERANCE_LONG_TRACK_S
if expected_length_s > _LONG_TRACK_THRESHOLD_S
else _DEFAULT_LENGTH_TOLERANCE_S
)
checks["length_tolerance_s"] = length_tolerance_s
drift_s = abs(actual_length_s - expected_length_s)
checks["length_drift_s"] = drift_s
if drift_s > length_tolerance_s:
return IntegrityResult(
ok=False,
reason=f"Duration mismatch: file is {actual_length_s:.1f}s, "
f"expected {expected_length_s:.1f}s "
f"(drift {drift_s:.1f}s > tolerance {length_tolerance_s:.1f}s) — "
"likely truncated download or wrong file matched",
checks=checks,
)
checks["length_check"] = "passed"
return IntegrityResult(ok=True, checks=checks)