From c32fe219fe95fc0df55d3c0646af742170c30c6e Mon Sep 17 00:00:00 2001 From: dev Date: Sun, 14 Jun 2026 14:35:21 +0200 Subject: [PATCH] fix(imports): silence guard catches mostly-silent preview/truncated files MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit HiFi/Monochrome HLS assembly can produce a file with the correct container duration but only ~30s of real audio + silence padding — the duration and quality guards both pass, so nothing caught it until you listened. Add core/imports/silence.py: ffmpeg silencedetect over the audio, reject when the silent fraction exceeds 50%. Wire it into the post-download pipeline with the same quarantine + next-candidate retry pattern as the quality guard (trigger='silence'), and surface it via import_rejection_reason. Fails open when ffmpeg/mutagen are unavailable so tooling problems never quarantine a legit file. Also mark 'quality filter' and 'silence guard' failures as recoverable quarantine rows in the downloads UI (were shown as plain failures). Verified end-to-end: a 30s-tone + 180s-silence FLAC is flagged '86% silence (only ~30s audible of 210s)'; a 210s tone passes. 7 parser unit tests. Co-Authored-By: Claude Opus 4.8 --- core/imports/pipeline.py | 48 +++++++++++ core/imports/silence.py | 129 ++++++++++++++++++++++++++++ tests/imports/test_silence_guard.py | 59 +++++++++++++ webui/static/downloads.js | 2 +- 4 files changed, 237 insertions(+), 1 deletion(-) create mode 100644 core/imports/silence.py create mode 100644 tests/imports/test_silence_guard.py diff --git a/core/imports/pipeline.py b/core/imports/pipeline.py index 10883a05..f816207a 100644 --- a/core/imports/pipeline.py +++ b/core/imports/pipeline.py @@ -35,6 +35,7 @@ from core.imports.context import ( from core.imports.file_integrity import check_audio_integrity, expected_duration_for_check, resolve_duration_tolerance from core.imports.filename import extract_track_number_from_filename from core.imports.guards import check_flac_bit_depth, check_quality_target, move_to_quarantine +from core.imports.silence import detect_mostly_silent from core.imports.quarantine import ( approve_quarantine_entry, entry_id_from_quarantined_filename, @@ -161,6 +162,8 @@ def import_rejection_reason(context: dict) -> str | None: ) if context.get('_bitdepth_rejected'): return "rejected by quality filter" + if context.get('_silence_rejected'): + return "rejected by silence guard (file is mostly silence)" if context.get('_race_guard_failed'): return "source file disappeared before import completed" return None @@ -628,6 +631,51 @@ def post_process_matched_download(context_key, context, file_path, runtime, meta _notify_download_completed(batch_id, task_id, success=False) return + # Silence guard — catch preview/truncated files that are mostly silence + # (correct container duration but only ~30s real audio, e.g. HiFi/ + # Monochrome HLS padding). Same quarantine + next-candidate retry + # pattern as the quality guard above. + _skip_silence = _should_skip_quarantine_check(context, 'silence') + silence_reason = None if _skip_silence else detect_mostly_silent(file_path) + if silence_reason: + try: + quarantine_path = move_to_quarantine( + file_path, context, silence_reason, automation_engine, + trigger='silence', + ) + _mark_task_quarantined(context, quarantine_path) + logger.warning("File quarantined — mostly silent: %s", quarantine_path) + except Exception as quarantine_error: + logger.error(f"Quarantine failed ({quarantine_error}), deleting file: {file_path}") + try: + os.remove(file_path) + except Exception as e: + logger.debug("delete quarantine fallback: %s", e) + + context['_silence_rejected'] = True + task_id = context.get('task_id') + batch_id = context.get('batch_id') + with matched_context_lock: + matched_downloads_context.pop(context_key, None) + + # Try the next-best candidate before giving up — same pattern as + # the AcoustID / integrity / quality failures. + if _requeue_quarantined_task_for_retry(task_id, batch_id, 'silence'): + logger.info( + "Mostly-silent file for task %s — retrying next-best candidate: %s", + task_id, silence_reason, + ) + return + + if task_id: + with tasks_lock: + if task_id in download_tasks: + download_tasks[task_id]['status'] = 'failed' + download_tasks[task_id]['error_message'] = f"Silence guard: {silence_reason}" + if task_id and batch_id: + _notify_download_completed(batch_id, task_id, success=False) + return + file_ext = os.path.splitext(file_path)[1] clean_track_name = get_import_clean_title( context, diff --git a/core/imports/silence.py b/core/imports/silence.py new file mode 100644 index 00000000..dbef1d4a --- /dev/null +++ b/core/imports/silence.py @@ -0,0 +1,129 @@ +"""Silence guard — detect files whose container duration looks right but whose +audio is mostly silence. + +Motivating bug: HiFi/Monochrome HLS assembly can yield a file with the full +track duration in its container while only the first ~30s carry real audio and +the rest is silence. The duration-agreement and quality guards both pass (the +container says 3:30 and the format/bitrate are fine), so nothing catches it — +until you listen. This guard runs ffmpeg ``silencedetect`` over the real audio +and flags a file whose silent fraction exceeds a threshold. + +The parser (``silence_ratio_from_output``) is pure and unit-tested; the ffmpeg +invocation is integration glue that fails open (returns None) when ffmpeg or +mutagen can't run, so it never blocks a legitimate import on tooling problems. +""" + +from __future__ import annotations + +import re +import subprocess +from typing import Optional + +from utils.logging_config import get_logger + +logger = get_logger("imports.silence") + +# Defaults: treat audio below -50 dB lasting >= 2s as silence, and reject when +# more than half the track is silent. A normal song — even with quiet intros/ +# outros — sits far below 0.5; a 30s-real + padded-silence file sits near 0.85. +DEFAULT_NOISE_DB = -50 +DEFAULT_MIN_SILENCE_S = 2.0 +DEFAULT_THRESHOLD = 0.5 + +_SILENCE_DURATION_RE = re.compile(r"silence_duration:\s*([0-9]+(?:\.[0-9]+)?)") + + +def silence_ratio_from_output(ffmpeg_stderr: str, total_duration_s: float) -> float: + """Fraction of *total_duration_s* covered by detected silence. + + Sums every ``silence_duration: X`` reported by ffmpeg ``silencedetect`` + and divides by the track length. Capped at 1.0; returns 0.0 when the + duration is unknown/zero or no silence was reported. + """ + if not total_duration_s or total_duration_s <= 0: + return 0.0 + total_silence = sum(float(m) for m in _SILENCE_DURATION_RE.findall(ffmpeg_stderr or "")) + if total_silence <= 0: + return 0.0 + return min(total_silence / total_duration_s, 1.0) + + +def is_mostly_silent_reason( + ffmpeg_stderr: str, + total_duration_s: float, + *, + threshold: float = DEFAULT_THRESHOLD, +) -> Optional[str]: + """Return a rejection reason when the silent fraction meets *threshold*.""" + ratio = silence_ratio_from_output(ffmpeg_stderr, total_duration_s) + if ratio >= threshold: + pct = round(ratio * 100) + audible_s = round(total_duration_s * (1 - ratio)) + return ( + f"Audio is mostly silent: {pct}% silence (only ~{audible_s}s audible of " + f"{round(total_duration_s)}s) — likely a truncated/preview file padded " + f"to full length" + ) + return None + + +def _ffmpeg_available() -> bool: + try: + subprocess.run( + ["ffmpeg", "-version"], + stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, + timeout=10, check=True, + ) + return True + except (FileNotFoundError, subprocess.SubprocessError, OSError): + return False + + +def _probe_duration_s(file_path: str) -> Optional[float]: + try: + from mutagen import File as MutagenFile + audio = MutagenFile(file_path) + if audio and audio.info and getattr(audio.info, "length", None): + return float(audio.info.length) + except Exception as exc: # pragma: no cover - defensive + logger.debug("silence guard duration probe failed for %s: %s", file_path, exc) + return None + + +def detect_mostly_silent( + file_path: str, + *, + threshold: float = DEFAULT_THRESHOLD, + noise_db: int = DEFAULT_NOISE_DB, + min_silence_s: float = DEFAULT_MIN_SILENCE_S, +) -> Optional[str]: + """Run ffmpeg ``silencedetect`` over *file_path* and return a rejection + reason when the file is mostly silence, else None. + + Fails open: returns None when ffmpeg/mutagen are unavailable or error, so + a tooling problem never quarantines a legitimate file. + """ + if not _ffmpeg_available(): + logger.debug("silence guard skipped — ffmpeg not available") + return None + + total_duration_s = _probe_duration_s(file_path) + if not total_duration_s: + return None + + try: + proc = subprocess.run( + [ + "ffmpeg", "-hide_banner", "-nostats", "-i", file_path, + "-af", f"silencedetect=noise={noise_db}dB:d={min_silence_s}", + "-f", "null", "-", + ], + stdout=subprocess.DEVNULL, stderr=subprocess.PIPE, + timeout=120, + ) + except (subprocess.SubprocessError, OSError) as exc: + logger.debug("silence guard ffmpeg run failed for %s: %s", file_path, exc) + return None + + stderr = proc.stderr.decode("utf-8", errors="replace") if proc.stderr else "" + return is_mostly_silent_reason(stderr, total_duration_s, threshold=threshold) diff --git a/tests/imports/test_silence_guard.py b/tests/imports/test_silence_guard.py new file mode 100644 index 00000000..2a596aa6 --- /dev/null +++ b/tests/imports/test_silence_guard.py @@ -0,0 +1,59 @@ +"""Silence guard — catches files whose container duration is correct but whose +audio is mostly silence (e.g. HiFi/Monochrome 30s-preview padded out to the +full track length). Pure parser is tested here; the ffmpeg call is integration. +""" + +import pytest + +from core.imports.silence import silence_ratio_from_output, is_mostly_silent_reason + + +_ONE_LONG_TAIL = """ +Input #0, flac, from 'song.flac': +[silencedetect @ 0x55] silence_start: 31.512 +[silencedetect @ 0x55] silence_end: 210.300 | silence_duration: 178.788 +""" + +_TWO_GAPS = """ +[silencedetect @ 0x55] silence_start: 0 +[silencedetect @ 0x55] silence_end: 1.5 | silence_duration: 1.5 +[silencedetect @ 0x55] silence_start: 200 +[silencedetect @ 0x55] silence_end: 203 | silence_duration: 3.0 +""" + +_NO_SILENCE = "Input #0, flac\n[some other ffmpeg chatter]\n" + + +def test_ratio_single_long_trailing_silence(): + r = silence_ratio_from_output(_ONE_LONG_TAIL, total_duration_s=210.3) + assert r == pytest.approx(178.788 / 210.3, rel=1e-3) + assert r > 0.8 + + +def test_ratio_sums_multiple_silences(): + r = silence_ratio_from_output(_TWO_GAPS, total_duration_s=210.0) + assert r == pytest.approx(4.5 / 210.0, rel=1e-3) + + +def test_ratio_no_silence_is_zero(): + assert silence_ratio_from_output(_NO_SILENCE, total_duration_s=210.0) == 0.0 + + +def test_ratio_zero_duration_is_zero(): + assert silence_ratio_from_output(_ONE_LONG_TAIL, total_duration_s=0) == 0.0 + + +def test_ratio_capped_at_one(): + # Defensive: bogus silence longer than the track can't exceed 1.0. + out = "[silencedetect @ 0x] silence_end: 999 | silence_duration: 999\n" + assert silence_ratio_from_output(out, total_duration_s=210.0) == 1.0 + + +def test_reason_when_mostly_silent(): + reason = is_mostly_silent_reason(_ONE_LONG_TAIL, total_duration_s=210.3, threshold=0.5) + assert reason is not None + assert "silent" in reason.lower() + + +def test_no_reason_for_normal_song(): + assert is_mostly_silent_reason(_TWO_GAPS, total_duration_s=210.0, threshold=0.5) is None diff --git a/webui/static/downloads.js b/webui/static/downloads.js index 26303e3b..9721d122 100644 --- a/webui/static/downloads.js +++ b/webui/static/downloads.js @@ -3711,7 +3711,7 @@ function processModalStatusUpdate(playlistId, data) { // Distinguish quarantine outcomes from generic // failures — the file is recoverable, not lost. const _em = (task.error_message || '').toLowerCase(); - if (_em.includes('integrity check failed') || _em.includes('bit depth filter') || _em.includes('verification failed') || _em.includes('quarantin')) { + if (_em.includes('integrity check failed') || _em.includes('bit depth filter') || _em.includes('verification failed') || _em.includes('quality filter') || _em.includes('silence guard') || _em.includes('quarantin')) { isQuarantinedTask = true; statusText = '🛡️ Quarantined'; } else {