From 4bce1932ba967f75f4570350ec7a7e9e7b3e99ab Mon Sep 17 00:00:00 2001 From: BoulderBadgeDad Date: Sun, 21 Jun 2026 16:56:46 -0700 Subject: [PATCH] =?UTF-8?q?HiFi:=20catch=20faked-header=20previews=20(full?= =?UTF-8?q?=20length=20claimed,=20~30s=20real=20audio)=20=E2=80=94=20#895?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The first fix (#895) only caught manifests that HONESTLY declared a short length. The real-world fakes are nastier: every length header is faked to FULL — HLS #EXTINF, the m4a moov, AND the demuxed FLAC's STREAMINFO total_samples — while the file holds only ~30s of real audio. So the manifest-sum and mutagen-length checks both read 'full' and passed it through. Measured 23 issue files: every one decoded to ~30s with a claimed full length; size/claimed gave an impossible 55-362 kbps 'lossless', size/30s gave a plausible ~600-1100 kbps. Detect it two independent ways (post-download), so it fires even when every header lies: - decoded real length via ffmpeg (-f null) vs the largest claimed length — the ground truth when a decoder is present (production demuxes with ffmpeg, so it is); - for lossless, an impossibly-low implied bitrate (< 30% of raw PCM) — needs no decoder, catches all 23 on its own. Pure is_fake_lossless_bitrate / is_preview_download / parse_ffmpeg_time helpers with seam tests pinned to the real #895 file numbers. Reference is max(expected, container-claimed) so the file's own faked claim becomes the bar its real audio must clear. --- core/hifi_client.py | 108 +++++++++++++++++++++++++++---- tests/test_hifi_preview_guard.py | 64 ++++++++++++++++++ 2 files changed, 158 insertions(+), 14 deletions(-) diff --git a/core/hifi_client.py b/core/hifi_client.py index 4450312a..4c2ed260 100644 --- a/core/hifi_client.py +++ b/core/hifi_client.py @@ -173,6 +173,48 @@ def is_short_audio(actual_seconds: float, expected_seconds: float, threshold: fl return a < e * threshold +def is_fake_lossless_bitrate(size_bytes, claimed_seconds, sample_rate, bits_per_sample, + channels, min_ratio: float = 0.30) -> bool: + """True when a 'lossless' file's data is FAR too small for its claimed length — the + fingerprint of a ~30s preview whose STREAMINFO/container was faked to the full + duration (so every length header reads 'full' and only the bitrate gives it away). + Real FLAC is ~40-75% of raw PCM; a preview padded to full length implies single-digit + %. Conservative: 0 / bad inputs return False (never reject on unknowns).""" + try: + sz, secs = float(size_bytes or 0), float(claimed_seconds or 0) + sr, bits, ch = int(sample_rate or 0), int(bits_per_sample or 0), int(channels or 0) + except (TypeError, ValueError): + return False + if sz <= 0 or secs <= 0 or sr <= 0 or bits <= 0 or ch <= 0: + return False + return (sz * 8 / secs) < (sr * bits * ch) * min_ratio + + +def parse_ffmpeg_time(stderr_text) -> float: + """The last ``time=HH:MM:SS.xx`` ffmpeg prints while decoding — the REAL decoded + length (immune to a faked container/STREAMINFO duration). 0.0 if not found.""" + last = 0.0 + for m in re.finditer(r'time=(\d+):(\d+):(\d+(?:\.\d+)?)', stderr_text or ''): + last = int(m.group(1)) * 3600 + int(m.group(2)) * 60 + float(m.group(3)) + return last + + +def is_preview_download(real_seconds, reference_seconds, *, is_lossless, size_bytes, + sample_rate, bits_per_sample, channels): + """Is a finished file a preview/truncated fake? Two independent signals, so it fires + even when the fakery declares full length at every layer: + 1. DECODED length far below the reference (the ground truth, when a decoder ran); + 2. for lossless, an impossibly-low implied bitrate (no decoder needed). + Returns ``(is_fake, reason)``.""" + if real_seconds and is_short_audio(real_seconds, reference_seconds): + return True, "decoded %.0fs of %.0fs" % (real_seconds, reference_seconds) + if is_lossless and is_fake_lossless_bitrate(size_bytes, reference_seconds, sample_rate, + bits_per_sample, channels): + kbps = (float(size_bytes) * 8 / reference_seconds / 1000) if reference_seconds else 0 + return True, "%.0fkbps lossless over %.0fs (far too low — a ~30s preview)" % (kbps, reference_seconds) + return False, "" + + # Run the new-default push at most once per process. _pushed_new_defaults = False @@ -763,15 +805,43 @@ class HiFiClient(DownloadSourcePlugin): pass return 0.0 + @staticmethod + def _find_ffmpeg(): + ff = shutil.which('ffmpeg') + if ff: + return ff + cand = Path(__file__).parent.parent / 'tools' / ('ffmpeg.exe' if os.name == 'nt' else 'ffmpeg') + return str(cand) if cand.exists() else None + + def _probe_real_seconds(self, path) -> float: + """REAL decoded audio length via ffmpeg — decodes the actual frames, so it sees + through a faked STREAMINFO/container duration (a 30s preview claiming full + length decodes to 30s). 0.0 if ffmpeg is unavailable or on error.""" + ff = self._find_ffmpeg() + if not ff: + return 0.0 + try: + proc = subprocess.run( + [ff, '-hide_banner', '-nostdin', '-i', str(path), '-map', '0:a:0', '-f', 'null', '-'], + capture_output=True, text=True, timeout=180) + return parse_ffmpeg_time(proc.stderr) + except Exception: + return 0.0 + + @staticmethod + def _flac_props(path): + """(sample_rate, bits_per_sample, channels) for the bitrate sanity check, or None.""" + try: + from mutagen.flac import FLAC + si = FLAC(str(path)).info + return (si.sample_rate, si.bits_per_sample, si.channels) + except Exception: + return None + def _demux_flac(self, input_path: Path, output_path: Path) -> None: - ffmpeg = shutil.which('ffmpeg') + ffmpeg = self._find_ffmpeg() if not ffmpeg: - tools_dir = Path(__file__).parent.parent / 'tools' - ffmpeg_candidate = tools_dir / ('ffmpeg.exe' if os.name == 'nt' else 'ffmpeg') - if ffmpeg_candidate.exists(): - ffmpeg = str(ffmpeg_candidate) - else: - raise RuntimeError('ffmpeg is required to demux FLAC from MP4. Install ffmpeg and retry.') + raise RuntimeError('ffmpeg is required to demux FLAC from MP4. Install ffmpeg and retry.') try: result = subprocess.run( @@ -1002,14 +1072,24 @@ class HiFiClient(DownloadSourcePlugin): out_path.unlink(missing_ok=True) continue - # Preview guard #2 (post-download): verify the finished file's REAL audio - # length. Backs up the manifest check for legacy/direct (no-EXTINF) - # downloads and catches any truncated/corrupt result. - actual_s = self._probe_audio_seconds(out_path) - if is_short_audio(actual_s, expected_s): + # Preview guard #2 (post-download): the real catch. HiFi previews fake the + # FULL length in every header — manifest EXTINF, m4a moov, FLAC + # total_samples — so only the DECODED audio (or, for lossless, the + # bitrate) reveals the ~30s truth. Reference = the largest length any + # header claims (so the file's own faked claim becomes the bar its real + # audio must clear); is_preview_download decodes + bitrate-checks. + ref_s = max(expected_s, self._probe_audio_seconds(out_path)) + real_s = self._probe_real_seconds(out_path) + props = self._flac_props(out_path) if is_flac else None + fake, why = is_preview_download( + real_s, ref_s, is_lossless=is_flac, size_bytes=final_size, + sample_rate=(props[0] if props else 0), + bits_per_sample=(props[1] if props else 0), + channels=(props[2] if props else 0)) + if fake: logger.warning( - "HiFi file too short at %s for '%s' (%.0fs of %.0fs) — likely a " - "preview, rejecting", q_key, display_name, actual_s, expected_s) + "HiFi served a PREVIEW/truncated file at %s for '%s' (%s) — rejecting", + q_key, display_name, why) out_path.unlink(missing_ok=True) continue diff --git a/tests/test_hifi_preview_guard.py b/tests/test_hifi_preview_guard.py index 3a3ac914..d8d7a38b 100644 --- a/tests/test_hifi_preview_guard.py +++ b/tests/test_hifi_preview_guard.py @@ -129,3 +129,67 @@ def test_download_sync_does_not_reject_when_track_length_unknown(tmp_path, monke c._download_sync('dl1', 12345, 'x') assert seg_calls # unknown length → no rejection, proceeds + + +# ── faked-header previews: claim full length everywhere, only ~30s of real audio ── +# (real numbers measured from issue #895's files: every "lossless" FLAC was a 30s +# preview with STREAMINFO total_samples faked to the full length.) +from core.hifi_client import is_fake_lossless_bitrate, is_preview_download, parse_ffmpeg_time + + +def test_real_issue895_files_are_all_flagged_by_bitrate(): + # (size_bytes, claimed_seconds) for the actual files — 16-bit/44.1kHz stereo FLAC. + samples = [ + (4_080_000, 216), # Save Your Tears (151 kbps claimed) + (6_770_000, 150), # I Ain't Worried (362 kbps — the highest, nearest the line) + (2_240_000, 326), # Lose Yourself (55 kbps) + (4_190_000, 285), # The Real Slim Shady + (4_910_000, 170), # APT + ] + for size, secs in samples: + assert is_fake_lossless_bitrate(size, secs, 44100, 16, 2) is True, (size, secs) + + +def test_real_full_lossless_is_not_flagged(): + # a genuine 16/44.1 lossless track is ~700-1100 kbps → well above the floor + assert is_fake_lossless_bitrate(25_000_000, 216, 44100, 16, 2) is False # ~926 kbps + assert is_fake_lossless_bitrate(12_000_000, 216, 44100, 16, 2) is False # ~444 kbps, still real + + +def test_bitrate_check_is_conservative_on_unknowns(): + assert is_fake_lossless_bitrate(0, 216, 44100, 16, 2) is False + assert is_fake_lossless_bitrate(4_080_000, 0, 44100, 16, 2) is False + assert is_fake_lossless_bitrate(4_080_000, 216, 0, 0, 0) is False + + +def test_is_preview_download_decode_path(): + # decoded 30s of a claimed 216s → fake, regardless of bitrate + fake, why = is_preview_download(30.0, 216.0, is_lossless=False, size_bytes=99_000_000, + sample_rate=44100, bits_per_sample=16, channels=2) + assert fake and "decoded 30s of 216s" in why + + +def test_is_preview_download_bitrate_path_when_no_decoder(): + # real_seconds=0 (no ffmpeg) → fall back to the lossless bitrate check + fake, why = is_preview_download(0.0, 216.0, is_lossless=True, size_bytes=4_080_000, + sample_rate=44100, bits_per_sample=16, channels=2) + assert fake and "kbps lossless" in why + + +def test_is_preview_download_passes_a_real_file(): + fake, _ = is_preview_download(214.0, 216.0, is_lossless=True, size_bytes=25_000_000, + sample_rate=44100, bits_per_sample=16, channels=2) + assert fake is False + + +def test_is_preview_download_lossy_no_decoder_is_not_flagged(): + # a lossy tier (mp3/m4a) with no decode info → can't bitrate-check → don't reject + fake, _ = is_preview_download(0.0, 216.0, is_lossless=False, size_bytes=2_000_000, + sample_rate=44100, bits_per_sample=16, channels=2) + assert fake is False + + +def test_parse_ffmpeg_time_reads_the_last_progress_line(): + stderr = "frame= ... time=00:00:12.34 bitrate=...\nframe= ... time=00:00:30.05 bitrate=..." + assert abs(parse_ffmpeg_time(stderr) - 30.05) < 0.01 + assert parse_ffmpeg_time("no time here") == 0.0