HiFi: catch faked-header previews (full length claimed, ~30s real audio) — #895
The first fix (#895) only caught manifests that HONESTLY declared a short length. The real-world fakes are nastier: every length header is faked to FULL — HLS #EXTINF, the m4a moov, AND the demuxed FLAC's STREAMINFO total_samples — while the file holds only ~30s of real audio. So the manifest-sum and mutagen-length checks both read 'full' and passed it through. Measured 23 issue files: every one decoded to ~30s with a claimed full length; size/claimed gave an impossible 55-362 kbps 'lossless', size/30s gave a plausible ~600-1100 kbps. Detect it two independent ways (post-download), so it fires even when every header lies: - decoded real length via ffmpeg (-f null) vs the largest claimed length — the ground truth when a decoder is present (production demuxes with ffmpeg, so it is); - for lossless, an impossibly-low implied bitrate (< 30% of raw PCM) — needs no decoder, catches all 23 on its own. Pure is_fake_lossless_bitrate / is_preview_download / parse_ffmpeg_time helpers with seam tests pinned to the real #895 file numbers. Reference is max(expected, container-claimed) so the file's own faked claim becomes the bar its real audio must clear.
This commit is contained in:
BoulderBadgeDad
parent
1249160ff5
commit
4bce1932ba
2 changed files with 158 additions and 14 deletions
|
|
@ -173,6 +173,48 @@ def is_short_audio(actual_seconds: float, expected_seconds: float, threshold: fl
|
|||
return a < e * threshold
|
||||
|
||||
|
||||
def is_fake_lossless_bitrate(size_bytes, claimed_seconds, sample_rate, bits_per_sample,
|
||||
channels, min_ratio: float = 0.30) -> bool:
|
||||
"""True when a 'lossless' file's data is FAR too small for its claimed length — the
|
||||
fingerprint of a ~30s preview whose STREAMINFO/container was faked to the full
|
||||
duration (so every length header reads 'full' and only the bitrate gives it away).
|
||||
Real FLAC is ~40-75% of raw PCM; a preview padded to full length implies single-digit
|
||||
%. Conservative: 0 / bad inputs return False (never reject on unknowns)."""
|
||||
try:
|
||||
sz, secs = float(size_bytes or 0), float(claimed_seconds or 0)
|
||||
sr, bits, ch = int(sample_rate or 0), int(bits_per_sample or 0), int(channels or 0)
|
||||
except (TypeError, ValueError):
|
||||
return False
|
||||
if sz <= 0 or secs <= 0 or sr <= 0 or bits <= 0 or ch <= 0:
|
||||
return False
|
||||
return (sz * 8 / secs) < (sr * bits * ch) * min_ratio
|
||||
|
||||
|
||||
def parse_ffmpeg_time(stderr_text) -> float:
|
||||
"""The last ``time=HH:MM:SS.xx`` ffmpeg prints while decoding — the REAL decoded
|
||||
length (immune to a faked container/STREAMINFO duration). 0.0 if not found."""
|
||||
last = 0.0
|
||||
for m in re.finditer(r'time=(\d+):(\d+):(\d+(?:\.\d+)?)', stderr_text or ''):
|
||||
last = int(m.group(1)) * 3600 + int(m.group(2)) * 60 + float(m.group(3))
|
||||
return last
|
||||
|
||||
|
||||
def is_preview_download(real_seconds, reference_seconds, *, is_lossless, size_bytes,
|
||||
sample_rate, bits_per_sample, channels):
|
||||
"""Is a finished file a preview/truncated fake? Two independent signals, so it fires
|
||||
even when the fakery declares full length at every layer:
|
||||
1. DECODED length far below the reference (the ground truth, when a decoder ran);
|
||||
2. for lossless, an impossibly-low implied bitrate (no decoder needed).
|
||||
Returns ``(is_fake, reason)``."""
|
||||
if real_seconds and is_short_audio(real_seconds, reference_seconds):
|
||||
return True, "decoded %.0fs of %.0fs" % (real_seconds, reference_seconds)
|
||||
if is_lossless and is_fake_lossless_bitrate(size_bytes, reference_seconds, sample_rate,
|
||||
bits_per_sample, channels):
|
||||
kbps = (float(size_bytes) * 8 / reference_seconds / 1000) if reference_seconds else 0
|
||||
return True, "%.0fkbps lossless over %.0fs (far too low — a ~30s preview)" % (kbps, reference_seconds)
|
||||
return False, ""
|
||||
|
||||
|
||||
# Run the new-default push at most once per process.
|
||||
_pushed_new_defaults = False
|
||||
|
||||
|
|
@ -763,15 +805,43 @@ class HiFiClient(DownloadSourcePlugin):
|
|||
pass
|
||||
return 0.0
|
||||
|
||||
@staticmethod
|
||||
def _find_ffmpeg():
|
||||
ff = shutil.which('ffmpeg')
|
||||
if ff:
|
||||
return ff
|
||||
cand = Path(__file__).parent.parent / 'tools' / ('ffmpeg.exe' if os.name == 'nt' else 'ffmpeg')
|
||||
return str(cand) if cand.exists() else None
|
||||
|
||||
def _probe_real_seconds(self, path) -> float:
|
||||
"""REAL decoded audio length via ffmpeg — decodes the actual frames, so it sees
|
||||
through a faked STREAMINFO/container duration (a 30s preview claiming full
|
||||
length decodes to 30s). 0.0 if ffmpeg is unavailable or on error."""
|
||||
ff = self._find_ffmpeg()
|
||||
if not ff:
|
||||
return 0.0
|
||||
try:
|
||||
proc = subprocess.run(
|
||||
[ff, '-hide_banner', '-nostdin', '-i', str(path), '-map', '0:a:0', '-f', 'null', '-'],
|
||||
capture_output=True, text=True, timeout=180)
|
||||
return parse_ffmpeg_time(proc.stderr)
|
||||
except Exception:
|
||||
return 0.0
|
||||
|
||||
@staticmethod
|
||||
def _flac_props(path):
|
||||
"""(sample_rate, bits_per_sample, channels) for the bitrate sanity check, or None."""
|
||||
try:
|
||||
from mutagen.flac import FLAC
|
||||
si = FLAC(str(path)).info
|
||||
return (si.sample_rate, si.bits_per_sample, si.channels)
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
def _demux_flac(self, input_path: Path, output_path: Path) -> None:
|
||||
ffmpeg = shutil.which('ffmpeg')
|
||||
ffmpeg = self._find_ffmpeg()
|
||||
if not ffmpeg:
|
||||
tools_dir = Path(__file__).parent.parent / 'tools'
|
||||
ffmpeg_candidate = tools_dir / ('ffmpeg.exe' if os.name == 'nt' else 'ffmpeg')
|
||||
if ffmpeg_candidate.exists():
|
||||
ffmpeg = str(ffmpeg_candidate)
|
||||
else:
|
||||
raise RuntimeError('ffmpeg is required to demux FLAC from MP4. Install ffmpeg and retry.')
|
||||
raise RuntimeError('ffmpeg is required to demux FLAC from MP4. Install ffmpeg and retry.')
|
||||
|
||||
try:
|
||||
result = subprocess.run(
|
||||
|
|
@ -1002,14 +1072,24 @@ class HiFiClient(DownloadSourcePlugin):
|
|||
out_path.unlink(missing_ok=True)
|
||||
continue
|
||||
|
||||
# Preview guard #2 (post-download): verify the finished file's REAL audio
|
||||
# length. Backs up the manifest check for legacy/direct (no-EXTINF)
|
||||
# downloads and catches any truncated/corrupt result.
|
||||
actual_s = self._probe_audio_seconds(out_path)
|
||||
if is_short_audio(actual_s, expected_s):
|
||||
# Preview guard #2 (post-download): the real catch. HiFi previews fake the
|
||||
# FULL length in every header — manifest EXTINF, m4a moov, FLAC
|
||||
# total_samples — so only the DECODED audio (or, for lossless, the
|
||||
# bitrate) reveals the ~30s truth. Reference = the largest length any
|
||||
# header claims (so the file's own faked claim becomes the bar its real
|
||||
# audio must clear); is_preview_download decodes + bitrate-checks.
|
||||
ref_s = max(expected_s, self._probe_audio_seconds(out_path))
|
||||
real_s = self._probe_real_seconds(out_path)
|
||||
props = self._flac_props(out_path) if is_flac else None
|
||||
fake, why = is_preview_download(
|
||||
real_s, ref_s, is_lossless=is_flac, size_bytes=final_size,
|
||||
sample_rate=(props[0] if props else 0),
|
||||
bits_per_sample=(props[1] if props else 0),
|
||||
channels=(props[2] if props else 0))
|
||||
if fake:
|
||||
logger.warning(
|
||||
"HiFi file too short at %s for '%s' (%.0fs of %.0fs) — likely a "
|
||||
"preview, rejecting", q_key, display_name, actual_s, expected_s)
|
||||
"HiFi served a PREVIEW/truncated file at %s for '%s' (%s) — rejecting",
|
||||
q_key, display_name, why)
|
||||
out_path.unlink(missing_ok=True)
|
||||
continue
|
||||
|
||||
|
|
|
|||
|
|
@ -129,3 +129,67 @@ def test_download_sync_does_not_reject_when_track_length_unknown(tmp_path, monke
|
|||
|
||||
c._download_sync('dl1', 12345, 'x')
|
||||
assert seg_calls # unknown length → no rejection, proceeds
|
||||
|
||||
|
||||
# ── faked-header previews: claim full length everywhere, only ~30s of real audio ──
|
||||
# (real numbers measured from issue #895's files: every "lossless" FLAC was a 30s
|
||||
# preview with STREAMINFO total_samples faked to the full length.)
|
||||
from core.hifi_client import is_fake_lossless_bitrate, is_preview_download, parse_ffmpeg_time
|
||||
|
||||
|
||||
def test_real_issue895_files_are_all_flagged_by_bitrate():
|
||||
# (size_bytes, claimed_seconds) for the actual files — 16-bit/44.1kHz stereo FLAC.
|
||||
samples = [
|
||||
(4_080_000, 216), # Save Your Tears (151 kbps claimed)
|
||||
(6_770_000, 150), # I Ain't Worried (362 kbps — the highest, nearest the line)
|
||||
(2_240_000, 326), # Lose Yourself (55 kbps)
|
||||
(4_190_000, 285), # The Real Slim Shady
|
||||
(4_910_000, 170), # APT
|
||||
]
|
||||
for size, secs in samples:
|
||||
assert is_fake_lossless_bitrate(size, secs, 44100, 16, 2) is True, (size, secs)
|
||||
|
||||
|
||||
def test_real_full_lossless_is_not_flagged():
|
||||
# a genuine 16/44.1 lossless track is ~700-1100 kbps → well above the floor
|
||||
assert is_fake_lossless_bitrate(25_000_000, 216, 44100, 16, 2) is False # ~926 kbps
|
||||
assert is_fake_lossless_bitrate(12_000_000, 216, 44100, 16, 2) is False # ~444 kbps, still real
|
||||
|
||||
|
||||
def test_bitrate_check_is_conservative_on_unknowns():
|
||||
assert is_fake_lossless_bitrate(0, 216, 44100, 16, 2) is False
|
||||
assert is_fake_lossless_bitrate(4_080_000, 0, 44100, 16, 2) is False
|
||||
assert is_fake_lossless_bitrate(4_080_000, 216, 0, 0, 0) is False
|
||||
|
||||
|
||||
def test_is_preview_download_decode_path():
|
||||
# decoded 30s of a claimed 216s → fake, regardless of bitrate
|
||||
fake, why = is_preview_download(30.0, 216.0, is_lossless=False, size_bytes=99_000_000,
|
||||
sample_rate=44100, bits_per_sample=16, channels=2)
|
||||
assert fake and "decoded 30s of 216s" in why
|
||||
|
||||
|
||||
def test_is_preview_download_bitrate_path_when_no_decoder():
|
||||
# real_seconds=0 (no ffmpeg) → fall back to the lossless bitrate check
|
||||
fake, why = is_preview_download(0.0, 216.0, is_lossless=True, size_bytes=4_080_000,
|
||||
sample_rate=44100, bits_per_sample=16, channels=2)
|
||||
assert fake and "kbps lossless" in why
|
||||
|
||||
|
||||
def test_is_preview_download_passes_a_real_file():
|
||||
fake, _ = is_preview_download(214.0, 216.0, is_lossless=True, size_bytes=25_000_000,
|
||||
sample_rate=44100, bits_per_sample=16, channels=2)
|
||||
assert fake is False
|
||||
|
||||
|
||||
def test_is_preview_download_lossy_no_decoder_is_not_flagged():
|
||||
# a lossy tier (mp3/m4a) with no decode info → can't bitrate-check → don't reject
|
||||
fake, _ = is_preview_download(0.0, 216.0, is_lossless=False, size_bytes=2_000_000,
|
||||
sample_rate=44100, bits_per_sample=16, channels=2)
|
||||
assert fake is False
|
||||
|
||||
|
||||
def test_parse_ffmpeg_time_reads_the_last_progress_line():
|
||||
stderr = "frame= ... time=00:00:12.34 bitrate=...\nframe= ... time=00:00:30.05 bitrate=..."
|
||||
assert abs(parse_ffmpeg_time(stderr) - 30.05) < 0.01
|
||||
assert parse_ffmpeg_time("no time here") == 0.0
|
||||
|
|
|
|||
Loading…
Reference in a new issue