soulsync/core/imports/file_ops.py
BoulderBadgeDad 4af7600fd5 lossy copy: support all lossless formats, not just FLAC (#941)
radoslav-orlov: "create lossy copies of lossless tracks" only recognized FLAC, even though ALAC/
WAV/AIFF/DSD are now quality-profile formats. the FLAC knowledge was hardcoded in 3 separate
places (the import path, the Lossy Converter scan, and the fix executor) — exactly how a format
gets added in one spot but not another.

kettui-style fix — one canonical seam both sites route through, instead of 3 more string edits:

- new core/quality/lossless.py: is_lossless_format / is_lossless_audio_path (pure; injects a
  codec probe for the ambiguous .m4a/.mp4 — ALAC vs AAC — so the decision stays testable with no
  I/O), LOSSLESS_FORMATS (single source of truth, derived-consistent with model.tier_score), and
  the lossy_output_would_overwrite_source safety invariant.
- create_lossy_copy + the Lossy Converter scan + repair_worker._fix_missing_lossy_copy all route
  through it. SQL pre-filters by candidate extensions, then each file is confirmed (probing .m4a).
- SAFETY: a lossy copy must never be written over its own source — an .m4a ALAC source + AAC
  target lands on the same .m4a path, and ffmpeg runs with -y. all three sites now bail on the
  overwrite case BEFORE ffmpeg (the existing delete-original guard was too late — the source was
  already clobbered). dropped a vestigial mutagen FLAC import; updated FLAC-only UI strings.

19 tests: full seam coverage (formats, the .m4a ALAC/AAC probe branch, candidate extensions, the
overwrite guard), a tier-model consistency test that fails if the lossless set drifts, and import-
site wiring tests — WAV now converts (was rejected), and the .m4a-ALAC+AAC overwrite case proves
ffmpeg NEVER runs. 286 quality/import/repair tests green, ruff clean.
2026-06-28 12:13:05 -07:00

660 lines
26 KiB
Python

"""File operation helpers for the import flow."""
from __future__ import annotations
import errno
import logging
import os
import re
import shutil
import subprocess
import time
from pathlib import Path
from typing import Iterable, List
from config.settings import config_manager
logger = logging.getLogger("imports.file_ops")
# slskd appends "_<19-digit unix-nanosecond timestamp>" to a downloaded
# filename when the destination already contains a file with the same
# name (concurrent downloads of the same track, partial-file retries
# after a connection drop, cancelled-then-redownloaded files, the same
# track surfacing in multiple synced playlists, etc.). The original
# canonical file usually gets imported and moved into the library while
# the timestamp-suffixed siblings sit orphaned in the downloads folder
# forever. Match the suffix conservatively (≥ 18 digits) so genuine
# user filenames containing trailing numbers don't get hit.
_SLSKD_DEDUP_SUFFIX_RE = re.compile(r"_\d{18,}$")
def _strip_slskd_dedup_suffix(stem: str) -> str:
"""Return the canonical stem with any slskd dedup suffix removed."""
return _SLSKD_DEDUP_SUFFIX_RE.sub("", stem)
def cleanup_slskd_dedup_siblings(source_path) -> List[str]:
"""Remove orphan ``<basename>_<timestamp>.<ext>`` siblings of a just-
imported file from the source directory.
Call this AFTER a successful import (the canonical file has already
moved away) using the path the canonical file came from. Looks at
siblings in the same directory whose stem, with the slskd dedup
suffix stripped, equals the imported file's canonical stem and the
same extension. Deletes them.
Returns the list of deleted paths so the caller can log a summary.
Failures (permissions, racing reader, etc.) are swallowed
individually so a single locked file doesn't block the rest of the
cleanup.
"""
source = Path(source_path)
parent = source.parent
if not parent.is_dir():
return []
canonical_name = source.name
canonical_stem, canonical_ext = os.path.splitext(canonical_name)
# If the imported file ITSELF already had a dedup suffix, the
# "canonical" name is the stripped form — every other sibling that
# also strips down to it is redundant.
canonical_stem = _strip_slskd_dedup_suffix(canonical_stem)
deleted: List[str] = []
try:
children: Iterable[Path] = list(parent.iterdir())
except OSError as e:
logger.debug(f"[Dedup Cleanup] could not list {parent}: {e}")
return []
for sibling in children:
if not sibling.is_file():
continue
# Skip the imported file itself if it's still on disk (it
# shouldn't be — caller invokes us after the move — but the
# check is cheap and keeps the function safe to call from
# other contexts later).
if sibling.name == canonical_name:
continue
sib_stem, sib_ext = os.path.splitext(sibling.name)
if sib_ext.lower() != canonical_ext.lower():
continue
sib_canonical_stem = _strip_slskd_dedup_suffix(sib_stem)
if sib_canonical_stem != canonical_stem:
continue
# Defensive: don't delete a file that doesn't actually carry
# the slskd dedup suffix — that would imply it's a legitimate
# different file the user intentionally placed there.
if sib_stem == sib_canonical_stem:
continue
try:
sibling.unlink()
deleted.append(str(sibling))
except OSError as e:
logger.debug(f"[Dedup Cleanup] could not remove {sibling}: {e}")
if deleted:
logger.info(
"[Dedup Cleanup] removed %d slskd dedup orphan(s) for %r",
len(deleted),
canonical_name,
)
return deleted
def _atomic_cross_device_move(src: Path, dst: Path) -> None:
"""Move ``src`` to ``dst`` across filesystems WITHOUT ever exposing a partial file at
the final path.
Copies into a hidden temp sibling of ``dst`` (same filesystem), fsyncs, then does an
atomic ``os.replace`` into place, then deletes ``src``. A media-server file watcher
(Jellyfin/Plex real-time monitoring) therefore only ever indexes the COMPLETE file —
an incremental in-place copy was what Jellyfin could catch mid-write and cache with
null/incomplete metadata (tracks landing with no disc). Cleans up the temp on failure.
"""
src, dst = Path(src), Path(dst)
tmp = dst.parent / f".{dst.name}.ssync-tmp"
try:
with open(src, "rb") as f_src, open(tmp, "wb") as f_dst:
shutil.copyfileobj(f_src, f_dst)
f_dst.flush()
os.fsync(f_dst.fileno())
try:
shutil.copystat(str(src), str(tmp)) # preserve mtime/permissions (copy2-like)
except OSError:
pass
os.replace(str(tmp), str(dst)) # atomic within dst's filesystem
except Exception:
try:
if tmp.exists():
tmp.unlink()
except OSError:
pass
raise
try:
src.unlink()
except OSError:
logger.info(f"Could not delete source after cross-device move (may be owned by another process): {src}")
def safe_move_file(src, dst):
"""Move a file safely across filesystems."""
src = Path(src)
dst = Path(dst)
dst.parent.mkdir(parents=True, exist_ok=True)
if not src.exists():
if dst.exists():
logger.info(f"Source gone but destination exists, file already transferred: {dst.name}")
return
raise FileNotFoundError(f"Source file not found and destination does not exist: {src}")
if dst.exists():
for _attempt in range(3):
try:
dst.unlink()
break
except PermissionError:
if _attempt < 2:
time.sleep(1)
else:
logger.warning(f"Could not remove locked destination after 3 attempts: {dst.name}")
except Exception:
break
try:
# Same-filesystem move: an atomic rename that also overwrites dst. A media-server
# watcher (Jellyfin/Plex real-time monitoring) therefore never sees a partial file
# at the final name. Cross-filesystem raises EXDEV (some network mounts raise
# EPERM/EACCES) and we copy atomically below instead of letting the move write the
# destination incrementally — the partial-file-at-final-name is what caused tracks
# to land in Jellyfin with null/incomplete metadata (no disc).
os.replace(str(src), str(dst))
return
except FileNotFoundError:
if dst.exists():
logger.info(f"Source moved by another thread, destination exists: {dst.name}")
return
raise
except (OSError, PermissionError) as e:
if dst.exists() and dst.stat().st_size > 0:
logger.warning(f"Move raised {type(e).__name__} but destination exists, treating as success: {e}")
try:
src.unlink()
except Exception:
logger.info(f"Could not delete source file (may be owned by another process): {src}")
return
error_msg = str(e).lower()
cross_device = (
getattr(e, "errno", None) in (errno.EXDEV, errno.EPERM, errno.EACCES)
or "cross-device" in error_msg
or "operation not permitted" in error_msg
or "permission denied" in error_msg
)
if cross_device:
logger.warning(f"Cross-device move, using atomic copy+rename: {e}")
try:
_atomic_cross_device_move(src, dst)
logger.info(f"Successfully moved file atomically across filesystems: {src} -> {dst}")
return
except Exception as fallback_error:
logger.error(f"Atomic cross-device move failed: {fallback_error}")
raise
raise
def cleanup_empty_directories(download_path, moved_file_path):
"""Remove empty directories after a move, ignoring hidden files."""
try:
current_dir = os.path.dirname(moved_file_path)
while current_dir != download_path and current_dir.startswith(download_path):
is_empty = not any(not f.startswith(".") for f in os.listdir(current_dir))
if is_empty:
logger.warning(f"Removing empty directory: {current_dir}")
os.rmdir(current_dir)
current_dir = os.path.dirname(current_dir)
else:
break
except Exception as e:
logger.error(f"An error occurred during directory cleanup: {e}")
def get_audio_quality_string(file_path):
"""Return a compact audio quality string for the given file."""
try:
ext = os.path.splitext(file_path)[1].lower()
if ext == ".flac":
from mutagen.flac import FLAC
audio = FLAC(file_path)
sr = getattr(audio.info, "sample_rate", 0) or 0
if sr:
return f"FLAC {audio.info.bits_per_sample}bit/{sr / 1000:g}kHz"
return f"FLAC {audio.info.bits_per_sample}bit"
if ext == ".mp3":
from mutagen.mp3 import MP3, BitrateMode
audio = MP3(file_path)
bitrate_kbps = audio.info.bitrate // 1000
if audio.info.bitrate_mode == BitrateMode.VBR:
return "MP3-VBR"
return f"MP3-{bitrate_kbps}"
if ext in (".m4a", ".aac", ".mp4"):
from mutagen.mp4 import MP4
audio = MP4(file_path)
return f"M4A-{audio.info.bitrate // 1000}"
if ext == ".ogg":
from mutagen.oggvorbis import OggVorbis
audio = OggVorbis(file_path)
return f"OGG-{audio.info.bitrate // 1000}"
if ext == ".opus":
from mutagen.oggopus import OggOpus
audio = OggOpus(file_path)
return f"OPUS-{audio.info.bitrate // 1000}"
return ""
except Exception as e:
logger.debug(f"Could not determine audio quality for {file_path}: {e}")
return ""
def probe_audio_quality(file_path: str):
"""Read the actual file and return an AudioQuality with real measured values.
Uses mutagen to extract sample_rate, bit_depth, and bitrate from the
downloaded file — these are ground-truth values, not estimates.
Returns None when the file cannot be read.
"""
from core.quality.model import AudioQuality
try:
ext = os.path.splitext(file_path)[1].lower().lstrip('.')
if ext == 'flac':
from mutagen.flac import FLAC
audio = FLAC(file_path)
return AudioQuality(
format='flac',
bitrate=audio.info.bitrate // 1000 if audio.info.bitrate else None,
sample_rate=audio.info.sample_rate,
bit_depth=audio.info.bits_per_sample,
)
if ext == 'mp3':
from mutagen.mp3 import MP3
audio = MP3(file_path)
return AudioQuality(
format='mp3',
bitrate=audio.info.bitrate // 1000,
sample_rate=audio.info.sample_rate,
)
if ext in ('m4a', 'aac', 'mp4'):
from mutagen.mp4 import MP4
audio = MP4(file_path)
# .m4a can carry AAC (lossy) OR ALAC (lossless) — only the real
# codec tells them apart, which is why extension-based classification
# defaults to 'aac' and we correct it here from the probed file.
codec = (getattr(audio.info, 'codec', '') or '').lower()
if 'alac' in codec:
return AudioQuality(
format='alac',
bitrate=audio.info.bitrate // 1000 if audio.info.bitrate else None,
sample_rate=audio.info.sample_rate,
bit_depth=getattr(audio.info, 'bits_per_sample', None) or None,
)
return AudioQuality(
format='aac',
bitrate=audio.info.bitrate // 1000,
sample_rate=audio.info.sample_rate,
)
if ext == 'ogg':
from mutagen.oggvorbis import OggVorbis
audio = OggVorbis(file_path)
return AudioQuality(
format='ogg',
bitrate=audio.info.bitrate // 1000,
sample_rate=audio.info.sample_rate,
)
if ext == 'opus':
from mutagen.oggopus import OggOpus
audio = OggOpus(file_path)
return AudioQuality(
format='opus',
bitrate=audio.info.bitrate // 1000,
sample_rate=audio.info.sample_rate,
)
if ext in ('wav', 'aiff', 'aif'):
# AIFF must use mutagen.aiff.AIFF — WAVE() can't parse it and would
# raise, making the file fail open and silently bypass the quality
# filter. Both are uncompressed PCM, so they share the 'wav' tier.
if ext == 'wav':
from mutagen.wave import WAVE
audio = WAVE(file_path)
else:
from mutagen.aiff import AIFF
audio = AIFF(file_path)
return AudioQuality(
format='wav',
bitrate=audio.info.bitrate // 1000 if audio.info.bitrate else None,
sample_rate=audio.info.sample_rate,
bit_depth=getattr(audio.info, 'bits_per_sample', None),
)
if ext == 'wma':
from mutagen.asf import ASF
audio = ASF(file_path)
return AudioQuality(
format='wma',
bitrate=audio.info.bitrate // 1000 if audio.info.bitrate else None,
sample_rate=getattr(audio.info, 'sample_rate', None),
)
if ext in ('dsf', 'dff'):
# DSD (DSD Stream File / DSDIFF) — 1-bit hi-res lossless (#939). mutagen
# reads .dsf (rate/bitrate/bit_depth); .dff has no mutagen reader, so it
# still classifies as the lossless 'dsf' tier just without measured detail.
sr = bd = br = None
if ext == 'dsf':
try:
from mutagen.dsf import DSF
info = DSF(file_path).info
sr = getattr(info, 'sample_rate', None)
bd = getattr(info, 'bits_per_sample', None)
br = info.bitrate // 1000 if getattr(info, 'bitrate', None) else None
except Exception: # noqa: S110 — unreadable DSF still classifies lossless, just without measured detail
pass
return AudioQuality(format='dsf', bitrate=br, sample_rate=sr, bit_depth=bd)
return None
except Exception as e:
logger.debug("probe_audio_quality failed for %s: %s", file_path, e)
return None
def get_quality_tier_from_extension(file_path):
"""Classify a file extension into a quality tier."""
if not file_path:
return ("unknown", 999)
ext = os.path.splitext(file_path)[1].lower()
quality_tiers = {
"lossless": {
"extensions": [".flac", ".ape", ".wav", ".alac", ".dsf", ".dff", ".aiff", ".aif"],
"tier": 1,
},
"high_lossy": {
"extensions": [".opus", ".ogg"],
"tier": 2,
},
"standard_lossy": {
"extensions": [".m4a", ".aac"],
"tier": 3,
},
"low_lossy": {
"extensions": [".mp3", ".wma"],
"tier": 4,
},
}
for tier_name, tier_data in quality_tiers.items():
if ext in tier_data["extensions"]:
return (tier_name, tier_data["tier"])
return ("unknown", 999)
def downsample_hires_flac(final_path, context):
"""Downsample a hi-res FLAC to 16-bit/44.1kHz if enabled."""
from mutagen.flac import FLAC
if not config_manager.get("lossy_copy.downsample_hires", False):
return None
if os.path.splitext(final_path)[1].lower() != ".flac":
return None
try:
audio = FLAC(final_path)
original_bits = audio.info.bits_per_sample
original_rate = audio.info.sample_rate
except Exception as e:
logger.error(f"[Downsample] Could not read FLAC info: {e}")
return None
if original_bits <= 16 and original_rate <= 44100:
return None
logger.info(f"[Downsample] Converting {original_bits}-bit/{original_rate}Hz -> 16-bit/44100Hz: {os.path.basename(final_path)}")
ffmpeg_bin = shutil.which("ffmpeg")
if not ffmpeg_bin:
local = os.path.join(os.path.dirname(__file__), "tools", "ffmpeg")
if os.path.isfile(local):
ffmpeg_bin = local
else:
logger.warning("[Downsample] ffmpeg not found - skipping hi-res conversion")
return None
temp_path = final_path + ".tmp.flac"
try:
result = subprocess.run(
[
ffmpeg_bin, "-i", final_path,
"-sample_fmt", "s16",
"-ar", "44100",
"-map_metadata", "0",
"-compression_level", "8",
"-y", temp_path,
],
capture_output=True,
text=True,
timeout=300,
)
if result.returncode != 0:
logger.error(f"[Downsample] ffmpeg failed: {result.stderr[:200]}")
if os.path.exists(temp_path):
os.remove(temp_path)
return None
if not os.path.isfile(temp_path) or os.path.getsize(temp_path) == 0:
logger.warning("[Downsample] Output file missing or empty")
if os.path.exists(temp_path):
os.remove(temp_path)
return None
verify_audio = FLAC(temp_path)
if verify_audio.info.bits_per_sample != 16:
logger.info(f"[Downsample] Output not 16-bit ({verify_audio.info.bits_per_sample}-bit), aborting")
os.remove(temp_path)
return None
os.replace(temp_path, final_path)
logger.info(f"[Downsample] Converted to 16-bit/44.1kHz: {os.path.basename(final_path)}")
new_quality = "FLAC 16bit"
try:
updated_audio = FLAC(final_path)
updated_audio["QUALITY"] = new_quality
updated_audio.save()
except Exception as tag_err:
logger.error(f"[Downsample] Could not update QUALITY tag: {tag_err}")
old_quality = context.get("_audio_quality", "")
context["_audio_quality"] = new_quality
if old_quality and old_quality != new_quality and old_quality in os.path.basename(final_path):
new_basename = os.path.basename(final_path).replace(old_quality, new_quality)
new_path = os.path.join(os.path.dirname(final_path), new_basename)
try:
os.rename(final_path, new_path)
logger.info(f"[Downsample] Renamed: {os.path.basename(final_path)} -> {new_basename}")
for lyrics_ext in (".lrc", ".txt"):
old_lyrics = os.path.splitext(final_path)[0] + lyrics_ext
if os.path.isfile(old_lyrics):
new_lyrics = os.path.splitext(new_path)[0] + lyrics_ext
os.rename(old_lyrics, new_lyrics)
return new_path
except Exception as rename_err:
logger.error(f"[Downsample] Could not rename file: {rename_err}")
return final_path
except subprocess.TimeoutExpired:
logger.info(f"[Downsample] Conversion timed out for: {os.path.basename(final_path)}")
if os.path.exists(temp_path):
os.remove(temp_path)
except Exception as e:
logger.error(f"[Downsample] Conversion error: {e}")
if os.path.exists(temp_path):
try:
os.remove(temp_path)
except Exception as _e:
logger.debug("cleanup downsample temp: %s", _e)
return None
def m4a_codec(path):
"""Codec of an .m4a/.mp4 container ('alac', 'aac', …) or None — lets the
lossless check tell ALAC (lossless) from AAC (lossy), since both are .m4a."""
try:
from mutagen.mp4 import MP4
return (getattr(MP4(path).info, 'codec', '') or '').lower() or None
except Exception:
return None
def create_lossy_copy(final_path):
"""Convert a lossless file (FLAC / ALAC / WAV / AIFF / DSD) to a lossy copy
using the configured codec. Non-lossless inputs are skipped (#941)."""
from core.quality.lossless import (
is_lossless_audio_path,
lossy_output_would_overwrite_source,
)
if not config_manager.get("lossy_copy.enabled", False):
return None
# Was FLAC-only; now any lossless source. .m4a is probed (ALAC vs AAC).
if not is_lossless_audio_path(final_path, probe_codec=m4a_codec):
return None
codec = config_manager.get("lossy_copy.codec", "mp3").lower()
bitrate = config_manager.get("lossy_copy.bitrate", "320")
if codec == "opus" and int(bitrate) > 256:
bitrate = "256"
codec_map = {
"mp3": ("libmp3lame", ".mp3", f"MP3-{bitrate}", ["-vn", "-id3v2_version", "3"]),
"opus": ("libopus", ".opus", f"OPUS-{bitrate}", ["-vn", "-map", "0:a", "-vbr", "on"]),
"aac": ("aac", ".m4a", f"AAC-{bitrate}", ["-vn", "-movflags", "+faststart"]),
}
if codec not in codec_map:
logger.info(f"[Lossy Copy] Unknown codec '{codec}' - skipping conversion")
return None
ffmpeg_codec, out_ext, quality_label, extra_args = codec_map[codec]
out_path = os.path.splitext(final_path)[0] + out_ext
original_quality = get_audio_quality_string(final_path)
if original_quality:
out_basename = os.path.basename(out_path)
if original_quality in out_basename:
out_basename = out_basename.replace(original_quality, quality_label)
out_path = os.path.join(os.path.dirname(out_path), out_basename)
# Safety invariant: never write the lossy copy over its own source (an .m4a
# ALAC source + AAC target lands on the same .m4a path). ffmpeg runs with -y,
# so this guard MUST precede it — the later delete-original guard is too late.
if lossy_output_would_overwrite_source(final_path, out_path):
logger.info(
f"[Lossy Copy] Skipping — {codec.upper()} output would overwrite the "
f"source: {os.path.basename(final_path)}"
)
return None
ffmpeg_bin = shutil.which("ffmpeg")
if not ffmpeg_bin:
local = os.path.join(os.path.dirname(__file__), "tools", "ffmpeg")
if os.path.isfile(local):
ffmpeg_bin = local
else:
logger.warning(f"[Lossy Copy] ffmpeg not found - skipping {codec.upper()} conversion")
return None
try:
logger.info(f"[Lossy Copy] Converting to {quality_label}: {os.path.basename(final_path)}")
cmd = [
ffmpeg_bin, "-i", final_path,
"-codec:a", ffmpeg_codec,
"-b:a", f"{bitrate}k",
"-map_metadata", "0",
] + extra_args + ["-y", out_path]
result = subprocess.run(cmd, capture_output=True, text=True, timeout=120)
if result.returncode == 0:
logger.info(f"[Lossy Copy] Created {quality_label} copy: {os.path.basename(out_path)}")
try:
from mutagen import File as MutagenFile
audio = MutagenFile(out_path)
if audio is not None:
if codec == "mp3":
from mutagen.id3 import TXXX
audio.tags.add(TXXX(encoding=3, desc="QUALITY", text=[quality_label]))
elif codec == "opus":
audio["QUALITY"] = [quality_label]
elif codec == "aac":
from mutagen.mp4 import MP4FreeForm
audio["----:com.apple.iTunes:QUALITY"] = [MP4FreeForm(quality_label.encode("utf-8"))]
audio.save()
except Exception as tag_err:
logger.error(f"[Lossy Copy] Could not update QUALITY tag: {tag_err}")
# Honor the lossy_copy.delete_original setting — without this
# the original FLAC was always kept alongside the converted
# MP3/OPUS/AAC even when the user explicitly opted into a
# lossy-only library (Discord-reported by CAL).
if config_manager.get("lossy_copy.delete_original", False):
if os.path.normpath(out_path) != os.path.normpath(final_path):
try:
os.remove(final_path)
logger.info(
f"[Lossy Copy] Deleted original lossless source after conversion: "
f"{os.path.basename(final_path)}"
)
except FileNotFoundError:
# Already gone — concurrent cleanup or another worker
# handled it. Not an error.
pass
except Exception as del_err:
logger.error(
f"[Lossy Copy] Could not delete original after conversion "
f"({os.path.basename(final_path)}): {del_err}"
)
return out_path
logger.error(f"[Lossy Copy] ffmpeg failed: {result.stderr[:200]}")
if os.path.exists(out_path):
try:
os.remove(out_path)
except Exception as _e:
logger.debug("cleanup lossy copy artifact: %s", _e)
return None
except subprocess.TimeoutExpired:
logger.warning(f"[Lossy Copy] Conversion timed out for: {os.path.basename(final_path)}")
except Exception as e:
logger.error(f"[Lossy Copy] Conversion error: {e}")
return None