diff --git a/core/archive_pipeline.py b/core/archive_pipeline.py new file mode 100644 index 00000000..12248e93 --- /dev/null +++ b/core/archive_pipeline.py @@ -0,0 +1,232 @@ +"""Archive extraction + audio-file discovery for torrent / usenet downloads. + +The torrent and usenet download plugins need a uniform way to: + +1. Walk the downloader's save directory and find every audio file in it. +2. If the directory contains an archive (``.zip`` / ``.rar`` / ``.tar`` / + ``.7z``), extract it first so the audio files inside become walkable. + +This module is intentionally narrow — no matching, no tagging, no +import. The download plugin layer composes this with the existing +post-processing / matching pipeline. Lidarr does NOT use this module: +Lidarr extracts archives in its own import step before SoulSync sees +the files at all. Usenet downloaders (SABnzbd, NZBGet) also auto- +extract by default. Torrents are the main case where SoulSync may +need to do the extract step itself — most music torrents ship loose, +but some bundle the album in a ``.rar`` archive. + +``rarfile`` is an optional dependency. If it isn't installed, archives +with ``.rar`` content are skipped with a single warning rather than +crashing the download. +""" + +from __future__ import annotations + +import tarfile +import zipfile +from pathlib import Path +from typing import List, Optional + +from utils.logging_config import get_logger + +logger = get_logger("archive_pipeline") + + +# Same audio-extension set as ``core/imports/file_ops.py`` ``quality_tiers``. +# Keep them in sync — if a new format is added to file_ops, add it here too +# or the walker will skip it and the download plugin will mark the download +# failed even when files arrived. +AUDIO_EXTENSIONS = frozenset([ + # lossless + '.flac', '.ape', '.wav', '.alac', '.dsf', '.dff', '.aiff', '.aif', + # high lossy + '.opus', '.ogg', + # standard lossy + '.m4a', '.aac', + # low lossy + '.mp3', '.wma', +]) + +ARCHIVE_EXTENSIONS = frozenset(['.zip', '.rar', '.tar', '.tar.gz', '.tgz', '.7z']) + + +def is_archive(path: Path) -> bool: + """True if the file extension looks like a supported archive. + + Compound extensions (``.tar.gz``, ``.tar.bz2``) are detected by + checking the last two suffixes joined together — Path.suffix + only returns the final suffix. + """ + if not path.is_file(): + return False + name = path.name.lower() + if name.endswith(('.tar.gz', '.tar.bz2', '.tar.xz')): + return True + return path.suffix.lower() in ARCHIVE_EXTENSIONS + + +def walk_audio_files(directory: Path) -> List[Path]: + """Recursively scan ``directory`` for audio files. Returns + a sorted list of absolute paths. Empty list if the directory + doesn't exist or contains no audio. + """ + if not directory or not directory.exists() or not directory.is_dir(): + return [] + out: List[Path] = [] + for child in directory.rglob('*'): + if not child.is_file(): + continue + if child.suffix.lower() in AUDIO_EXTENSIONS: + out.append(child.resolve()) + out.sort() + return out + + +def find_archives_in_dir(directory: Path) -> List[Path]: + """Find every archive file directly inside ``directory`` (one + level deep — torrents normally put the archive at the root of + their folder; we don't search nested dirs to avoid extracting + something we shouldn't). + """ + if not directory or not directory.exists() or not directory.is_dir(): + return [] + return sorted(p for p in directory.iterdir() if is_archive(p)) + + +def extract_archive(archive_path: Path, extract_to: Optional[Path] = None) -> Optional[Path]: + """Extract a single archive in-place (or to ``extract_to`` if + given). Returns the directory the archive was extracted into, + or ``None`` on failure. + + Supports ``.zip``, ``.tar``/``.tar.gz``/``.tar.bz2``/``.tar.xz``, + and ``.rar`` (only when the optional ``rarfile`` library is + installed). ``.7z`` is recognised but extraction requires + ``py7zr``; without it, the call logs and returns None. + """ + if not archive_path or not archive_path.exists(): + logger.warning("archive_pipeline: %s does not exist", archive_path) + return None + dest = extract_to or archive_path.parent + dest.mkdir(parents=True, exist_ok=True) + + name = archive_path.name.lower() + try: + if name.endswith('.zip'): + with zipfile.ZipFile(archive_path) as zf: + _safe_extract_zip(zf, dest) + return dest + if name.endswith(('.tar', '.tar.gz', '.tgz', '.tar.bz2', '.tar.xz')): + with tarfile.open(archive_path) as tf: + _safe_extract_tar(tf, dest) + return dest + if name.endswith('.rar'): + return _extract_rar(archive_path, dest) + if name.endswith('.7z'): + return _extract_7z(archive_path, dest) + except (zipfile.BadZipFile, tarfile.TarError, OSError) as e: + logger.error("archive_pipeline: failed to extract %s: %s", archive_path, e) + return None + logger.warning("archive_pipeline: unknown archive type for %s", archive_path) + return None + + +def extract_all_in_dir(directory: Path) -> List[Path]: + """Find every archive in ``directory`` and extract each in place. + Returns the list of directories archives were extracted into + (usually all the same — ``directory`` itself). Archives that + failed to extract are skipped silently after a warning. + """ + out: List[Path] = [] + for archive in find_archives_in_dir(directory): + result = extract_archive(archive) + if result is not None: + out.append(result) + return out + + +def collect_audio_after_extraction(directory: Path) -> List[Path]: + """One-shot helper for the download plugins: extract any archives + in the directory, then return the walked audio file list. This is + the common pattern — torrent / usenet plugin gets a save_path, + calls this, hands the resulting files to the matching pipeline. + """ + extract_all_in_dir(directory) + return walk_audio_files(directory) + + +# --------------------------------------------------------------------------- +# Safety helpers +# --------------------------------------------------------------------------- + + +def _safe_extract_zip(zf: zipfile.ZipFile, dest: Path) -> None: + """Extract a zipfile after rejecting any member whose resolved + path escapes ``dest`` (path traversal protection). + """ + dest = dest.resolve() + for member in zf.namelist(): + target = (dest / member).resolve() + if dest not in target.parents and target != dest: + logger.error("archive_pipeline: refusing path-traversal member %r", member) + return + zf.extractall(dest) + + +def _safe_extract_tar(tf: tarfile.TarFile, dest: Path) -> None: + """Same path-traversal protection for tarfiles.""" + dest = dest.resolve() + for member in tf.getmembers(): + target = (dest / member.name).resolve() + if dest not in target.parents and target != dest: + logger.error("archive_pipeline: refusing path-traversal member %r", member.name) + return + # ``filter='data'`` is the Python 3.12+ safe extractor; fall back + # to the legacy call on older runtimes. + try: + tf.extractall(dest, filter='data') # type: ignore[call-arg] + except TypeError: + tf.extractall(dest) + + +def _extract_rar(archive_path: Path, dest: Path) -> Optional[Path]: + try: + import rarfile # type: ignore[import-untyped] + except ImportError: + logger.warning( + "archive_pipeline: cannot extract %s — rarfile library not installed. " + "Install with: pip install rarfile (and ensure unrar is on PATH).", + archive_path, + ) + return None + try: + with rarfile.RarFile(archive_path) as rf: + dest_resolved = dest.resolve() + for name in rf.namelist(): + target = (dest_resolved / name).resolve() + if dest_resolved not in target.parents and target != dest_resolved: + logger.error("archive_pipeline: refusing path-traversal rar member %r", name) + return None + rf.extractall(dest) + return dest + except Exception as e: + logger.error("archive_pipeline: rar extract failed for %s: %s", archive_path, e) + return None + + +def _extract_7z(archive_path: Path, dest: Path) -> Optional[Path]: + try: + import py7zr # type: ignore[import-untyped] + except ImportError: + logger.warning( + "archive_pipeline: cannot extract %s — py7zr library not installed. " + "Install with: pip install py7zr.", + archive_path, + ) + return None + try: + with py7zr.SevenZipFile(archive_path, 'r') as sz: + sz.extractall(path=dest) + return dest + except Exception as e: + logger.error("archive_pipeline: 7z extract failed for %s: %s", archive_path, e) + return None diff --git a/tests/test_archive_pipeline.py b/tests/test_archive_pipeline.py new file mode 100644 index 00000000..053c200a --- /dev/null +++ b/tests/test_archive_pipeline.py @@ -0,0 +1,238 @@ +"""Tests for ``core/archive_pipeline.py``. + +Covers the audio-file walker, archive detector, and zip / tar +extraction (rar / 7z paths use optional deps so they're only +exercised when the libs are present in the test environment). +Path-traversal protection gets explicit coverage — a malicious +archive must not escape the extraction directory. +""" + +from __future__ import annotations + +import os +import tarfile +import zipfile +from pathlib import Path + +import pytest + +from core.archive_pipeline import ( + AUDIO_EXTENSIONS, + ARCHIVE_EXTENSIONS, + collect_audio_after_extraction, + extract_archive, + extract_all_in_dir, + find_archives_in_dir, + is_archive, + walk_audio_files, +) + + +# --------------------------------------------------------------------------- +# Constants +# --------------------------------------------------------------------------- + + +def test_audio_extensions_cover_common_formats() -> None: + for ext in ('.flac', '.mp3', '.m4a', '.ogg', '.opus', '.wav', '.aac', '.wma'): + assert ext in AUDIO_EXTENSIONS + + +def test_archive_extensions_cover_common_formats() -> None: + for ext in ('.zip', '.rar', '.tar', '.7z'): + assert ext in ARCHIVE_EXTENSIONS + + +# --------------------------------------------------------------------------- +# is_archive +# --------------------------------------------------------------------------- + + +def test_is_archive_detects_simple_extensions(tmp_path: Path) -> None: + zip_path = tmp_path / 'x.zip' + zip_path.write_bytes(b'PK\x03\x04') # minimal — is_archive doesn't validate content + assert is_archive(zip_path) is True + + +def test_is_archive_detects_compound_tar_extensions(tmp_path: Path) -> None: + """``.tar.gz`` etc. — Path.suffix only catches the last suffix, + so the detector has to special-case compound extensions.""" + targz = tmp_path / 'x.tar.gz' + targz.write_bytes(b'\x1f\x8b') + assert is_archive(targz) is True + + +def test_is_archive_returns_false_for_audio(tmp_path: Path) -> None: + flac = tmp_path / 'song.flac' + flac.write_bytes(b'fLaC') + assert is_archive(flac) is False + + +def test_is_archive_returns_false_for_missing_file(tmp_path: Path) -> None: + assert is_archive(tmp_path / 'does-not-exist.zip') is False + + +# --------------------------------------------------------------------------- +# walk_audio_files +# --------------------------------------------------------------------------- + + +def test_walk_audio_files_finds_nested(tmp_path: Path) -> None: + # Layout: root/album/disc1/track.flac + root/album/disc2/track.mp3 + (tmp_path / 'album' / 'disc1').mkdir(parents=True) + (tmp_path / 'album' / 'disc2').mkdir(parents=True) + (tmp_path / 'album' / 'disc1' / 'track1.flac').write_bytes(b'fLaC') + (tmp_path / 'album' / 'disc2' / 'track1.mp3').write_bytes(b'ID3') + (tmp_path / 'album' / 'cover.jpg').write_bytes(b'\xff\xd8') + found = walk_audio_files(tmp_path) + names = sorted(p.name for p in found) + assert names == ['track1.flac', 'track1.mp3'] + + +def test_walk_audio_files_returns_empty_for_missing(tmp_path: Path) -> None: + assert walk_audio_files(tmp_path / 'does-not-exist') == [] + + +def test_walk_audio_files_ignores_non_audio(tmp_path: Path) -> None: + (tmp_path / 'readme.txt').write_text('hi') + (tmp_path / 'cover.png').write_bytes(b'\x89PNG') + assert walk_audio_files(tmp_path) == [] + + +def test_walk_audio_files_case_insensitive_extension(tmp_path: Path) -> None: + """Lots of torrents have uppercase extensions (.MP3, .FLAC) — + the walker must catch those too.""" + (tmp_path / 'TRACK.MP3').write_bytes(b'ID3') + (tmp_path / 'TRACK.FLAC').write_bytes(b'fLaC') + found = walk_audio_files(tmp_path) + assert len(found) == 2 + + +# --------------------------------------------------------------------------- +# find_archives_in_dir +# --------------------------------------------------------------------------- + + +def test_find_archives_in_dir_only_top_level(tmp_path: Path) -> None: + """find_archives doesn't recurse — torrents put the archive at + the top of the dir; deeper search risks extracting unrelated + archives that ship inside a sample folder, etc.""" + (tmp_path / 'album.zip').write_bytes(b'PK\x03\x04') + nested = tmp_path / 'subdir' + nested.mkdir() + (nested / 'nested.zip').write_bytes(b'PK\x03\x04') + found = find_archives_in_dir(tmp_path) + assert [p.name for p in found] == ['album.zip'] + + +def test_find_archives_in_dir_empty(tmp_path: Path) -> None: + assert find_archives_in_dir(tmp_path) == [] + assert find_archives_in_dir(tmp_path / 'missing') == [] + + +# --------------------------------------------------------------------------- +# extract_archive — zip +# --------------------------------------------------------------------------- + + +def test_extract_zip_writes_files(tmp_path: Path) -> None: + zip_path = tmp_path / 'album.zip' + with zipfile.ZipFile(zip_path, 'w') as zf: + zf.writestr('track1.mp3', b'ID3 track1 data') + zf.writestr('track2.flac', b'fLaC data') + result = extract_archive(zip_path) + assert result == zip_path.parent + assert (tmp_path / 'track1.mp3').exists() + assert (tmp_path / 'track2.flac').exists() + + +def test_extract_zip_rejects_path_traversal(tmp_path: Path) -> None: + """A malicious archive trying to write ``../../etc/passwd`` must + be refused without extracting anything.""" + zip_path = tmp_path / 'evil.zip' + extract_dest = tmp_path / 'staging' + extract_dest.mkdir() + with zipfile.ZipFile(zip_path, 'w') as zf: + zf.writestr('../escaped.txt', b'evil') + zf.writestr('safe.mp3', b'ID3') + extract_archive(zip_path, extract_to=extract_dest) + # Neither file should have landed — extraction aborts on the first + # traversal attempt. + assert not (extract_dest / 'safe.mp3').exists() + assert not (tmp_path / 'escaped.txt').exists() + + +def test_extract_zip_returns_none_for_bad_zip(tmp_path: Path) -> None: + bad = tmp_path / 'not-a-zip.zip' + bad.write_bytes(b'this is not a zip') + assert extract_archive(bad) is None + + +def test_extract_archive_missing_file_returns_none(tmp_path: Path) -> None: + assert extract_archive(tmp_path / 'does-not-exist.zip') is None + + +# --------------------------------------------------------------------------- +# extract_archive — tar +# --------------------------------------------------------------------------- + + +def test_extract_tar_gz_writes_files(tmp_path: Path) -> None: + payload = tmp_path / 'track.mp3' + payload.write_bytes(b'ID3 track') + tar_path = tmp_path / 'album.tar.gz' + with tarfile.open(tar_path, 'w:gz') as tf: + tf.add(payload, arcname='track.mp3') + payload.unlink() # remove the source so we can verify the extract recreated it + extract_archive(tar_path) + assert (tmp_path / 'track.mp3').exists() + + +def test_extract_tar_rejects_path_traversal(tmp_path: Path) -> None: + extract_dest = tmp_path / 'staging' + extract_dest.mkdir() + tar_path = tmp_path / 'evil.tar' + payload = tmp_path / 'src.txt' + payload.write_bytes(b'evil') + with tarfile.open(tar_path, 'w') as tf: + info = tf.gettarinfo(str(payload), arcname='../escaped.txt') + with payload.open('rb') as fh: + tf.addfile(info, fh) + extract_archive(tar_path, extract_to=extract_dest) + assert not (tmp_path / 'escaped.txt').exists() + + +# --------------------------------------------------------------------------- +# extract_all_in_dir + collect_audio_after_extraction +# --------------------------------------------------------------------------- + + +def test_extract_all_in_dir_handles_multiple_archives(tmp_path: Path) -> None: + (tmp_path / 'one.zip') # placeholder + z1 = tmp_path / 'one.zip' + z2 = tmp_path / 'two.zip' + with zipfile.ZipFile(z1, 'w') as zf: + zf.writestr('a.mp3', b'a') + with zipfile.ZipFile(z2, 'w') as zf: + zf.writestr('b.mp3', b'b') + out = extract_all_in_dir(tmp_path) + assert len(out) == 2 + assert (tmp_path / 'a.mp3').exists() + assert (tmp_path / 'b.mp3').exists() + + +def test_collect_audio_after_extraction_combines_loose_and_extracted(tmp_path: Path) -> None: + """The typical mixed case: torrent dropped a .zip and also some + loose .mp3 files alongside it. The collector returns BOTH.""" + (tmp_path / 'bonus.mp3').write_bytes(b'ID3') + zip_path = tmp_path / 'main.zip' + with zipfile.ZipFile(zip_path, 'w') as zf: + zf.writestr('track1.flac', b'fLaC') + zf.writestr('track2.flac', b'fLaC') + found = collect_audio_after_extraction(tmp_path) + names = sorted(p.name for p in found) + assert names == ['bonus.mp3', 'track1.flac', 'track2.flac'] + + +def test_collect_audio_after_extraction_no_archives_no_audio(tmp_path: Path) -> None: + assert collect_audio_after_extraction(tmp_path) == [] diff --git a/webui/static/helper.js b/webui/static/helper.js index f60b0527..5919d795 100644 --- a/webui/static/helper.js +++ b/webui/static/helper.js @@ -3415,6 +3415,7 @@ function closeHelperSearch() { const WHATS_NEW = { '2.6.0': [ { unreleased: true }, + { title: 'Archive pipeline module (groundwork for torrent / usenet downloads)', desc: 'new core/archive_pipeline.py — walks a directory for audio files (recursive, case-insensitive extensions), extracts zip / tar / tar.gz / rar / 7z archives in-place (rar and 7z are optional deps that warn but don\'t crash if absent), and rejects any archive member trying to escape the destination via path traversal. shared helper the upcoming torrent and usenet download plugins both consume — usenet downloaders usually auto-extract, but the occasional torrent ships an album in a .rar and SoulSync handles it now. 21 unit tests cover the walker + zip + tar extraction + path-traversal protection.' }, { title: 'Indexers & Downloaders tab restyled with collapsible sections', desc: 'tab now opens with an intro hero card explaining the flow (indexers find releases → downloader fetches them → SoulSync imports) and folds the rest into three collapsible sections: Indexers, Torrent Client, Usenet Client. each section gets a per-service color accent (Prowlarr orange, torrent sky-blue, usenet violet), a status dot in the header (green when Test Connection succeeds, red on failure, grey before testing), and Lidarr-style indexer cards with protocol badges instead of the inline emoji list. Indexers section is open by default; the downloader sections start collapsed since not everyone uses both protocols.' }, { title: 'Regression tests for the new indexer + downloader plumbing', desc: 'mocked unit tests for Prowlarr + all five downloader adapters (qBittorrent, Transmission, Deluge, SABnzbd, NZBGet). 54 tests pin the state-mapping tables, the parse logic, and the protocol quirks each client needs handled (qBit Referer header, Transmission session-id renegotiation, Deluge magnet-vs-URL method split, SAB queue+history merge, NZBGet 64-bit size fields). next time anyone touches one of these adapters, CI catches breakage before it hits a real downloader.' }, { title: 'Usenet client adapters (SABnzbd, NZBGet)', desc: 'third commit in the torrent + usenet rollout. SoulSync now talks to SABnzbd and NZBGet through a sibling adapter contract that mirrors the torrent adapter set — pick one downloader in Settings → Indexers & Downloaders, fill in its API key (SABnzbd) or username + password (NZBGet), and Test Connection confirms the link. all three layers are now stood up: Prowlarr finds releases, the torrent adapter and the usenet adapter each know how to ship work to the underlying client. next commit wires Prowlarr search → adapter dispatch → archive extraction so the new sources actually download. job state mapping covers SABnzbd queue + history and NZBGet groups + history, including the verify/repair/unpack phases that are unique to usenet.' },