From 3ea5c2788a03309f65399c57b719bd375f99d3ab Mon Sep 17 00:00:00 2001 From: Broque Thomas <26755000+Nezreka@users.noreply.github.com> Date: Wed, 20 May 2026 09:22:08 -0700 Subject: [PATCH 01/13] Update README.md --- README.md | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/README.md b/README.md index 579dc488..eae068d0 100644 --- a/README.md +++ b/README.md @@ -279,10 +279,29 @@ The template points at `boulderbadgedad/soulsync:latest` (stable) by default. To git clone https://github.com/Nezreka/SoulSync cd SoulSync python -m pip install -r requirements.txt + +# Build the React WebUI bundle used by the Python server. +# Docker does this automatically; Python installs must do it manually. +cd webui +npm ci +npm run build +cd .. + gunicorn -c gunicorn.conf.py wsgi:application # Open http://localhost:8008 ``` +When updating a Python/no-Docker install with `git pull`, rebuild the WebUI before restarting SoulSync: + +```bash +cd webui +npm ci +npm run build +cd .. +``` + +If `webui/static/dist/.vite/manifest.json` is missing or stale, React-owned routes and route handoffs may not load correctly. + ### Local Development Use two terminals so the backend and Vite stay independent: From f02ef4dd6acbccde0234ae345941f46dd08b7f88 Mon Sep 17 00:00:00 2001 From: Broque Thomas <26755000+Nezreka@users.noreply.github.com> Date: Wed, 20 May 2026 09:23:48 -0700 Subject: [PATCH 02/13] Update README.md --- README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index eae068d0..a79eeb7e 100644 --- a/README.md +++ b/README.md @@ -304,7 +304,9 @@ If `webui/static/dist/.vite/manifest.json` is missing or stale, React-owned rout ### Local Development -Use two terminals so the backend and Vite stay independent: +This is only for contributors working on the WebUI with hot reload. Normal Python/no-Docker installs should build once with `npm run build` as shown above, then run only Gunicorn. + +For active frontend development, use two terminals so the backend and Vite stay independent: 1. Backend ```bash From 6e5ea1d490b3d566eafaae70f55b4defcb217d8a Mon Sep 17 00:00:00 2001 From: Broque Thomas <26755000+Nezreka@users.noreply.github.com> Date: Wed, 20 May 2026 09:39:56 -0700 Subject: [PATCH 03/13] fix(downloads): wait for post-processing result Do not mark a monitored transfer as successful as soon as slskd reports completion. The monitor now only submits the post-processing worker; that worker reports the real success or failure after finding, verifying, and importing the file. If post-processing cannot be scheduled, mark the task failed and release the batch slot. Add a regression test for the premature success path. --- core/downloads/lifecycle.py | 2 +- core/downloads/monitor.py | 18 +++++-- tests/test_manual_pick_no_auto_retry.py | 72 +++++++++++++++++++++++++ 3 files changed, 87 insertions(+), 5 deletions(-) diff --git a/core/downloads/lifecycle.py b/core/downloads/lifecycle.py index 43e1e970..2a29feec 100644 --- a/core/downloads/lifecycle.py +++ b/core/downloads/lifecycle.py @@ -173,7 +173,7 @@ def on_download_completed(batch_id: str, task_id: str, success: bool, deps: Life # Guard against double-calling: track which tasks have already been completed # This prevents active_count from being decremented multiple times for the same task - # (e.g. monitor detects completion AND post-processing calls this again) + # (e.g. status polling and post-processing both observe the same terminal task) # NOTE: On duplicate calls, we skip decrement/tracking but STILL check batch completion. # This is critical because the first call may see the task in 'post_processing' (not finished), # and the second call (from post-processing worker) arrives after the task is truly 'completed'. diff --git a/core/downloads/monitor.py b/core/downloads/monitor.py index 79a90d8d..62d515e5 100644 --- a/core/downloads/monitor.py +++ b/core/downloads/monitor.py @@ -227,18 +227,28 @@ class WebUIDownloadMonitor: except Exception as e: logger.error(f"[Deferred] Error executing deferred operation {op[0]}: {e}") - # Handle completed downloads outside the lock to prevent deadlock - # (_on_download_completed acquires tasks_lock internally) + # Handle completed transfers outside the lock. The transfer engine's + # "complete" state only means the remote download finished; the + # post-processing worker still has to find, verify, tag, and move the + # file before it can report real batch success or failure. for batch_id, task_id in completed_tasks: try: # Submit post-processing worker (file move, tagging, AcoustID verification) # This makes batch downloads fully independent of browser polling. logger.info(f"[Monitor] Submitting post-processing worker for task {task_id}") missing_download_executor.submit(_run_post_processing_worker, task_id, batch_id) - # Chain to next download in the batch queue - _on_download_completed(batch_id, task_id, success=True) except Exception as e: logger.error(f"[Monitor] Error handling completed task {task_id}: {e}") + with tasks_lock: + if task_id in download_tasks: + download_tasks[task_id]['status'] = 'failed' + download_tasks[task_id]['error_message'] = f'Post-processing could not be scheduled: {e}' + try: + _on_download_completed(batch_id, task_id, success=False) + except Exception as completion_error: + logger.error( + f"[Monitor] Error marking failed post-processing submit for task {task_id}: {completion_error}" + ) # Handle exhausted retry tasks outside the lock to prevent deadlock for batch_id, task_id in exhausted_tasks: try: diff --git a/tests/test_manual_pick_no_auto_retry.py b/tests/test_manual_pick_no_auto_retry.py index c5b23cfe..efa88359 100644 --- a/tests/test_manual_pick_no_auto_retry.py +++ b/tests/test_manual_pick_no_auto_retry.py @@ -93,3 +93,75 @@ def test_manual_pick_skips_retry_on_errored_state(fake_monitor): assert task['status'] == 'downloading' +def test_monitor_waits_for_post_processing_before_batch_success(monkeypatch): + """Engine completion is not the same as a successful import. + + The monitor should start post-processing when slskd reports a completed + transfer, but the post-processing worker must be the only code path that + reports final success/failure to the batch lifecycle. + """ + monkeypatch.setattr(dm, '_make_context_key', lambda u, f: f"{u}::{f}") + monkeypatch.setattr(dm.WebUIDownloadMonitor, '_validate_worker_counts', lambda self: None) + + submitted = [] + completions = [] + + class FakeExecutor: + def submit(self, func, task_id, batch_id): + submitted.append((func, task_id, batch_id)) + + def fake_post_processing_worker(task_id, batch_id): + return None + + monkeypatch.setattr(dm, 'missing_download_executor', FakeExecutor()) + monkeypatch.setattr(dm, '_run_post_processing_worker', fake_post_processing_worker) + monkeypatch.setattr( + dm, + '_on_download_completed', + lambda batch_id, task_id, success: completions.append((batch_id, task_id, success)), + ) + + with dm.tasks_lock: + previous_tasks = dict(dm.download_tasks) + previous_batches = dict(dm.download_batches) + dm.download_tasks.clear() + dm.download_batches.clear() + dm.download_tasks['task-1'] = { + 'track_info': {'name': 'Test Track'}, + 'username': 'Pinasound', + 'filename': r'@@tmllb\Music\Album\01. Track.flac', + 'status': 'downloading', + 'download_id': 'download-1', + 'status_change_time': time.time(), + } + dm.download_batches['batch-1'] = {'queue': ['task-1']} + + try: + monitor = dm.WebUIDownloadMonitor() + monitor.monitoring = True + monitor.monitored_batches.add('batch-1') + monkeypatch.setattr( + monitor, + '_get_live_transfers', + lambda: { + r'Pinasound::@@tmllb\Music\Album\01. Track.flac': { + 'state': 'Completed, Succeeded', + 'size': 100, + 'bytesTransferred': 100, + } + }, + ) + + monitor._check_all_downloads() + + assert submitted == [(fake_post_processing_worker, 'task-1', 'batch-1')] + assert completions == [] + assert dm.download_tasks['task-1']['status'] == 'post_processing' + finally: + with dm.tasks_lock: + dm.download_tasks.clear() + dm.download_tasks.update(previous_tasks) + dm.download_batches.clear() + dm.download_batches.update(previous_batches) + + From 136d665c8a26b1c20cfb9dbf45e77aaf7b8428ff Mon Sep 17 00:00:00 2001 From: Broque Thomas <26755000+Nezreka@users.noreply.github.com> Date: Wed, 20 May 2026 10:43:47 -0700 Subject: [PATCH 04/13] feat(webui): cache artwork images on disk Add a disk-backed image cache with hashed browser URLs, SQLite metadata, size/type validation, stale fallback, and per-image fetch locking. Route normalized artwork through /api/image-cache while keeping /api/image-proxy as a compatibility shim, and align browser max-age with the image cache TTL. Add focused tests for cache behavior and image URL normalization. --- .gitignore | 1 + config/settings.py | 7 + core/image_cache.py | 342 ++++++++++++++++++ core/metadata/artwork.py | 15 +- .../metadata/test_image_url_normalization.py | 33 +- tests/test_image_cache.py | 66 ++++ web_server.py | 69 ++-- 7 files changed, 497 insertions(+), 36 deletions(-) create mode 100644 core/image_cache.py create mode 100644 tests/test_image_cache.py diff --git a/.gitignore b/.gitignore index 8266b83b..8857f8d5 100644 --- a/.gitignore +++ b/.gitignore @@ -17,6 +17,7 @@ database/music_library.db-shm database/music_library.db-wal database/music_library.db.backup_* database/api_call_history.json +storage/image_cache/ logs/*.log logs/*.log.* diff --git a/config/settings.py b/config/settings.py index 2cfc3eeb..b2c6b8c4 100644 --- a/config/settings.py +++ b/config/settings.py @@ -552,6 +552,13 @@ class ConfigManager: "path": os.environ.get('DATABASE_PATH', 'database/music_library.db'), "max_workers": 5 }, + "image_cache": { + "enabled": True, + "path": "storage/image_cache", + "ttl_seconds": 2592000, + "failed_ttl_seconds": 21600, + "max_download_mb": 15 + }, "metadata_enhancement": { "enabled": True, "embed_album_art": True, diff --git a/core/image_cache.py b/core/image_cache.py new file mode 100644 index 00000000..9ee0be47 --- /dev/null +++ b/core/image_cache.py @@ -0,0 +1,342 @@ +"""Disk-backed image cache for browser-facing artwork URLs.""" + +from __future__ import annotations + +import hashlib +import mimetypes +import os +import sqlite3 +import threading +import time +from dataclasses import dataclass +from pathlib import Path +from typing import Callable, Optional +from urllib.parse import urlparse + +import requests + +from config.settings import config_manager +from core.metadata.artwork import is_internal_image_host +from utils.logging_config import get_logger + +logger = get_logger("image_cache") + +DEFAULT_TTL_SECONDS = 30 * 24 * 60 * 60 +DEFAULT_FAILED_TTL_SECONDS = 6 * 60 * 60 +DEFAULT_MAX_DOWNLOAD_BYTES = 15 * 1024 * 1024 + + +class ImageCacheError(Exception): + """Raised when an image cannot be served from the cache.""" + + +@dataclass +class CachedImage: + key: str + path: Path + mime_type: str + size: int + status: str + + +class ImageCache: + def __init__( + self, + cache_dir: str | os.PathLike[str], + *, + ttl_seconds: int = DEFAULT_TTL_SECONDS, + failed_ttl_seconds: int = DEFAULT_FAILED_TTL_SECONDS, + max_download_bytes: int = DEFAULT_MAX_DOWNLOAD_BYTES, + fetcher: Optional[Callable[..., requests.Response]] = None, + ): + self.cache_dir = Path(cache_dir) + self.ttl_seconds = int(ttl_seconds) + self.failed_ttl_seconds = int(failed_ttl_seconds) + self.max_download_bytes = int(max_download_bytes) + self.fetcher = fetcher or requests.get + self.db_path = self.cache_dir / "image_cache.sqlite3" + self._db_lock = threading.RLock() + self._key_locks: dict[str, threading.Lock] = {} + self._key_locks_lock = threading.Lock() + self.cache_dir.mkdir(parents=True, exist_ok=True) + self._init_db() + + def cache_url_for(self, url: str | None) -> str | None: + """Register a URL and return its browser-facing cached path.""" + if not url: + return None + if str(url).startswith("/api/image-cache/"): + return str(url) + if not self.is_cacheable_url(str(url)): + return str(url) + + key = self.key_for_url(str(url)) + now = time.time() + with self._db_lock: + with self._connect() as conn: + conn.execute( + """ + INSERT INTO image_cache + (key, original_url, status, created_at, updated_at, last_accessed, + expires_at, size, mime_type, file_path, last_error) + VALUES (?, ?, 'pending', ?, ?, ?, 0, 0, '', '', '') + ON CONFLICT(key) DO UPDATE SET + original_url=excluded.original_url, + last_accessed=excluded.last_accessed + """, + (key, str(url), now, now, now), + ) + return f"/api/image-cache/{key}" + + def get(self, key: str) -> CachedImage: + row = self._get_row(key) + if not row: + raise ImageCacheError("Image cache key not found") + return self.get_url(row["original_url"]) + + def get_url(self, url: str) -> CachedImage: + if not self.is_cacheable_url(url): + raise ImageCacheError("URL is not cacheable") + + key = self.key_for_url(url) + lock = self._lock_for_key(key) + with lock: + row = self._get_row(key) + now = time.time() + if row and row["status"] == "ok" and row["file_path"]: + path = Path(row["file_path"]) + if path.exists(): + self._touch(key, now) + if float(row["expires_at"] or 0) > now: + return CachedImage(key, path, row["mime_type"] or "image/jpeg", int(row["size"] or 0), "hit") + + try: + return self._fetch_and_store(url, key, now) + except Exception as exc: + if row and row["status"] == "ok" and row["file_path"]: + stale_path = Path(row["file_path"]) + if stale_path.exists(): + logger.warning("Serving stale cached image for %s after refresh failed: %s", key, exc) + self._record_error(key, str(exc), now, keep_status=True) + return CachedImage( + key, + stale_path, + row["mime_type"] or "image/jpeg", + int(row["size"] or 0), + "stale", + ) + self._record_error(key, str(exc), now) + raise ImageCacheError(str(exc)) from exc + + @staticmethod + def key_for_url(url: str) -> str: + return hashlib.sha256(url.encode("utf-8")).hexdigest() + + @staticmethod + def is_cacheable_url(url: str) -> bool: + try: + parsed = urlparse(url) + if parsed.scheme not in {"http", "https"}: + return False + if parsed.username or parsed.password: + return False + if not parsed.hostname: + return False + return True + except Exception: + return False + + def _fetch_and_store(self, url: str, key: str, now: float) -> CachedImage: + if not self._is_fetch_allowed(url): + raise ImageCacheError("Image host is not allowed") + + response = self.fetcher( + url, + timeout=10, + stream=True, + headers={ + "User-Agent": ( + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) " + "AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36" + ), + "Accept": "image/avif,image/webp,image/apng,image/svg+xml,image/*,*/*;q=0.8", + "Referer": "https://www.deezer.com/", + }, + ) + try: + if response.status_code != 200: + raise ImageCacheError(f"Upstream image returned HTTP {response.status_code}") + + mime_type = (response.headers.get("Content-Type") or "image/jpeg").split(";", 1)[0].strip() + if not mime_type.startswith("image/"): + raise ImageCacheError(f"Upstream response is not an image: {mime_type}") + + declared_size = response.headers.get("Content-Length") + try: + if declared_size and int(declared_size) > self.max_download_bytes: + raise ImageCacheError("Image exceeds configured size limit") + except ValueError: + pass + + ext = mimetypes.guess_extension(mime_type) or ".img" + if ext == ".jpe": + ext = ".jpg" + path = self._path_for_key(key, ext) + path.parent.mkdir(parents=True, exist_ok=True) + tmp_path = path.with_suffix(path.suffix + ".tmp") + + total = 0 + try: + with open(tmp_path, "wb") as handle: + for chunk in response.iter_content(chunk_size=64 * 1024): + if not chunk: + continue + total += len(chunk) + if total > self.max_download_bytes: + raise ImageCacheError("Image exceeds configured size limit") + handle.write(chunk) + except Exception: + try: + tmp_path.unlink(missing_ok=True) + except Exception: + pass + raise + + if total <= 0: + raise ImageCacheError("Image response was empty") + + os.replace(tmp_path, path) + expires_at = now + self.ttl_seconds + with self._db_lock: + with self._connect() as conn: + conn.execute( + """ + INSERT INTO image_cache + (key, original_url, status, created_at, updated_at, last_accessed, + expires_at, size, mime_type, file_path, last_error) + VALUES (?, ?, 'ok', ?, ?, ?, ?, ?, ?, ?, '') + ON CONFLICT(key) DO UPDATE SET + original_url=excluded.original_url, + status='ok', + updated_at=excluded.updated_at, + last_accessed=excluded.last_accessed, + expires_at=excluded.expires_at, + size=excluded.size, + mime_type=excluded.mime_type, + file_path=excluded.file_path, + last_error='' + """, + (key, url, now, now, now, expires_at, total, mime_type, str(path)), + ) + return CachedImage(key, path, mime_type, total, "miss") + finally: + response.close() + + def _path_for_key(self, key: str, extension: str) -> Path: + return self.cache_dir / key[:2] / key[2:4] / f"{key}{extension}" + + def _is_fetch_allowed(self, url: str) -> bool: + parsed = urlparse(url) + if parsed.scheme not in {"http", "https"}: + return False + if parsed.username or parsed.password: + return False + if not parsed.hostname: + return False + + # Internal hosts are explicitly supported because Plex/Jellyfin/Navidrome + # artwork often lives behind Docker/LAN-only URLs. Public hosts are allowed + # as image-only responses with size limits. + return bool(parsed.hostname) or is_internal_image_host(url) + + def _connect(self) -> sqlite3.Connection: + conn = sqlite3.connect(self.db_path) + conn.row_factory = sqlite3.Row + return conn + + def _init_db(self) -> None: + with self._db_lock: + with self._connect() as conn: + conn.execute( + """ + CREATE TABLE IF NOT EXISTS image_cache ( + key TEXT PRIMARY KEY, + original_url TEXT NOT NULL, + status TEXT NOT NULL DEFAULT 'pending', + created_at REAL NOT NULL, + updated_at REAL NOT NULL, + last_accessed REAL NOT NULL, + expires_at REAL NOT NULL DEFAULT 0, + size INTEGER NOT NULL DEFAULT 0, + mime_type TEXT NOT NULL DEFAULT '', + file_path TEXT NOT NULL DEFAULT '', + last_error TEXT NOT NULL DEFAULT '' + ) + """ + ) + conn.execute("CREATE INDEX IF NOT EXISTS idx_image_cache_accessed ON image_cache(last_accessed)") + + def _get_row(self, key: str) -> Optional[sqlite3.Row]: + with self._db_lock: + with self._connect() as conn: + return conn.execute("SELECT * FROM image_cache WHERE key = ?", (key,)).fetchone() + + def _touch(self, key: str, now: float) -> None: + with self._db_lock: + with self._connect() as conn: + conn.execute("UPDATE image_cache SET last_accessed = ? WHERE key = ?", (now, key)) + + def _record_error(self, key: str, error: str, now: float, *, keep_status: bool = False) -> None: + status_sql = "status" if keep_status else "'failed'" + with self._db_lock: + with self._connect() as conn: + conn.execute( + f""" + UPDATE image_cache + SET status = {status_sql}, + updated_at = ?, + last_accessed = ?, + expires_at = ?, + last_error = ? + WHERE key = ? + """, + (now, now, now + self.failed_ttl_seconds, error[:500], key), + ) + + def _lock_for_key(self, key: str) -> threading.Lock: + with self._key_locks_lock: + lock = self._key_locks.get(key) + if lock is None: + lock = threading.Lock() + self._key_locks[key] = lock + return lock + + +_image_cache: Optional[ImageCache] = None +_image_cache_lock = threading.Lock() + + +def get_image_cache() -> ImageCache: + global _image_cache + with _image_cache_lock: + if _image_cache is None: + cache_dir = config_manager.get("image_cache.path", "storage/image_cache") + if not os.path.isabs(cache_dir): + cache_dir = str(config_manager.base_dir / cache_dir) + _image_cache = ImageCache( + cache_dir, + ttl_seconds=int(config_manager.get("image_cache.ttl_seconds", DEFAULT_TTL_SECONDS)), + failed_ttl_seconds=int(config_manager.get("image_cache.failed_ttl_seconds", DEFAULT_FAILED_TTL_SECONDS)), + max_download_bytes=int(config_manager.get("image_cache.max_download_mb", 15)) * 1024 * 1024, + ) + return _image_cache + + +def cached_image_url(url: str | None) -> str | None: + if not url or config_manager.get("image_cache.enabled", True) is False: + return url + try: + return get_image_cache().cache_url_for(url) + except Exception as exc: + logger.debug("image cache registration failed: %s", exc) + return url diff --git a/core/metadata/artwork.py b/core/metadata/artwork.py index 98af8431..694933d9 100644 --- a/core/metadata/artwork.py +++ b/core/metadata/artwork.py @@ -192,13 +192,13 @@ def normalize_image_url(thumb_url: str | None) -> str | None: def is_image_proxy_url(url: str) -> bool: - """Return True for SoulSync image-proxy URLs, absolute or relative.""" + """Return True for SoulSync image proxy/cache URLs, absolute or relative.""" if not url: return False try: parsed = urlparse(url) - return parsed.path == '/api/image-proxy' + return parsed.path == '/api/image-proxy' or parsed.path.startswith('/api/image-cache/') except Exception: return False @@ -235,10 +235,19 @@ def _browser_safe_image_url(url: str) -> str: if is_image_proxy_url(url): return url - if url.startswith('/api/image-proxy?url='): + if url.startswith('/api/image-proxy?url=') or url.startswith('/api/image-cache/'): return url if url.startswith('http://') or url.startswith('https://'): + try: + from core.image_cache import cached_image_url + + cached_url = cached_image_url(url) + if cached_url: + return cached_url + except Exception as exc: + logger.debug("image cache URL registration failed: %s", exc) + if is_internal_image_host(url): return f"/api/image-proxy?url={quote(url, safe='')}" return url diff --git a/tests/metadata/test_image_url_normalization.py b/tests/metadata/test_image_url_normalization.py index 2832ba4c..01ba4933 100644 --- a/tests/metadata/test_image_url_normalization.py +++ b/tests/metadata/test_image_url_normalization.py @@ -10,6 +10,7 @@ from urllib.parse import quote "thumb_url", [ "/api/image-proxy?url=https%3A%2F%2Fexample.com%2Fcover.jpg", + "/api/image-cache/" + ("a" * 64), "http://host.docker.internal:4533/api/image-proxy?u=ketiska&t=abc&s=def&v=1.16.1&c=SoulSync&f=json", ], ) @@ -20,10 +21,11 @@ def test_normalize_image_url_leaves_existing_image_proxy_urls_alone(thumb_url): assert normalize_image_url(thumb_url) == thumb_url -def test_normalize_image_url_proxies_internal_http_urls(monkeypatch): - """Raw internal image URLs should still be routed through SoulSync's proxy.""" +def test_normalize_image_url_registers_internal_http_urls_with_image_cache(monkeypatch): + """Raw internal image URLs should be routed through SoulSync's hashed cache URL.""" from core.metadata import normalize_image_url from core.metadata import artwork + import core.image_cache as image_cache class _FakeConfig: def get_active_media_server(self): @@ -39,6 +41,33 @@ def test_normalize_image_url_proxies_internal_http_urls(monkeypatch): return {} monkeypatch.setattr(artwork, "get_config_manager", lambda: _FakeConfig()) + monkeypatch.setattr(image_cache, "cached_image_url", lambda u: "/api/image-cache/" + ("b" * 64)) + + url = "http://localhost:4533/cover.jpg" + assert normalize_image_url(url) == "/api/image-cache/" + ("b" * 64) + + +def test_normalize_image_url_falls_back_to_proxy_when_cache_registration_fails(monkeypatch): + """If cache registration breaks, internal URLs keep the old image-proxy behavior.""" + from core.metadata import normalize_image_url + from core.metadata import artwork + import core.image_cache as image_cache + + class _FakeConfig: + def get_active_media_server(self): + return "spotify" + + def get_plex_config(self): + return {} + + def get_jellyfin_config(self): + return {} + + def get_navidrome_config(self): + return {} + + monkeypatch.setattr(artwork, "get_config_manager", lambda: _FakeConfig()) + monkeypatch.setattr(image_cache, "cached_image_url", lambda u: (_ for _ in ()).throw(RuntimeError("boom"))) url = "http://localhost:4533/cover.jpg" assert normalize_image_url(url) == f"/api/image-proxy?url={quote(url, safe='')}" diff --git a/tests/test_image_cache.py b/tests/test_image_cache.py new file mode 100644 index 00000000..f4556757 --- /dev/null +++ b/tests/test_image_cache.py @@ -0,0 +1,66 @@ +from __future__ import annotations + +from core.image_cache import ImageCache + + +class FakeResponse: + def __init__(self, body: bytes, *, status_code: int = 200, content_type: str = "image/jpeg"): + self.body = body + self.status_code = status_code + self.headers = { + "Content-Type": content_type, + "Content-Length": str(len(body)), + } + self.closed = False + + def iter_content(self, chunk_size=65536): + yield self.body + + def close(self): + self.closed = True + + +def test_cache_url_for_registers_hashed_browser_path(tmp_path): + cache = ImageCache(tmp_path) + url = "https://images.example.test/cover.jpg?token=secret" + + cached_url = cache.cache_url_for(url) + + assert cached_url == f"/api/image-cache/{ImageCache.key_for_url(url)}" + assert "secret" not in cached_url + + +def test_get_url_fetches_once_then_serves_cached_file(tmp_path): + calls = [] + + def fetcher(url, **kwargs): + calls.append((url, kwargs)) + return FakeResponse(b"fake-jpeg-bytes") + + cache = ImageCache(tmp_path, fetcher=fetcher) + url = "https://images.example.test/cover.jpg" + + first = cache.get_url(url) + second = cache.get_url(url) + + assert first.status == "miss" + assert second.status == "hit" + assert first.path == second.path + assert first.path.read_bytes() == b"fake-jpeg-bytes" + assert first.mime_type == "image/jpeg" + assert len(calls) == 1 + + +def test_get_url_rejects_non_image_responses(tmp_path): + cache = ImageCache( + tmp_path, + fetcher=lambda url, **kwargs: FakeResponse(b"", content_type="text/html"), + ) + + try: + cache.get_url("https://images.example.test/not-image") + except Exception as exc: + assert "not an image" in str(exc) + else: + raise AssertionError("Expected non-image response to fail") + diff --git a/web_server.py b/web_server.py index e87b4006..893b7cb2 100644 --- a/web_server.py +++ b/web_server.py @@ -28015,45 +28015,52 @@ def get_your_artist_info(artist_id): @app.route('/api/image-proxy', methods=['GET']) def image_proxy(): - """Proxy external images to avoid CORS issues for canvas rendering.""" + """Proxy external images to avoid CORS issues for canvas rendering. + + Kept for backwards compatibility; new normalized artwork URLs use + /api/image-cache/, but older browser sessions may still hold this + query-string form. + """ url = request.args.get('url', '') if not url or not url.startswith('http'): return '', 400 - host = urlparse(url).hostname or '' - allowed_hosts = [ - 'i.scdn.co', 'mosaic.scdn.co', # Spotify - 'lastfm.freetls.fastly.net', 'lastfm-img2.akamaized.net', # Last.fm - 'e-cdns-images.dzcdn.net', 'cdns-images.dzcdn.net', 'api.deezer.com', # Deezer - 'is1-ssl.mzstatic.com', 'is2-ssl.mzstatic.com', 'is3-ssl.mzstatic.com', - 'is4-ssl.mzstatic.com', 'is5-ssl.mzstatic.com', # iTunes/Apple - 'img.discogs.com', 'i.discogs.com', # Discogs - 'localhost', '127.0.0.1', 'host.docker.internal', # Local/Docker media servers - ] - if not any(host == h or host.endswith('.' + h) for h in allowed_hosts) and not is_internal_image_host(url): - return '', 403 try: - resp = requests.get(url, timeout=10, stream=True, headers={ - 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36', - 'Referer': 'https://www.deezer.com/', - }) - if resp.status_code != 200: - return '', resp.status_code - content_type = resp.headers.get('Content-Type', 'image/jpeg') - if not content_type.startswith('image/'): - return '', 400 - return Response( - resp.content, - content_type=content_type, - headers={ - 'Cache-Control': 'public, max-age=86400', - 'Access-Control-Allow-Origin': '*', - } - ) - except Exception: + from core.image_cache import get_image_cache + + cached = get_image_cache().get_url(url) + response = send_file(cached.path, mimetype=cached.mime_type, conditional=True) + max_age = int(config_manager.get("image_cache.ttl_seconds", 2592000)) + response.headers['Cache-Control'] = f'private, max-age={max_age}' + response.headers['Access-Control-Allow-Origin'] = '*' + response.headers['X-SoulSync-Image-Cache'] = cached.status + return response + except Exception as exc: + logger.debug("image proxy failed: %s", exc) return '', 502 +@app.route('/api/image-cache/', methods=['GET']) +def serve_cached_image(cache_key): + """Serve a registered image URL from SoulSync's disk cache.""" + if not re.fullmatch(r'[a-f0-9]{64}', cache_key or ''): + return '', 404 + + try: + from core.image_cache import get_image_cache + + cached = get_image_cache().get(cache_key) + response = send_file(cached.path, mimetype=cached.mime_type, conditional=True) + max_age = int(config_manager.get("image_cache.ttl_seconds", 2592000)) + response.headers['Cache-Control'] = f'private, max-age={max_age}' + response.headers['Access-Control-Allow-Origin'] = '*' + response.headers['X-SoulSync-Image-Cache'] = cached.status + return response + except Exception as exc: + logger.debug("cached image serve failed for %s: %s", cache_key, exc) + return '', 404 + + from core.artists.map import ( _artmap_cache_invalidate, _artmap_cache_get, From 2fc08e199eebd3e08209d1f29a012771e1606a9e Mon Sep 17 00:00:00 2001 From: Broque Thomas <26755000+Nezreka@users.noreply.github.com> Date: Wed, 20 May 2026 11:24:57 -0700 Subject: [PATCH 05/13] Enforce duration tolerance for strict sources Add duration tolerance logic and pre-download rejection for structured sources (tidal, qobuz, hifi, deezer_dl, amazon) when candidate duration deviates beyond allowed tolerance. Introduces helper functions _duration_tolerance_seconds and _duration_mismatch_exceeds_integrity_tolerance and uses resolve_duration_tolerance from core.imports.file_integrity. Log and skip candidates that would fail post-processing integrity checks to avoid wasted downloads. Update tests to include matching engine stub and new cases covering rejection and acceptance based on duration tolerance; also adjust imports and test fixtures. --- core/downloads/validation.py | 33 +++++++++++++++++ tests/downloads/test_downloads_validation.py | 37 ++++++++++++++++++-- 2 files changed, 68 insertions(+), 2 deletions(-) diff --git a/core/downloads/validation.py b/core/downloads/validation.py index 79131472..3ac38751 100644 --- a/core/downloads/validation.py +++ b/core/downloads/validation.py @@ -9,6 +9,7 @@ import logging import re from config.settings import config_manager +from core.imports.file_integrity import resolve_duration_tolerance logger = logging.getLogger(__name__) @@ -61,6 +62,24 @@ def filter_soundcloud_previews(results, expected_track): return [r for r in results if not _is_preview(r)] +def _duration_tolerance_seconds(expected_duration_ms): + override = resolve_duration_tolerance( + config_manager.get('post_processing.duration_tolerance_seconds', 0) + ) + if override is not None: + return override + expected_seconds = expected_duration_ms / 1000.0 + return 5.0 if expected_seconds > 600.0 else 3.0 + + +def _duration_mismatch_exceeds_integrity_tolerance(expected_duration_ms, candidate_duration_ms): + if not expected_duration_ms or not candidate_duration_ms: + return False + tolerance = _duration_tolerance_seconds(expected_duration_ms) + drift = abs((candidate_duration_ms / 1000.0) - (expected_duration_ms / 1000.0)) + return drift > tolerance + + def get_valid_candidates(results, spotify_track, query): """ This function is a direct port from sync.py. It scores and filters @@ -98,7 +117,21 @@ def get_valid_candidates(results, spotify_track, query): expected_is_version = any(kw in expected_title_lower for kw in _version_keywords) scored = [] + _strict_duration_sources = {'tidal', 'qobuz', 'hifi', 'deezer_dl', 'amazon'} for r in results: + if ( + r.username in _strict_duration_sources + and _duration_mismatch_exceeds_integrity_tolerance(expected_duration, r.duration or 0) + ): + logger.info( + "[%s] Rejecting candidate due to duration mismatch before download: " + "expected %.1fs, candidate %.1fs", + source_label, + expected_duration / 1000.0, + (r.duration or 0) / 1000.0, + ) + continue + # Score using matching engine's generic scorer (same weights as Soulseek) confidence, match_type = matching_engine.score_track_match( source_title=expected_title, diff --git a/tests/downloads/test_downloads_validation.py b/tests/downloads/test_downloads_validation.py index 62798523..ddb5346a 100644 --- a/tests/downloads/test_downloads_validation.py +++ b/tests/downloads/test_downloads_validation.py @@ -11,19 +11,32 @@ from __future__ import annotations from dataclasses import dataclass from typing import Optional -from core.downloads.validation import filter_soundcloud_previews +from core.downloads import validation +from core.downloads.validation import filter_soundcloud_previews, get_valid_candidates @dataclass class _Track: duration_ms: int + name: str = 'Song' + artists: tuple[str, ...] = ('Artist',) @dataclass class _Candidate: username: str duration: Optional[int] # milliseconds - title: str = '' + title: str = 'Song' + artist: str = 'Artist' + filename: str = 'candidate' + + +class _MatchingEngine: + def score_track_match(self, **kwargs): + return 0.99, 'core_title_match' + + def normalize_string(self, text): + return (text or '').lower() def test_drops_soundcloud_30s_preview_when_expected_long(): @@ -90,3 +103,23 @@ def test_keeps_soundcloud_candidate_at_threshold(): # 110s passes both checks: > 35s AND > 100s (half of 200s) cand = _Candidate(username='soundcloud', duration=110_000) assert filter_soundcloud_previews([cand], expected) == [cand] + + +def test_rejects_tidal_candidate_that_would_fail_integrity_duration(monkeypatch): + """Structured sources should not download candidates that post-processing + will immediately quarantine for the same duration mismatch.""" + monkeypatch.setattr(validation, 'matching_engine', _MatchingEngine()) + expected = _Track(duration_ms=338_000) + wrong_tidal = _Candidate(username='tidal', duration=30_000) + + assert get_valid_candidates([wrong_tidal], expected, 'Artist Song') == [] + + +def test_keeps_tidal_candidate_inside_integrity_duration_tolerance(monkeypatch): + monkeypatch.setattr(validation, 'matching_engine', _MatchingEngine()) + expected = _Track(duration_ms=338_000) + tidal = _Candidate(username='tidal', duration=340_000) + + result = get_valid_candidates([tidal], expected, 'Artist Song') + + assert result == [tidal] From a3f0018b2901e96b56ba7a582446a4d8488c35b4 Mon Sep 17 00:00:00 2001 From: Broque Thomas <26755000+Nezreka@users.noreply.github.com> Date: Wed, 20 May 2026 12:32:42 -0700 Subject: [PATCH 06/13] Bump release to 2.5.8 and add changelog Prepare 2.5.8 release: update the workflow default version_tag and the app _SOULSYNC_BASE_VERSION to 2.5.8, add WHATS_NEW entries for 2.5.8 (fix blank artist pages for Python/git-pull installs, fix premature download completion before post-processing, add disk-backed artwork cache with SQLite, and add pre-download duration tolerancing for strict sources), and update the whats-new fallback to 2.5.8. --- .github/workflows/docker-publish.yml | 4 ++-- web_server.py | 2 +- webui/static/helper.js | 9 ++++++++- 3 files changed, 11 insertions(+), 4 deletions(-) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index d11761e1..f049a16e 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -9,9 +9,9 @@ on: workflow_dispatch: inputs: version_tag: - description: 'Version tag (e.g. 2.5.7)' + description: 'Version tag (e.g. 2.5.8)' required: true - default: '2.5.7' + default: '2.5.8' jobs: build-and-push: diff --git a/web_server.py b/web_server.py index 893b7cb2..0fbb760c 100644 --- a/web_server.py +++ b/web_server.py @@ -40,7 +40,7 @@ logger = setup_logging(_log_level, _log_path) # App version — single source of truth for backup metadata, system-info, update check, etc. # Semver: MAJOR.MINOR.PATCH. Bump at each dev→main release. -_SOULSYNC_BASE_VERSION = "2.5.7" +_SOULSYNC_BASE_VERSION = "2.5.8" def _build_version_string(): """Append short commit hash to version when available (e.g. 2.35+abc1234).""" diff --git a/webui/static/helper.js b/webui/static/helper.js index 61df89d5..346dae36 100644 --- a/webui/static/helper.js +++ b/webui/static/helper.js @@ -3413,6 +3413,13 @@ function closeHelperSearch() { // projects that span multiple commits before shipping. Strip the flag at // release time and add a real `date:` line at the top of the version block. const WHATS_NEW = { + '2.5.8': [ + { date: 'May 20, 2026 — 2.5.8 release' }, + { title: 'Fix: blank artist pages on Python / git-pull installs', desc: 'PR #644 moved the artist detail page behind a TanStack React route. installs that pull from git but never run `npm install && npm run build` ship without the Vite bundle, so the legacy shell saw `/artist-detail//` URLs and bailed — every click left a blank pane. the legacy startup path now parses the URL itself and hands off to the existing artist detail loader, so Python users get artist pages back without needing to build the webui. Docker / built installs still take the React route as before.' }, + { title: 'Fix: downloads marked complete before post-processing finished', desc: 'monitored Soulseek transfers were getting flipped to "successful" the moment slskd reported the file done — before SoulSync had actually run the post-processing worker (find on disk, fingerprint-verify, import to library). a failed import after that point left a phantom "completed" row that never made it into the library. completion now waits for the real post-processing result; if the worker can\'t even be scheduled, the task is marked failed and the batch slot is released so the queue keeps moving.' }, + { title: 'Disk-backed artwork cache', desc: 'image fetches now route through a disk + SQLite cache with hashed URLs, size / mime validation, stale fallback when the upstream is down, and per-image fetch locking so 12 simultaneous requests for the same album cover share one network round-trip. cuts repeat-load latency and survives metadata source rate limits. served from new `/api/image-cache`; `/api/image-proxy` stays as a compatibility shim.' }, + { title: 'Strict-source downloads check duration before pulling', desc: 'Tidal / Qobuz / HiFi / Deezer-DL / Amazon candidates now get a duration-tolerance check before the download starts, using the same tolerance logic post-processing would apply after. tracks whose duration is far enough off the metadata reference to fail the integrity check are skipped at pick time instead of after wasting the download. Soulseek and YouTube unchanged (they don\'t expose reliable pre-download duration).' }, + ], '2.5.7': [ { date: 'May 19, 2026 — 2.5.7 release' }, { title: 'Fix: MusicBrainz manual search missing results', desc: 'the Fix popup and manual library service search were using strict Lucene phrase-match queries against the `recording` / `release` / `artist` fields — diacritics ("Bjork" vs canonical "Björk"), bracketed suffixes like "(Live)", and any AND-clause mismatch all killed recall. switched user-facing manual lookups to bare queries that hit MB\'s alias / sortname indexes with diacritic folding. enrichment workers stay strict for precision.' }, @@ -3795,7 +3802,7 @@ function _getLatestWhatsNewVersion() { const versions = Object.keys(WHATS_NEW) .filter(v => _compareVersions(v, buildVer) <= 0) .sort((a, b) => _compareVersions(b, a)); - return versions[0] || '2.5.7'; + return versions[0] || '2.5.8'; } function openWhatsNew() { From 3375b6c4bd48b97ab789673c8a949d1f1dc1d1e1 Mon Sep 17 00:00:00 2001 From: Broque Thomas <26755000+Nezreka@users.noreply.github.com> Date: Wed, 20 May 2026 14:04:45 -0700 Subject: [PATCH 07/13] Handle non-JSON Tidal auth responses Detect JSON decode-like exceptions from Tidal's token endpoint and return a safer, more actionable error message. Adds a _looks_like_json_decode_error helper and special-cases that error in check_device_auth to log the non-JSON response and advise disabling VPN/proxy/network filtering and restarting SoulSync. A test was added to ensure the user-facing message does not leak the raw exception text while still returning an error status. Other errors continue to fall back to the existing behavior. --- core/tidal_download_client.py | 22 ++++++++++++++++++++++ tests/downloads/test_tidal_pinning.py | 20 ++++++++++++++++++++ 2 files changed, 42 insertions(+) diff --git a/core/tidal_download_client.py b/core/tidal_download_client.py index a7efb40f..6ae2027e 100644 --- a/core/tidal_download_client.py +++ b/core/tidal_download_client.py @@ -95,6 +95,16 @@ HLS_MAP_TAG_RE = re.compile(r'#EXT-X-MAP:.*URI="([^"]+)"') from core.download_plugins.base import DownloadSourcePlugin +def _looks_like_json_decode_error(exc: Exception) -> bool: + name = exc.__class__.__name__.lower() + message = str(exc).lower() + return ( + "jsondecodeerror" in name + or "expecting value" in message + or "could not decode json" in message + ) + + class TidalDownloadClient(DownloadSourcePlugin): """ Tidal download client using tidalapi. @@ -230,6 +240,18 @@ class TidalDownloadClient(DownloadSourcePlugin): return {'status': 'error', 'message': 'Auth completed but session invalid'} except Exception as e: + if _looks_like_json_decode_error(e): + logger.error( + "Tidal device auth check received a non-JSON response from Tidal's token endpoint: %s", + e, + ) + return { + 'status': 'error', + 'message': ( + "Tidal returned an invalid auth response while SoulSync was finishing login. " + "Try again after disabling VPN/proxy/network filtering, then restart SoulSync if it repeats." + ), + } logger.error(f"Tidal device auth check error: {e}") return {'status': 'error', 'message': str(e)} diff --git a/tests/downloads/test_tidal_pinning.py b/tests/downloads/test_tidal_pinning.py index 74327f37..bb4db8e9 100644 --- a/tests/downloads/test_tidal_pinning.py +++ b/tests/downloads/test_tidal_pinning.py @@ -61,6 +61,26 @@ def test_is_authenticated_false_when_session_check_login_raises(tidal_client_wit assert client.is_authenticated() is False +def test_check_device_auth_returns_helpful_message_for_non_json_tidal_response(tidal_client_with_engine): + client, _ = tidal_client_with_engine + + class FakeFuture: + def running(self): + return False + + def result(self, timeout=0): + raise ValueError("Expecting value: line 1 column 1 (char 0)") + + client._device_auth_future = FakeFuture() + client._device_auth_link = {'verification_uri': 'https://link.tidal.com', 'user_code': 'ABCD'} + + result = client.check_device_auth() + + assert result['status'] == 'error' + assert 'invalid auth response' in result['message'] + assert 'Expecting value' not in result['message'] + + # --------------------------------------------------------------------------- # download() — filename parsing + id contract # --------------------------------------------------------------------------- From 579eff8807fcfb15916e1df8cc6aa60ee6ef95bc Mon Sep 17 00:00:00 2001 From: Broque Thomas <26755000+Nezreka@users.noreply.github.com> Date: Wed, 20 May 2026 14:41:54 -0700 Subject: [PATCH 08/13] feat(settings): add Prowlarr integration as indexer aggregator MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit First commit toward torrent and usenet download sources. Prowlarr is the indexer manager component of the *arr stack — it exposes Usenet and torrent indexers behind a single Newznab-style API so SoulSync doesn't have to integrate each indexer individually. This commit wires up Prowlarr as a search-only source; the torrent and usenet download client adapters land in the next commits and plug into this search surface. - core/prowlarr_client.py: sync-backed async client. is_configured, check_connection, get_indexers, search by Newznab category. Music category constants (3000 all / 3010 MP3 / 3040 lossless / etc.). - core/connection_test.py: 'prowlarr' branch hits /api/v1/system/status for the Test Connection button. - web_server.py: GET /api/prowlarr/indexers returns the live indexer list (id, name, protocol, enabled, privacy). Settings POST allow-list now accepts 'prowlarr' so saved config persists. - config/settings.py: prowlarr.{url, api_key, indexer_ids} defaults plus prowlarr.api_key in the encrypted-at-rest secrets list. - webui/index.html: new "Indexers & Downloaders" tab on Settings with the Prowlarr panel (URL, API key, Test, Refresh Indexer List, optional indexer-ID allowlist). - webui/static/settings.js: load/save wiring, testProwlarrConnection, loadProwlarrIndexers (HTML-escapes user-supplied indexer names). - webui/static/helper.js: WHATS_NEW 2.6.0 unreleased block plus a curated VERSION_MODAL_SECTIONS entry. --- config/settings.py | 10 ++ core/connection_test.py | 15 +++ core/prowlarr_client.py | 241 +++++++++++++++++++++++++++++++++++++++ web_server.py | 32 +++++- webui/index.html | 50 ++++++++ webui/static/helper.js | 17 +++ webui/static/settings.js | 66 +++++++++++ 7 files changed, 430 insertions(+), 1 deletion(-) create mode 100644 core/prowlarr_client.py diff --git a/config/settings.py b/config/settings.py index b2c6b8c4..21a7bfe5 100644 --- a/config/settings.py +++ b/config/settings.py @@ -87,6 +87,7 @@ class ConfigManager: 'soulseek.api_key', 'deezer_download.arl', 'lidarr_download.api_key', + 'prowlarr.api_key', # Enrichment services 'listenbrainz.token', 'acoustid.api_key', @@ -519,6 +520,15 @@ class ConfigManager: "quality_profile": "Any", "cleanup_after_import": True, }, + # Prowlarr — indexer aggregator. Feeds the torrent / usenet + # download plugins. Not a standalone source. + "prowlarr": { + "url": "", + "api_key": "", + # Comma-separated list of indexer IDs to limit searches to. + # Empty = search all enabled indexers. + "indexer_ids": "", + }, "soundcloud_download": { # Anonymous-only for now — SoundCloud Go+ OAuth tier could be # added later, with credentials living under a "session" subkey diff --git a/core/connection_test.py b/core/connection_test.py index 2019b4fe..ae866258 100644 --- a/core/connection_test.py +++ b/core/connection_test.py @@ -305,6 +305,21 @@ def run_service_test(service, test_config): return False, "Invalid Genius access token." except Exception as e: return False, f"Genius connection error: {str(e)}" + elif service == "prowlarr": + url = config_manager.get('prowlarr.url', '') + api_key = config_manager.get('prowlarr.api_key', '') + if not url or not api_key: + return False, "Prowlarr URL and API key are required." + try: + import requests as _req + resp = _req.get(f"{url.rstrip('/')}/api/v1/system/status", + headers={'X-Api-Key': api_key}, timeout=10) + if resp.ok: + version = resp.json().get('version', '?') + return True, f"Connected to Prowlarr v{version}" + return False, f"Prowlarr returned HTTP {resp.status_code}" + except Exception as e: + return False, f"Prowlarr connection error: {str(e)}" elif service == "lidarr" or service == "lidarr_download": url = config_manager.get('lidarr_download.url', '') api_key = config_manager.get('lidarr_download.api_key', '') diff --git a/core/prowlarr_client.py b/core/prowlarr_client.py new file mode 100644 index 00000000..4290f336 --- /dev/null +++ b/core/prowlarr_client.py @@ -0,0 +1,241 @@ +"""Prowlarr client — indexer aggregator. + +Prowlarr is the indexer manager component of the *arr stack. It exposes +configured Usenet / torrent indexers behind a single Newznab-style API +so downstream apps (Lidarr, Sonarr, Radarr, SoulSync) don't have to +implement an indexer integration per provider. + +This client is NOT a download source plugin. It does not implement +``DownloadSourcePlugin`` — Prowlarr only *searches*. The torrent / +usenet download plugins (built in subsequent commits) own the +add-to-client / poll-status / extract flow and call this client for +the search step. + +Surface: +- ``is_configured()`` — URL + API key present. +- ``check_connection()`` — hits ``/api/v1/system/status``. +- ``get_indexers()`` — list of configured indexers (id, name, protocol, + capabilities). +- ``search(query, categories, indexer_ids)`` — Newznab search across + selected indexers. Music categories default to the full audio tree. + +Auth: ``X-Api-Key`` header. Found in Prowlarr → Settings → General → +Security → API Key. +""" + +from __future__ import annotations + +import asyncio +from dataclasses import dataclass, field +from typing import Any, Dict, List, Optional, Sequence + +import requests as http_requests + +from config.settings import config_manager +from utils.logging_config import get_logger + +logger = get_logger("prowlarr_client") + + +# Newznab Music category tree. Prowlarr / Jackett / Newznab indexers +# all agree on these numeric IDs. 3000 is the parent — most indexers +# tag releases against the parent OR a leaf; searching the parent +# pulls everything. +MUSIC_CATEGORY_ALL = 3000 +MUSIC_CATEGORY_MP3 = 3010 +MUSIC_CATEGORY_VIDEO = 3020 +MUSIC_CATEGORY_AUDIOBOOK = 3030 +MUSIC_CATEGORY_LOSSLESS = 3040 +MUSIC_CATEGORY_OTHER = 3050 +MUSIC_CATEGORY_FOREIGN = 3060 + +DEFAULT_MUSIC_CATEGORIES: tuple = ( + MUSIC_CATEGORY_ALL, + MUSIC_CATEGORY_MP3, + MUSIC_CATEGORY_LOSSLESS, + MUSIC_CATEGORY_OTHER, +) + + +@dataclass +class ProwlarrIndexer: + """One configured indexer exposed by Prowlarr.""" + + id: int + name: str + protocol: str # "torrent" | "usenet" + enable: bool + privacy: str # "public" | "private" | "semiPrivate" + categories: List[int] = field(default_factory=list) + capabilities: Dict[str, Any] = field(default_factory=dict) + + +@dataclass +class ProwlarrSearchResult: + """One release returned by a Prowlarr search. + + ``download_url`` is the link the torrent / usenet client gets fed. + For torrent indexers it may be either a ``.torrent`` HTTP URL or + a magnet URI (sometimes both — ``magnet_uri`` is set when the + indexer exposes the magnet separately). + """ + + guid: str + title: str + indexer_id: int + indexer_name: str + protocol: str # "torrent" | "usenet" + download_url: Optional[str] = None + magnet_uri: Optional[str] = None + info_url: Optional[str] = None + size: int = 0 # bytes + seeders: Optional[int] = None + leechers: Optional[int] = None + grabs: Optional[int] = None + publish_date: Optional[str] = None + categories: List[int] = field(default_factory=list) + raw: Dict[str, Any] = field(default_factory=dict) + + +class ProwlarrClient: + """Thin sync-backed async wrapper around the Prowlarr v1 API.""" + + DEFAULT_TIMEOUT = 15 + + def __init__(self) -> None: + self._load_config() + + def _load_config(self) -> None: + self._url = (config_manager.get('prowlarr.url', '') or '').rstrip('/') + self._api_key = config_manager.get('prowlarr.api_key', '') or '' + + def reload_settings(self) -> None: + self._load_config() + logger.info("Prowlarr settings reloaded") + + def is_configured(self) -> bool: + return bool(self._url and self._api_key) + + async def check_connection(self) -> bool: + if not self.is_configured(): + return False + loop = asyncio.get_event_loop() + return await loop.run_in_executor(None, self._check_connection_sync) + + def _check_connection_sync(self) -> bool: + data = self._api_get('system/status') + return bool(data and 'version' in data) + + async def get_indexers(self) -> List[ProwlarrIndexer]: + if not self.is_configured(): + return [] + loop = asyncio.get_event_loop() + return await loop.run_in_executor(None, self._get_indexers_sync) + + def _get_indexers_sync(self) -> List[ProwlarrIndexer]: + data = self._api_get('indexer') + if not isinstance(data, list): + return [] + return [self._parse_indexer(entry) for entry in data if isinstance(entry, dict)] + + async def search( + self, + query: str, + categories: Sequence[int] = DEFAULT_MUSIC_CATEGORIES, + indexer_ids: Optional[Sequence[int]] = None, + limit: int = 100, + ) -> List[ProwlarrSearchResult]: + """Run a Newznab search across the selected indexers. + + ``indexer_ids`` is the list of Prowlarr internal indexer IDs to + query. ``None`` means all enabled indexers. + """ + if not self.is_configured() or not query.strip(): + return [] + loop = asyncio.get_event_loop() + return await loop.run_in_executor( + None, self._search_sync, query, list(categories), list(indexer_ids or []), limit + ) + + def _search_sync( + self, + query: str, + categories: List[int], + indexer_ids: List[int], + limit: int, + ) -> List[ProwlarrSearchResult]: + # Prowlarr's search endpoint accepts repeated params: ``categories=3000&categories=3010``. + # ``requests`` serializes lists in that exact form when passed as tuples of pairs. + params: List[tuple] = [('query', query), ('type', 'search'), ('limit', limit)] + for cat in categories: + params.append(('categories', cat)) + for indexer_id in indexer_ids: + params.append(('indexerIds', indexer_id)) + + data = self._api_get('search', params=params) + if not isinstance(data, list): + return [] + return [self._parse_result(entry) for entry in data if isinstance(entry, dict)] + + def _parse_indexer(self, entry: Dict[str, Any]) -> ProwlarrIndexer: + return ProwlarrIndexer( + id=int(entry.get('id') or 0), + name=entry.get('name') or '', + protocol=entry.get('protocol') or '', + enable=bool(entry.get('enable', True)), + privacy=entry.get('privacy') or '', + categories=[int(c.get('id') or 0) for c in entry.get('capabilities', {}).get('categories', []) if isinstance(c, dict)], + capabilities=entry.get('capabilities', {}) or {}, + ) + + def _parse_result(self, entry: Dict[str, Any]) -> ProwlarrSearchResult: + cats = entry.get('categories') or [] + category_ids: List[int] = [] + for cat in cats: + if isinstance(cat, dict) and cat.get('id') is not None: + try: + category_ids.append(int(cat['id'])) + except (TypeError, ValueError): + continue + elif isinstance(cat, int): + category_ids.append(cat) + + return ProwlarrSearchResult( + guid=str(entry.get('guid') or entry.get('infoUrl') or entry.get('downloadUrl') or ''), + title=entry.get('title') or '', + indexer_id=int(entry.get('indexerId') or 0), + indexer_name=entry.get('indexer') or '', + protocol=entry.get('protocol') or '', + download_url=entry.get('downloadUrl') or None, + magnet_uri=entry.get('magnetUrl') or None, + info_url=entry.get('infoUrl') or None, + size=int(entry.get('size') or 0), + seeders=entry.get('seeders'), + leechers=entry.get('leechers'), + grabs=entry.get('grabs'), + publish_date=entry.get('publishDate'), + categories=category_ids, + raw=entry, + ) + + def _api_get(self, path: str, params=None) -> Optional[Any]: + if not self.is_configured(): + return None + url = f"{self._url}/api/v1/{path.lstrip('/')}" + try: + resp = http_requests.get( + url, + headers={'X-Api-Key': self._api_key, 'Accept': 'application/json'}, + params=params, + timeout=self.DEFAULT_TIMEOUT, + ) + if not resp.ok: + logger.warning("Prowlarr %s returned HTTP %s", path, resp.status_code) + return None + return resp.json() + except http_requests.exceptions.RequestException as e: + logger.error("Prowlarr request to %s failed: %s", path, e) + return None + except ValueError as e: + logger.error("Prowlarr response to %s was not JSON: %s", path, e) + return None diff --git a/web_server.py b/web_server.py index 0fbb760c..fa697733 100644 --- a/web_server.py +++ b/web_server.py @@ -2753,7 +2753,7 @@ def handle_settings(): if 'active_media_server' in new_settings: config_manager.set_active_media_server(new_settings['active_media_server']) - for service in ['spotify', 'plex', 'jellyfin', 'navidrome', 'soulseek', 'download_source', 'settings', 'database', 'metadata_enhancement', 'file_organization', 'playlist_sync', 'tidal', 'tidal_download', 'qobuz', 'hifi_download', 'deezer_download', 'amazon_download', 'lidarr_download', 'listenbrainz', 'acoustid', 'lastfm', 'genius', 'import', 'lossy_copy', 'listening_stats', 'ui_appearance', 'youtube', 'content_filter', 'itunes', 'm3u_export', 'musicbrainz', 'deezer', 'audiodb', 'metadata', 'hydrabase', 'security', 'discogs', 'library', 'discover', 'wishlist', 'genre_whitelist', 'post_processing']: + for service in ['spotify', 'plex', 'jellyfin', 'navidrome', 'soulseek', 'download_source', 'settings', 'database', 'metadata_enhancement', 'file_organization', 'playlist_sync', 'tidal', 'tidal_download', 'qobuz', 'hifi_download', 'deezer_download', 'amazon_download', 'lidarr_download', 'prowlarr', 'listenbrainz', 'acoustid', 'lastfm', 'genius', 'import', 'lossy_copy', 'listening_stats', 'ui_appearance', 'youtube', 'content_filter', 'itunes', 'm3u_export', 'musicbrainz', 'deezer', 'audiodb', 'metadata', 'hydrabase', 'security', 'discogs', 'library', 'discover', 'wishlist', 'genre_whitelist', 'post_processing']: if service in new_settings: for key, value in new_settings[service].items(): config_manager.set(f'{service}.{key}', value) @@ -3043,6 +3043,36 @@ def hydrabase_send(): _hydrabase_ws = None return jsonify({"success": False, "error": str(e)}), 500 +@app.route('/api/prowlarr/indexers', methods=['GET']) +def prowlarr_indexers_endpoint(): + """List indexers Prowlarr currently exposes — name, protocol, enabled state. + + Drives the Indexers panel on Settings → Indexers & Downloaders so + the user can see what they're searching against without leaving + SoulSync. Returns ``[]`` if Prowlarr isn't configured / reachable. + """ + try: + from core.prowlarr_client import ProwlarrClient + client = ProwlarrClient() + if not client.is_configured(): + return jsonify({"success": False, "error": "Prowlarr not configured", "indexers": []}), 200 + indexers = run_async(client.get_indexers()) + items = [ + { + 'id': idx.id, + 'name': idx.name, + 'protocol': idx.protocol, + 'enable': idx.enable, + 'privacy': idx.privacy, + } + for idx in indexers + ] + return jsonify({"success": True, "indexers": items}) + except Exception as e: + logger.error(f"prowlarr indexers fetch error: {e}") + return jsonify({"success": False, "error": str(e), "indexers": []}), 500 + + @app.route('/api/settings/log-level', methods=['GET', 'POST']) @admin_only def handle_log_level(): diff --git a/webui/index.html b/webui/index.html index 4bcfba98..fc4b1c87 100644 --- a/webui/index.html +++ b/webui/index.html @@ -3635,6 +3635,7 @@
+ @@ -4945,6 +4946,55 @@
+ +
+

🔎 Prowlarr (Indexer Aggregator)

+
+ Prowlarr manages your Usenet and torrent indexers and exposes them through one API. SoulSync uses Prowlarr to search across every indexer at once for the Torrent and Usenet download sources. +

+ Don't have it? Grab Prowlarr from prowlarr.com (or your *arr stack). You point Prowlarr at your indexers, then point SoulSync at Prowlarr. +
+
+ + +
+ Full URL to your Prowlarr instance (e.g. http://192.168.1.100:9696). +
+
+
+ + +
+ Found in Prowlarr → Settings → General → Security → API Key. +
+
+
+ +
+ + + +
+
+
+ +
+ Connect to Prowlarr and click Refresh Indexer List to see the indexers SoulSync can search. +
+
+
+ + +
+ Comma-separated Prowlarr indexer IDs. Leave blank to search every enabled indexer. Restrict when you have a private tracker you want to prioritise or a noisy public one to exclude. +
+
+
+ + +
+

🧲 Torrent Client

+
+ Where SoulSync sends torrents once Prowlarr finds them. Pick the client you already use — only one can be active at a time. Add the URL of the client's WebUI, log in if it asks for credentials, and hit Test Connection. +
+
+ + +
+ qBittorrent: WebUI port (default 8080). Transmission: RPC port (default 9091). Deluge: WebUI port (default 8112). +
+
+
+ + +
+
+ + +
+ qBittorrent and Transmission use username + password. Deluge uses password only — paste it in the password field below. +
+
+
+ + +
+
+ + +
+ SoulSync tags every torrent it adds with this label so it's easy to spot in your client. Deluge needs the Label plugin installed for this to stick. +
+
+
+ + +
+ Override where the torrent client writes downloads. This path is on the torrent client's machine, not SoulSync's. +
+
+
+ +
+ + +
+
+
+ + +
+

📰 Usenet Client

+
+ Where SoulSync sends NZBs once Prowlarr finds them. Pick one usenet downloader. SABnzbd uses an API key for auth; NZBGet uses a username + password. +
+
+ + +
+ SABnzbd: default WebUI port 8080. NZBGet: default WebUI port 6789. +
+
+
+ + +
+
+ + +
+ SABnzbd → Config → General → API Key. Used by SABnzbd only. +
+
+ + +
+ + +
+ SoulSync tags every NZB with this category so it ends up in a predictable post-processing folder. +
+
+
+ +
+ + +
+
+
+ -
-

🔎 Prowlarr (Indexer Aggregator)

-
- Prowlarr manages your Usenet and torrent indexers and exposes them through one API. SoulSync uses Prowlarr to search across every indexer at once for the Torrent and Usenet download sources. -

- Don't have it? Grab Prowlarr from prowlarr.com (or your *arr stack). You point Prowlarr at your indexers, then point SoulSync at Prowlarr. + +
+
Indexers & Downloaders
+
+ 1 Indexers find releases + → + 2 Downloader fetches them + → + 3 SoulSync imports to library
-
- - -
- Full URL to your Prowlarr instance (e.g. http://192.168.1.100:9696). -
+
+ Configure your Prowlarr instance to feed search results, and pick a torrent client and/or usenet client to handle the actual downloads. You only need to set up the protocols you actually use.
-
- - -
- Found in Prowlarr → Settings → General → Security → API Key. -
-
-
- -
- - - -
-
-
- -
- Connect to Prowlarr and click Refresh Indexer List to see the indexers SoulSync can search. -
-
-
- - -
- Comma-separated Prowlarr indexer IDs. Leave blank to search every enabled indexer. Restrict when you have a private tracker you want to prioritise or a noisy public one to exclude. +
+ + +
+ ▼ +

🔎 Indexers

+ Prowlarr — search aggregator + +
+
+
+
+

⬢ Prowlarr

+
+ Prowlarr manages your Usenet and torrent indexers and exposes them through one API. SoulSync uses Prowlarr to search across every indexer at once for the Torrent and Usenet download sources. +

+ Don't have it? Grab Prowlarr from prowlarr.com (or your *arr stack). You point Prowlarr at your indexers, then point SoulSync at Prowlarr. +
+
+ + +
+ Full URL to your Prowlarr instance (e.g. http://192.168.1.100:9696). +
+
+
+ + +
+ Found in Prowlarr → Settings → General → Security → API Key. +
+
+
+ + +
+ Comma-separated Prowlarr indexer IDs. Leave blank to search every enabled indexer. Restrict when you have a private tracker you want to prioritise or a noisy public one to exclude. +
+
+
+ +
+ + + +
+
+
+ +
+ Connect to Prowlarr and click Refresh Indexer List to see the indexers SoulSync can search. +
+
- -
+ +