From e84d1bcde56e384c72aa1266a4c0c390b9c65474 Mon Sep 17 00:00:00 2001 From: BoulderBadgeDad Date: Fri, 26 Jun 2026 10:27:28 -0700 Subject: [PATCH] slskd search: throttle creation to avoid 429 storms MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit the real cause of the movie-search failures: slskd rate-limits search CREATION and 429s when exceeded — and the video path had NO throttle (the music side caps ~35/220s). the thread-pool auto-grab fired searches in bursts (and each 429 returned instantly, cascading into more), storming slskd into 429s; only a few slipped through. add a shared throttle on start_search: min 2s between creations + a 35-per-220s window cap (mirrors music) + a cooldown when a 429 IS hit (honors Retry-After) so it backs off instead of hammering. pure reserve/cooldown logic tested. --- core/video/slskd_search.py | 49 ++++++++++++++++++++++++++++++++ tests/test_video_slskd_search.py | 32 +++++++++++++++++++++ 2 files changed, 81 insertions(+) diff --git a/core/video/slskd_search.py b/core/video/slskd_search.py index 86e04c28..cefa68fa 100644 --- a/core/video/slskd_search.py +++ b/core/video/slskd_search.py @@ -13,11 +13,56 @@ the HTTP poll is thin I/O glue. from __future__ import annotations +import threading import time from typing import Any import requests +# slskd rate-limits search CREATION and returns 429 when exceeded. The music side caps at +# ~35 searches / 220s; we mirror that, plus a small min-gap between creations (the auto-grab +# fires from a thread pool, so without it a burst — or a 429 that returns instantly — storms +# slskd). A 429 sets a cooldown so we back off instead of hammering. +_SEARCH_LOCK = threading.Lock() +_SEARCH_TIMES: list = [] # reserved creation times (monotonic), pruned to the window +_COOLDOWN_UNTIL = [0.0] +_MAX_PER_WINDOW = 35 +_WINDOW_SECONDS = 220.0 +_MIN_GAP_SECONDS = 2.0 + + +def _reserve_search_slot() -> float: + """Reserve the next allowed search-creation time under the rate limit; returns it.""" + with _SEARCH_LOCK: + now = time.monotonic() + while _SEARCH_TIMES and _SEARCH_TIMES[0] <= now - _WINDOW_SECONDS: + _SEARCH_TIMES.pop(0) + at = now + if _SEARCH_TIMES: + at = max(at, _SEARCH_TIMES[-1] + _MIN_GAP_SECONDS) # space from last + if len(_SEARCH_TIMES) >= _MAX_PER_WINDOW: + at = max(at, _SEARCH_TIMES[0] + _WINDOW_SECONDS) # window full → wait + at = max(at, _COOLDOWN_UNTIL[0]) # honor a 429 cooldown + _SEARCH_TIMES.append(at) + return at + + +def _throttle_search() -> None: + """Block until we're allowed to create the next slskd search.""" + wait = _reserve_search_slot() - time.monotonic() + if wait > 0: + time.sleep(wait) + + +def _note_rate_limited(retry_after: Any = None) -> None: + """slskd returned 429 — back off before the next search.""" + try: + secs = float(retry_after) if retry_after else 30.0 + except (TypeError, ValueError): + secs = 30.0 + with _SEARCH_LOCK: + _COOLDOWN_UNTIL[0] = time.monotonic() + max(5.0, min(secs, 120.0)) + # Container extensions we treat as the actual video (everything else — subs, nfo, # art, samples — is ignored for quality purposes). VIDEO_EXTS = frozenset(( @@ -141,12 +186,16 @@ def start_search(query: str) -> dict: base, headers = _conn() if not base: return {"configured": False} + _throttle_search() # stay under slskd's search-creation rate limit search_id = str(uuid.uuid4()) payload = {"id": search_id, "searchText": query, "timeout": search_timeout_ms(), "filterResponses": True, "minimumResponseFileCount": 1, "minimumPeerUploadSpeed": _min_speed_bytes()} try: r = requests.post(base + "/api/v0/searches", json=payload, headers=headers, timeout=15) + if r.status_code == 429: # rate limited — back off, don't cascade + _note_rate_limited(r.headers.get("Retry-After")) + return {"configured": True, "error": "429 rate limited (backing off)"} r.raise_for_status() except Exception as e: # noqa: BLE001 - surface any slskd/network failure to the UI return {"configured": True, "error": str(e)} diff --git a/tests/test_video_slskd_search.py b/tests/test_video_slskd_search.py index c5d3052c..413d6e2f 100644 --- a/tests/test_video_slskd_search.py +++ b/tests/test_video_slskd_search.py @@ -49,3 +49,35 @@ def test_group_skips_non_video_and_samples(): def test_group_handles_garbage(): assert group_video_files(None) == [] assert group_video_files([{"nope": 1}, "junk"]) == [] + + +# ── search-creation rate-limit throttle (avoids slskd 429s) ────────────────── +import core.video.slskd_search as _ss # noqa: E402 + + +def _reset_throttle(): + _ss._SEARCH_TIMES.clear() + _ss._COOLDOWN_UNTIL[0] = 0.0 + + +def test_throttle_spaces_consecutive_creations(): + _reset_throttle() + t1 = _ss._reserve_search_slot() + t2 = _ss._reserve_search_slot() + assert t2 - t1 >= _ss._MIN_GAP_SECONDS - 0.01 # min gap between creations + + +def test_throttle_window_cap_holds_the_overflow(): + _reset_throttle() + times = [_ss._reserve_search_slot() for _ in range(_ss._MAX_PER_WINDOW + 1)] + # the one past the window cap waits ~a full window past the first + assert times[-1] >= times[0] + _ss._WINDOW_SECONDS - 0.5 + + +def test_429_sets_a_cooldown(): + import time + _reset_throttle() + _ss._note_rate_limited("10") # Retry-After: 10s + nxt = _ss._reserve_search_slot() + assert nxt >= time.monotonic() + 8 # next search waits out the cooldown + _reset_throttle()