slskd search: throttle creation to avoid 429 storms

the real cause of the movie-search failures: slskd rate-limits search CREATION and
429s when exceeded — and the video path had NO throttle (the music side caps ~35/220s).
the thread-pool auto-grab fired searches in bursts (and each 429 returned instantly,
cascading into more), storming slskd into 429s; only a few slipped through.

add a shared throttle on start_search: min 2s between creations + a 35-per-220s window
cap (mirrors music) + a cooldown when a 429 IS hit (honors Retry-After) so it backs off
instead of hammering. pure reserve/cooldown logic tested.
This commit is contained in:
BoulderBadgeDad 2026-06-26 10:27:28 -07:00
parent 6a1fa78995
commit e84d1bcde5
2 changed files with 81 additions and 0 deletions

View file

@ -13,11 +13,56 @@ the HTTP poll is thin I/O glue.
from __future__ import annotations
import threading
import time
from typing import Any
import requests
# slskd rate-limits search CREATION and returns 429 when exceeded. The music side caps at
# ~35 searches / 220s; we mirror that, plus a small min-gap between creations (the auto-grab
# fires from a thread pool, so without it a burst — or a 429 that returns instantly — storms
# slskd). A 429 sets a cooldown so we back off instead of hammering.
_SEARCH_LOCK = threading.Lock()
_SEARCH_TIMES: list = [] # reserved creation times (monotonic), pruned to the window
_COOLDOWN_UNTIL = [0.0]
_MAX_PER_WINDOW = 35
_WINDOW_SECONDS = 220.0
_MIN_GAP_SECONDS = 2.0
def _reserve_search_slot() -> float:
"""Reserve the next allowed search-creation time under the rate limit; returns it."""
with _SEARCH_LOCK:
now = time.monotonic()
while _SEARCH_TIMES and _SEARCH_TIMES[0] <= now - _WINDOW_SECONDS:
_SEARCH_TIMES.pop(0)
at = now
if _SEARCH_TIMES:
at = max(at, _SEARCH_TIMES[-1] + _MIN_GAP_SECONDS) # space from last
if len(_SEARCH_TIMES) >= _MAX_PER_WINDOW:
at = max(at, _SEARCH_TIMES[0] + _WINDOW_SECONDS) # window full → wait
at = max(at, _COOLDOWN_UNTIL[0]) # honor a 429 cooldown
_SEARCH_TIMES.append(at)
return at
def _throttle_search() -> None:
"""Block until we're allowed to create the next slskd search."""
wait = _reserve_search_slot() - time.monotonic()
if wait > 0:
time.sleep(wait)
def _note_rate_limited(retry_after: Any = None) -> None:
"""slskd returned 429 — back off before the next search."""
try:
secs = float(retry_after) if retry_after else 30.0
except (TypeError, ValueError):
secs = 30.0
with _SEARCH_LOCK:
_COOLDOWN_UNTIL[0] = time.monotonic() + max(5.0, min(secs, 120.0))
# Container extensions we treat as the actual video (everything else — subs, nfo,
# art, samples — is ignored for quality purposes).
VIDEO_EXTS = frozenset((
@ -141,12 +186,16 @@ def start_search(query: str) -> dict:
base, headers = _conn()
if not base:
return {"configured": False}
_throttle_search() # stay under slskd's search-creation rate limit
search_id = str(uuid.uuid4())
payload = {"id": search_id, "searchText": query, "timeout": search_timeout_ms(),
"filterResponses": True, "minimumResponseFileCount": 1,
"minimumPeerUploadSpeed": _min_speed_bytes()}
try:
r = requests.post(base + "/api/v0/searches", json=payload, headers=headers, timeout=15)
if r.status_code == 429: # rate limited — back off, don't cascade
_note_rate_limited(r.headers.get("Retry-After"))
return {"configured": True, "error": "429 rate limited (backing off)"}
r.raise_for_status()
except Exception as e: # noqa: BLE001 - surface any slskd/network failure to the UI
return {"configured": True, "error": str(e)}

View file

@ -49,3 +49,35 @@ def test_group_skips_non_video_and_samples():
def test_group_handles_garbage():
assert group_video_files(None) == []
assert group_video_files([{"nope": 1}, "junk"]) == []
# ── search-creation rate-limit throttle (avoids slskd 429s) ──────────────────
import core.video.slskd_search as _ss # noqa: E402
def _reset_throttle():
_ss._SEARCH_TIMES.clear()
_ss._COOLDOWN_UNTIL[0] = 0.0
def test_throttle_spaces_consecutive_creations():
_reset_throttle()
t1 = _ss._reserve_search_slot()
t2 = _ss._reserve_search_slot()
assert t2 - t1 >= _ss._MIN_GAP_SECONDS - 0.01 # min gap between creations
def test_throttle_window_cap_holds_the_overflow():
_reset_throttle()
times = [_ss._reserve_search_slot() for _ in range(_ss._MAX_PER_WINDOW + 1)]
# the one past the window cap waits ~a full window past the first
assert times[-1] >= times[0] + _ss._WINDOW_SECONDS - 0.5
def test_429_sets_a_cooldown():
import time
_reset_throttle()
_ss._note_rate_limited("10") # Retry-After: 10s
nxt = _ss._reserve_search_slot()
assert nxt >= time.monotonic() + 8 # next search waits out the cooldown
_reset_throttle()