82 lines
3.2 KiB
Python
82 lines
3.2 KiB
Python
"""Stall detection for the database-update job.
|
|
|
|
The DB updater keeps a single in-memory state dict whose ``status`` is set to
|
|
``running`` at start and only flipped to ``finished``/``error`` by the worker's
|
|
completion/error callbacks. If the worker thread hangs — e.g. a media-server API
|
|
call with no timeout, a DB lock — those callbacks never fire, so ``status`` stays
|
|
``running`` forever and the UI shows a frozen progress bar with no way to recover
|
|
(GitHub #859).
|
|
|
|
This module is the single, *pure* decision for "is a running job stalled?". It
|
|
takes the state dict plus the current wall-clock time and a timeout, and answers
|
|
yes/no — no DB, no globals, no clock of its own. That keeps it unit-testable and
|
|
lets the watchdog wiring in web_server.py stay a thin call. The job carries a
|
|
``last_progress_at`` epoch timestamp that the start path and every progress/phase
|
|
callback bump; staleness is simply "running, and that timestamp is older than the
|
|
timeout".
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
from typing import Any, Mapping
|
|
|
|
# 5 minutes with zero forward progress = presumed hung. A healthy scan ticks
|
|
# progress (per-artist) far more often than this even for large libraries, so
|
|
# the timeout won't false-positive a slow-but-working run.
|
|
DEFAULT_STALL_TIMEOUT_SECONDS = 300
|
|
|
|
|
|
def is_db_update_stalled(
|
|
state: Mapping[str, Any],
|
|
now: float,
|
|
timeout_seconds: float = DEFAULT_STALL_TIMEOUT_SECONDS,
|
|
) -> bool:
|
|
"""Return True when the job is ``running`` but has made no progress within
|
|
``timeout_seconds``.
|
|
|
|
Conservative by design — it only ever reports a stall it can prove:
|
|
- Only a ``running`` job can stall (idle/finished/error never do).
|
|
- With no usable ``last_progress_at`` timestamp we cannot judge, so we return
|
|
False rather than risk killing a job we have no clock for.
|
|
- A non-positive timeout is treated as "disabled" (never stalls).
|
|
"""
|
|
if not isinstance(state, Mapping):
|
|
return False
|
|
if state.get("status") != "running":
|
|
return False
|
|
if timeout_seconds is None or timeout_seconds <= 0:
|
|
return False
|
|
last = state.get("last_progress_at")
|
|
if not last:
|
|
return False
|
|
try:
|
|
elapsed = float(now) - float(last)
|
|
except (TypeError, ValueError):
|
|
return False
|
|
return elapsed >= float(timeout_seconds)
|
|
|
|
|
|
def stalled_error_message(state: Mapping[str, Any], now: float) -> str:
|
|
"""Build a clear, human-facing message for a stalled job, including how long
|
|
it has been silent and the phase it died in."""
|
|
last = state.get("last_progress_at") if isinstance(state, Mapping) else None
|
|
phase = state.get("phase") if isinstance(state, Mapping) else None
|
|
try:
|
|
secs = int(float(now) - float(last)) if last else 0
|
|
except (TypeError, ValueError):
|
|
secs = 0
|
|
msg = "Update appears stuck — no progress"
|
|
if secs > 0:
|
|
msg += f" for {secs}s"
|
|
if phase:
|
|
msg += f" (last phase: {phase})"
|
|
msg += (". The worker may be hung on the media server. Start a new update "
|
|
"to try again, or restart SoulSync if it keeps stalling.")
|
|
return msg
|
|
|
|
|
|
__all__ = [
|
|
"DEFAULT_STALL_TIMEOUT_SECONDS",
|
|
"is_db_update_stalled",
|
|
"stalled_error_message",
|
|
]
|