92 lines
3.4 KiB
Python
92 lines
3.4 KiB
Python
"""Seam tests for the database-update stall watchdog (GitHub #859).
|
|
|
|
A DB-update job can hang (media-server call with no timeout, DB lock) and sit at
|
|
status='running' forever because the worker's finished/error callbacks never
|
|
fire. `is_db_update_stalled` is the pure decision that lets the watchdog flip
|
|
such a job to 'error' so the UI recovers. These tests pin that decision —
|
|
including the conservative cases where it must NOT false-positive.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
from core.database_update_health import (
|
|
DEFAULT_STALL_TIMEOUT_SECONDS,
|
|
is_db_update_stalled,
|
|
stalled_error_message,
|
|
)
|
|
|
|
|
|
def _state(**over):
|
|
base = {"status": "running", "phase": "Incremental: scanning",
|
|
"processed": 2, "total": 3, "progress": 66.7, "last_progress_at": 1000.0}
|
|
base.update(over)
|
|
return base
|
|
|
|
|
|
def test_running_and_heartbeat_stale_is_stalled():
|
|
# last tick at t=1000, now=1000+timeout → exactly at the boundary counts.
|
|
now = 1000.0 + DEFAULT_STALL_TIMEOUT_SECONDS
|
|
assert is_db_update_stalled(_state(), now) is True
|
|
|
|
|
|
def test_running_and_heartbeat_fresh_is_not_stalled():
|
|
now = 1000.0 + 5 # ticked 5s ago, well within timeout
|
|
assert is_db_update_stalled(_state(), now) is False
|
|
|
|
|
|
def test_just_under_timeout_is_not_stalled():
|
|
now = 1000.0 + DEFAULT_STALL_TIMEOUT_SECONDS - 0.001
|
|
assert is_db_update_stalled(_state(), now) is False
|
|
|
|
|
|
def test_non_running_statuses_never_stall():
|
|
now = 1000.0 + 10_000 # very stale heartbeat
|
|
for status in ("idle", "finished", "error"):
|
|
assert is_db_update_stalled(_state(status=status), now) is False, status
|
|
|
|
|
|
def test_missing_heartbeat_cannot_judge():
|
|
# No usable timestamp → we refuse to kill a job we have no clock for.
|
|
now = 1_000_000.0
|
|
assert is_db_update_stalled(_state(last_progress_at=0), now) is False
|
|
assert is_db_update_stalled(_state(last_progress_at=None), now) is False
|
|
s = _state()
|
|
del s["last_progress_at"]
|
|
assert is_db_update_stalled(s, now) is False
|
|
|
|
|
|
def test_non_positive_timeout_disables_watchdog():
|
|
now = 1000.0 + 10_000
|
|
assert is_db_update_stalled(_state(), now, timeout_seconds=0) is False
|
|
assert is_db_update_stalled(_state(), now, timeout_seconds=-1) is False
|
|
|
|
|
|
def test_bad_inputs_are_safe():
|
|
assert is_db_update_stalled(None, 123.0) is False
|
|
assert is_db_update_stalled("not a dict", 123.0) is False
|
|
assert is_db_update_stalled(_state(last_progress_at="oops"), 1_000_000.0) is False
|
|
|
|
|
|
def test_custom_timeout_respected():
|
|
now = 1000.0 + 120
|
|
assert is_db_update_stalled(_state(), now, timeout_seconds=60) is True
|
|
assert is_db_update_stalled(_state(), now, timeout_seconds=180) is False
|
|
|
|
|
|
def test_stalled_message_is_informative():
|
|
now = 1000.0 + 360
|
|
msg = stalled_error_message(_state(phase="Incremental: scanning"), now)
|
|
assert "stuck" in msg.lower()
|
|
assert "360s" in msg
|
|
assert "Incremental: scanning" in msg
|
|
|
|
|
|
def test_issue_859_frozen_running_job_is_caught():
|
|
"""Regression: the reported state — running, frozen at 2/3 (66.7%), heartbeat
|
|
long stale — is detected as stalled so the card can self-heal."""
|
|
state = _state(status="running", processed=2, total=3, progress=66.7,
|
|
last_progress_at=1000.0)
|
|
now = 1000.0 + DEFAULT_STALL_TIMEOUT_SECONDS + 30
|
|
assert is_db_update_stalled(state, now) is True
|
|
# And a healthy job that's actively ticking is left alone.
|
|
assert is_db_update_stalled(_state(last_progress_at=now - 2), now) is False
|