soulsync/core/automation/handlers/database_update.py

"""Automation handlers: ``start_database_update`` and
``deep_scan_library`` actions.

Lifted from ``web_server._register_automation_handlers`` (the
``_auto_start_database_update`` and ``_auto_deep_scan_library``
closures). Both share the same ``db_update_state`` / executor / lock
infrastructure -- the only difference is which task they submit
(``run_db_update_task`` vs ``run_deep_scan_task``).

Pattern: pre-set state to running, submit task to executor, then
poll the state dict until it transitions away from ``running``.
Stall-detection emits a warning every 10 minutes when progress
hasn't budged. 2-hour outer timeout caps the worst case.
"""

from __future__ import annotations

import time
from typing import Any, Dict

from core.automation.deps import AutomationDeps


# Time out on STALL (no progress), not total runtime: a large library can scan
# for many hours while progressing fine — a hard total cap would falsely mark a
# healthy scan 'error' (the scan thread keeps running uncancelled). We only give
# up when progress hasn't moved for a long stretch, with a generous absolute
# backstop against a truly stuck monitor loop.
_STALL_WARNING_SECONDS = 600     # warn after 10 min with no progress (repeats)
_STALL_TIMEOUT_SECONDS = 1800    # 30 min with no progress at all = genuinely stalled
_ABSOLUTE_CAP_SECONDS = 86400    # 24h hard backstop (runaway-loop guard only)
_POLL_INTERVAL_SECONDS = 3
_INITIAL_DELAY_SECONDS = 1


def scan_wait_action(
    *,
    status: str,
    idle_seconds: float,
    total_seconds: float,
    stall_timeout_s: float = _STALL_TIMEOUT_SECONDS,
    stall_warn_s: float = _STALL_WARNING_SECONDS,
    abs_cap_s: float = _ABSOLUTE_CAP_SECONDS,
) -> str:
    """Decide what the monitor loop should do on a poll tick (pure/testable).

    ``idle_seconds`` is time since progress last changed; ``total_seconds`` is
    time since the wait began. Returns one of:
    ``'finished'`` (task no longer running), ``'stall_timeout'`` (no progress for
    too long → give up), ``'abs_timeout'`` (absolute backstop), ``'warn'``
    (stalled long enough to warn but not give up), or ``'continue'``.

    Crucially, an actively-progressing scan keeps resetting ``idle_seconds``, so
    it never hits ``stall_timeout`` no matter how long the whole scan takes.
    """
    if status != 'running':
        return 'finished'
    if total_seconds >= abs_cap_s:
        return 'abs_timeout'
    if idle_seconds >= stall_timeout_s:
        return 'stall_timeout'
    if idle_seconds >= stall_warn_s:
        return 'warn'
    return 'continue'


def auto_start_database_update(config: Dict[str, Any], deps: AutomationDeps) -> Dict[str, Any]:
    """Run a full or incremental DB update via ``run_db_update_task``."""
    return _run_with_progress(
        config, deps,
        task=deps.run_db_update_task,
        task_args=(config.get('full_refresh', False), deps.config_manager.get_active_media_server()),
        initial_phase='Initializing...',
        stall_label='Database update',
        finished_extras=lambda: {'full_refresh': str(config.get('full_refresh', False))},
        timeout_label='Database update timed out after 24 hours',
    )


def auto_deep_scan_library(config: Dict[str, Any], deps: AutomationDeps) -> Dict[str, Any]:
    """Run a deep library scan via ``run_deep_scan_task``."""
    return _run_with_progress(
        config, deps,
        task=deps.run_deep_scan_task,
        task_args=(deps.config_manager.get_active_media_server(),),
        initial_phase='Deep scan: Initializing...',
        stall_label='Deep scan',
        finished_extras=lambda: {},
        timeout_label='Deep scan timed out after 24 hours',
    )


def _run_with_progress(
    config: Dict[str, Any],
    deps: AutomationDeps,
    *,
    task,
    task_args: tuple,
    initial_phase: str,
    stall_label: str,
    finished_extras,
    timeout_label: str,
) -> Dict[str, Any]:
    """Shared poll-and-wait body for both DB-update handlers."""
    automation_id = config.get('_automation_id')
    state = deps.get_db_update_state()
    if state.get('status') == 'running':
        return {'status': 'skipped', 'reason': 'Database update already running'}
    deps.state.db_update_automation_id = automation_id
    # Sync legacy module global so the DB-update progress callbacks
    # (still living in web_server.py) emit against this automation.
    deps.set_db_update_automation_id(automation_id)

    with deps.db_update_lock:
        state.update({
            'status': 'running', 'phase': initial_phase,
            'progress': 0, 'current_item': '', 'processed': 0, 'total': 0,
            'error_message': '',
        })
    deps.db_update_executor.submit(task, *task_args)

    # Monitor progress (callbacks handle card updates, we just block until done).
    # We time out on STALL, not total runtime: ``processed`` advances on every
    # artist, so an actively-progressing scan keeps resetting the idle clock and
    # is never falsely failed no matter how long the whole library takes.
    time.sleep(_INITIAL_DELAY_SECONDS)
    poll_start = time.time()
    last_progress_time = time.time()
    # Any of these advancing means the scan is alive. current_item (the artist
    # being processed) changes every artist even when the rounded progress %
    # holds steady, so it guards against a false stall during slow stretches.
    last_progress_val = (0, 0, '')
    last_warn_time = 0.0
    outcome = 'finished'
    while True:
        time.sleep(_POLL_INTERVAL_SECONDS)
        now = time.time()
        with deps.db_update_lock:
            current_status = state.get('status', 'idle')
            current_val = (state.get('processed', 0), state.get('progress', 0),
                           state.get('current_item', ''))
        if current_val != last_progress_val:
            last_progress_val = current_val
            last_progress_time = now

        action = scan_wait_action(
            status=current_status,
            idle_seconds=now - last_progress_time,
            total_seconds=now - poll_start,
        )
        if action in ('finished', 'stall_timeout', 'abs_timeout'):
            outcome = action
            break
        if action == 'warn' and (now - last_warn_time) > _STALL_WARNING_SECONDS:
            idle_min = int((now - last_progress_time) / 60)
            deps.update_progress(
                automation_id,
                log_line=f'{stall_label} — no progress for {idle_min} min, still waiting...',
                log_type='warning',
            )
            last_warn_time = now

    if outcome == 'stall_timeout':
        deps.update_progress(
            automation_id, status='error', phase='Stalled',
            log_line=f'{stall_label} made no progress for {_STALL_TIMEOUT_SECONDS // 60} minutes — giving up',
            log_type='error',
        )
        return {'status': 'error', 'reason': 'Stalled (no progress)', '_manages_own_progress': True}
    if outcome == 'abs_timeout':
        deps.update_progress(
            automation_id, status='error',
            phase='Timed out', log_line=timeout_label, log_type='error',
        )
        return {'status': 'error', 'reason': 'Timed out', '_manages_own_progress': True}

    # Finished/error callback already updated the card — return matching status.
    with deps.db_update_lock:
        final_status = state.get('status', 'unknown')
    if final_status == 'error':
        return {
            'status': 'error',
            'reason': state.get('error_message', 'Unknown error'),
            '_manages_own_progress': True,
        }
    with deps.db_update_lock:
        stats = {
            'status': 'completed', '_manages_own_progress': True,
            'artists': state.get('total', 0),
            'albums': state.get('total_albums', 0),
            'tracks': state.get('total_tracks', 0),
            'removed_artists': state.get('removed_artists', 0),
            'removed_albums': state.get('removed_albums', 0),
            'removed_tracks': state.get('removed_tracks', 0),
        }
    stats.update(finished_extras())
    return stats