Tighten up teardown logic
- Stop active DB update work before tearing down executor pools. - Short-circuit scan completion callbacks during shutdown so in-flight timer ticks don’t queue follow-up work. - Prevent the download monitor from draining deferred/completed tasks after shutdown starts. - Make listening stats startup stop-aware so it exits cleanly if teardown begins during warmup. - If a metadata update is already running, it can now observe should_stop and exit cleanly instead of continuing after SIGTERM.
This commit is contained in:
parent
aec3047216
commit
29d964e8b0
5 changed files with 42 additions and 11 deletions
|
|
@ -90,15 +90,20 @@ class ListeningStatsWorker:
|
||||||
logger.info("Listening stats worker thread started")
|
logger.info("Listening stats worker thread started")
|
||||||
|
|
||||||
# Build cache from existing data immediately (before first poll)
|
# Build cache from existing data immediately (before first poll)
|
||||||
interruptible_sleep(self._stop_event, 5)
|
if interruptible_sleep(self._stop_event, 5):
|
||||||
|
return
|
||||||
try:
|
try:
|
||||||
self._build_stats_cache()
|
self._build_stats_cache()
|
||||||
logger.info("Initial stats cache built from existing data")
|
logger.info("Initial stats cache built from existing data")
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.debug(f"Initial cache build skipped: {e}")
|
logger.debug(f"Initial cache build skipped: {e}")
|
||||||
|
|
||||||
|
if self.should_stop:
|
||||||
|
return
|
||||||
|
|
||||||
# Wait before first poll
|
# Wait before first poll
|
||||||
interruptible_sleep(self._stop_event, 10)
|
if interruptible_sleep(self._stop_event, 10):
|
||||||
|
return
|
||||||
|
|
||||||
while not self.should_stop:
|
while not self.should_stop:
|
||||||
try:
|
try:
|
||||||
|
|
|
||||||
|
|
@ -256,6 +256,8 @@ class MediaScanManager:
|
||||||
if is_scanning:
|
if is_scanning:
|
||||||
# Still scanning - trigger database update and continue periodic updates
|
# Still scanning - trigger database update and continue periodic updates
|
||||||
logger.info(f"{server_type.upper()} still scanning - triggering database update")
|
logger.info(f"{server_type.upper()} still scanning - triggering database update")
|
||||||
|
if self._shutting_down:
|
||||||
|
return
|
||||||
self._call_completion_callbacks()
|
self._call_completion_callbacks()
|
||||||
|
|
||||||
# Schedule next periodic update
|
# Schedule next periodic update
|
||||||
|
|
@ -268,6 +270,8 @@ class MediaScanManager:
|
||||||
else:
|
else:
|
||||||
# Scanning stopped - final update and cleanup
|
# Scanning stopped - final update and cleanup
|
||||||
logger.info(f"{server_type.upper()} scanning completed - doing final database update")
|
logger.info(f"{server_type.upper()} scanning completed - doing final database update")
|
||||||
|
if self._shutting_down:
|
||||||
|
return
|
||||||
self._call_completion_callbacks()
|
self._call_completion_callbacks()
|
||||||
self._stop_periodic_updates()
|
self._stop_periodic_updates()
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -174,6 +174,8 @@ class PlexScanManager:
|
||||||
if is_scanning:
|
if is_scanning:
|
||||||
# Still scanning - trigger database update and continue periodic updates
|
# Still scanning - trigger database update and continue periodic updates
|
||||||
logger.info("Plex still scanning - triggering database update")
|
logger.info("Plex still scanning - triggering database update")
|
||||||
|
if self._shutting_down:
|
||||||
|
return
|
||||||
self._call_completion_callbacks()
|
self._call_completion_callbacks()
|
||||||
|
|
||||||
# Schedule next periodic update
|
# Schedule next periodic update
|
||||||
|
|
@ -186,6 +188,8 @@ class PlexScanManager:
|
||||||
else:
|
else:
|
||||||
# Scanning stopped - final update and cleanup
|
# Scanning stopped - final update and cleanup
|
||||||
logger.info("Plex scanning completed - doing final database update")
|
logger.info("Plex scanning completed - doing final database update")
|
||||||
|
if self._shutting_down:
|
||||||
|
return
|
||||||
self._call_completion_callbacks()
|
self._call_completion_callbacks()
|
||||||
self._stop_periodic_updates()
|
self._stop_periodic_updates()
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -290,12 +290,19 @@ class WebScanManager:
|
||||||
|
|
||||||
def _handle_scan_completion(self):
|
def _handle_scan_completion(self):
|
||||||
"""Handle scan completion and trigger callbacks"""
|
"""Handle scan completion and trigger callbacks"""
|
||||||
logger.info(f"Web {self._current_server_type.upper()} library scan completed")
|
|
||||||
|
|
||||||
# Call completion callbacks
|
|
||||||
callbacks_to_call = []
|
|
||||||
with self._lock:
|
with self._lock:
|
||||||
|
if self._shutting_down:
|
||||||
|
return
|
||||||
|
server_type = self._current_server_type
|
||||||
callbacks_to_call = self._scan_completion_callbacks.copy()
|
callbacks_to_call = self._scan_completion_callbacks.copy()
|
||||||
|
downloads_during_scan = self._downloads_during_scan
|
||||||
|
|
||||||
|
if not server_type:
|
||||||
|
logger.debug("Skipping web scan completion: no active server type")
|
||||||
|
self._reset_scan_state()
|
||||||
|
return
|
||||||
|
|
||||||
|
logger.info(f"Web {server_type.upper()} library scan completed")
|
||||||
|
|
||||||
for callback in callbacks_to_call:
|
for callback in callbacks_to_call:
|
||||||
try:
|
try:
|
||||||
|
|
@ -308,10 +315,9 @@ class WebScanManager:
|
||||||
self._reset_scan_state()
|
self._reset_scan_state()
|
||||||
|
|
||||||
# Check if we need another scan due to downloads during this scan
|
# Check if we need another scan due to downloads during this scan
|
||||||
with self._lock:
|
if downloads_during_scan:
|
||||||
if self._downloads_during_scan:
|
logger.info("Web scan follow-up needed for downloads during scan")
|
||||||
logger.info("Web scan follow-up needed for downloads during scan")
|
self.request_scan("Follow-up scan for downloads during previous scan")
|
||||||
self.request_scan("Follow-up scan for downloads during previous scan")
|
|
||||||
|
|
||||||
def _reset_scan_state(self):
|
def _reset_scan_state(self):
|
||||||
"""Reset internal scan state"""
|
"""Reset internal scan state"""
|
||||||
|
|
|
||||||
|
|
@ -2638,6 +2638,8 @@ class WebUIDownloadMonitor:
|
||||||
completed_tasks.append((batch_id, task_id))
|
completed_tasks.append((batch_id, task_id))
|
||||||
|
|
||||||
# ---- All work below runs WITHOUT tasks_lock held ----
|
# ---- All work below runs WITHOUT tasks_lock held ----
|
||||||
|
if globals().get('IS_SHUTTING_DOWN', False) or not self.monitoring:
|
||||||
|
return
|
||||||
|
|
||||||
# Execute deferred operations from _should_retry_task (network calls, nested locks)
|
# Execute deferred operations from _should_retry_task (network calls, nested locks)
|
||||||
for op in deferred_ops:
|
for op in deferred_ops:
|
||||||
|
|
@ -3460,6 +3462,11 @@ def _shutdown_runtime_components():
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
print(f"Error saving API call history: {e}")
|
print(f"Error saving API call history: {e}")
|
||||||
|
|
||||||
|
# Stop the active DB update worker before tearing down the executor it runs on.
|
||||||
|
# This lets an in-flight update observe should_stop and exit cleanly.
|
||||||
|
_stop_component(db_update_worker, "db update worker")
|
||||||
|
_stop_component(metadata_update_runtime_worker, "metadata update worker")
|
||||||
|
|
||||||
# Stop long-lived worker components in parallel so shutdown waits for the
|
# Stop long-lived worker components in parallel so shutdown waits for the
|
||||||
# slowest worker instead of serially burning the timeout for each one.
|
# slowest worker instead of serially burning the timeout for each one.
|
||||||
_stop_components_parallel([
|
_stop_components_parallel([
|
||||||
|
|
@ -40789,6 +40796,7 @@ metadata_update_state = {
|
||||||
}
|
}
|
||||||
|
|
||||||
metadata_update_worker = None
|
metadata_update_worker = None
|
||||||
|
metadata_update_runtime_worker = None
|
||||||
metadata_update_executor = ThreadPoolExecutor(max_workers=1, thread_name_prefix="metadata_update")
|
metadata_update_executor = ThreadPoolExecutor(max_workers=1, thread_name_prefix="metadata_update")
|
||||||
|
|
||||||
# ===============================
|
# ===============================
|
||||||
|
|
@ -44903,7 +44911,7 @@ def _old_get_listenbrainz_playlist_tracks_DEPRECATED(playlist_mbid):
|
||||||
@app.route('/api/metadata/start', methods=['POST'])
|
@app.route('/api/metadata/start', methods=['POST'])
|
||||||
def start_metadata_update():
|
def start_metadata_update():
|
||||||
"""Start the metadata update process - EXACT copy of dashboard.py logic"""
|
"""Start the metadata update process - EXACT copy of dashboard.py logic"""
|
||||||
global metadata_update_worker, metadata_update_state
|
global metadata_update_worker, metadata_update_runtime_worker, metadata_update_state
|
||||||
|
|
||||||
try:
|
try:
|
||||||
# Check if already running
|
# Check if already running
|
||||||
|
|
@ -44971,6 +44979,7 @@ def start_metadata_update():
|
||||||
|
|
||||||
# Start the metadata update worker - EXACTLY like dashboard.py
|
# Start the metadata update worker - EXACTLY like dashboard.py
|
||||||
def run_metadata_update():
|
def run_metadata_update():
|
||||||
|
global metadata_update_runtime_worker
|
||||||
try:
|
try:
|
||||||
metadata_worker = WebMetadataUpdateWorker(
|
metadata_worker = WebMetadataUpdateWorker(
|
||||||
None, # Artists will be loaded in the worker thread - EXACTLY like dashboard.py
|
None, # Artists will be loaded in the worker thread - EXACTLY like dashboard.py
|
||||||
|
|
@ -44979,12 +44988,15 @@ def start_metadata_update():
|
||||||
active_server,
|
active_server,
|
||||||
refresh_interval_days
|
refresh_interval_days
|
||||||
)
|
)
|
||||||
|
metadata_update_runtime_worker = metadata_worker
|
||||||
metadata_worker.run()
|
metadata_worker.run()
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
print(f"Error in metadata update worker: {e}")
|
print(f"Error in metadata update worker: {e}")
|
||||||
metadata_update_state['status'] = 'error'
|
metadata_update_state['status'] = 'error'
|
||||||
metadata_update_state['error'] = str(e)
|
metadata_update_state['error'] = str(e)
|
||||||
add_activity_item("", "Metadata Error", str(e), "Now")
|
add_activity_item("", "Metadata Error", str(e), "Now")
|
||||||
|
finally:
|
||||||
|
metadata_update_runtime_worker = None
|
||||||
|
|
||||||
metadata_update_worker = metadata_update_executor.submit(run_metadata_update)
|
metadata_update_worker = metadata_update_executor.submit(run_metadata_update)
|
||||||
|
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue