Auto-reconnect Hydrabase WebSocket when server restarts
Background monitor checks connection every 30s. If auto_connect is enabled and socket is dead, reconnects using saved credentials. Exponential backoff (30s→60s→120s→max 300s) on repeated failures. Log suppression after 3 consecutive failures to prevent spam. Fixes: Hydrabase server restart required manual reconnect from UI. Now recovers automatically within 30s of server becoming available.
This commit is contained in:
parent
835ddcdbd5
commit
db104ec155
1 changed files with 55 additions and 0 deletions
|
|
@ -44672,6 +44672,59 @@ def _build_watchlist_count_payload(profile_id=1):
|
|||
'next_run_in_seconds': next_run_in_seconds
|
||||
}
|
||||
|
||||
def _hydrabase_reconnect_loop():
|
||||
"""Background thread that monitors Hydrabase connection and auto-reconnects if needed."""
|
||||
global _hydrabase_ws
|
||||
_consecutive_failures = 0
|
||||
|
||||
while True:
|
||||
socketio.sleep(30)
|
||||
try:
|
||||
# Only attempt reconnect if auto_connect is enabled
|
||||
hydra_cfg = config_manager.get_hydrabase_config()
|
||||
if not hydra_cfg.get('auto_connect') or not hydra_cfg.get('url') or not hydra_cfg.get('api_key'):
|
||||
_consecutive_failures = 0
|
||||
continue
|
||||
|
||||
# Check if already connected
|
||||
try:
|
||||
if _hydrabase_ws is not None and _hydrabase_ws.connected:
|
||||
_consecutive_failures = 0
|
||||
continue
|
||||
except Exception:
|
||||
pass # Socket in bad state — treat as disconnected
|
||||
|
||||
# Disconnected with auto_connect enabled — try to reconnect
|
||||
# Back off: 30s, 60s, 120s, max 300s between attempts
|
||||
backoff = min(30 * (2 ** _consecutive_failures), 300)
|
||||
if _consecutive_failures > 0:
|
||||
socketio.sleep(backoff - 30) # Already slept 30s at top of loop
|
||||
|
||||
import websocket
|
||||
try:
|
||||
with _hydrabase_lock:
|
||||
if _hydrabase_ws:
|
||||
try:
|
||||
_hydrabase_ws.close()
|
||||
except:
|
||||
pass
|
||||
ws = websocket.create_connection(
|
||||
hydra_cfg['url'],
|
||||
header={"x-api-key": hydra_cfg['api_key']},
|
||||
timeout=10
|
||||
)
|
||||
_hydrabase_ws = ws
|
||||
_consecutive_failures = 0
|
||||
print(f"🔄 [Hydrabase] Auto-reconnected to {hydra_cfg['url']}")
|
||||
except Exception as e:
|
||||
_consecutive_failures += 1
|
||||
if _consecutive_failures <= 3:
|
||||
print(f"⚠️ [Hydrabase] Reconnect attempt failed ({_consecutive_failures}): {e}")
|
||||
elif _consecutive_failures == 4:
|
||||
print(f"⚠️ [Hydrabase] Reconnect failing repeatedly — suppressing further logs until success")
|
||||
except Exception:
|
||||
pass # Don't crash the monitor loop
|
||||
|
||||
def _emit_service_status_loop():
|
||||
"""Background thread that pushes service status every 5 seconds."""
|
||||
while True:
|
||||
|
|
@ -45183,6 +45236,8 @@ if __name__ == '__main__':
|
|||
socketio.start_background_task(_emit_automation_progress_loop)
|
||||
# Phase 7: Repair job progress
|
||||
socketio.start_background_task(_emit_repair_progress_loop)
|
||||
# Hydrabase auto-reconnect monitor
|
||||
socketio.start_background_task(_hydrabase_reconnect_loop)
|
||||
print("✅ WebSocket emitters started (Phase 1-7: global/dashboard/enrichment/tools/sync/automations/repair)")
|
||||
|
||||
socketio.run(app, host='0.0.0.0', port=8008, debug=False, allow_unsafe_werkzeug=True)
|
||||
|
|
|
|||
Loading…
Reference in a new issue