From 697a2550471b9d3da730f1fdd82676613161c5f6 Mon Sep 17 00:00:00 2001 From: Broque Thomas Date: Sat, 30 Aug 2025 21:18:10 -0700 Subject: [PATCH] move wishlist processing to backend --- web_server.py | 201 +++++++++++++++++++++++++++++++++++++- webui/static/script.js | 216 ++++++++++++++--------------------------- 2 files changed, 274 insertions(+), 143 deletions(-) diff --git a/web_server.py b/web_server.py index 2b657a81..d5a63f18 100644 --- a/web_server.py +++ b/web_server.py @@ -120,6 +120,13 @@ download_tasks = {} # task_id -> task state dict download_batches = {} # batch_id -> {queue, active_count, max_concurrent} tasks_lock = threading.Lock() +# --- Automatic Wishlist Processing Infrastructure --- +# Server-side timer system for automatic wishlist processing (replaces client-side JavaScript timers) +wishlist_auto_processor = ThreadPoolExecutor(max_workers=1, thread_name_prefix="WishlistAutoProcessor") +wishlist_auto_timer = None # threading.Timer for scheduling next auto-processing +wishlist_auto_processing = False # Flag to prevent concurrent auto-processing +wishlist_timer_lock = threading.Lock() # Thread safety for timer operations + # --- Shared Transfer Data Cache --- # Cache transfer data to avoid hammering the Soulseek API with multiple concurrent modals transfer_data_cache = { @@ -3384,6 +3391,139 @@ def start_simple_background_monitor(): monitor_thread.daemon = True monitor_thread.start() +# =============================== +# == AUTOMATIC WISHLIST PROCESSING == +# =============================== + +def start_wishlist_auto_processing(): + """Start automatic wishlist processing with 1-minute initial delay.""" + global wishlist_auto_timer + + with wishlist_timer_lock: + # Stop any existing timer to prevent duplicates + if wishlist_auto_timer is not None: + wishlist_auto_timer.cancel() + + print("๐Ÿ”„ Starting automatic wishlist processing system (1 minute initial delay)") + wishlist_auto_timer = threading.Timer(60.0, _process_wishlist_automatically) # 1 minute + wishlist_auto_timer.daemon = True + wishlist_auto_timer.start() + +def stop_wishlist_auto_processing(): + """Stop automatic wishlist processing and cleanup timer.""" + global wishlist_auto_timer, wishlist_auto_processing + + with wishlist_timer_lock: + if wishlist_auto_timer is not None: + wishlist_auto_timer.cancel() + wishlist_auto_timer = None + print("โน๏ธ Stopped automatic wishlist processing") + + wishlist_auto_processing = False + +def schedule_next_wishlist_processing(): + """Schedule next automatic wishlist processing in 10 minutes.""" + global wishlist_auto_timer + + with wishlist_timer_lock: + print("โฐ Scheduling next automatic wishlist processing in 10 minutes") + wishlist_auto_timer = threading.Timer(600.0, _process_wishlist_automatically) # 10 minutes + wishlist_auto_timer.daemon = True + wishlist_auto_timer.start() + +def _process_wishlist_automatically(): + """Main automatic processing logic that runs in background thread.""" + global wishlist_auto_processing + + print("๐Ÿค– Starting automatic wishlist processing...") + + try: + with wishlist_timer_lock: + if wishlist_auto_processing: + print("โš ๏ธ Wishlist auto-processing already running, skipping.") + schedule_next_wishlist_processing() + return + + wishlist_auto_processing = True + + # Use app context for database operations + with app.app_context(): + from core.wishlist_service import get_wishlist_service + wishlist_service = get_wishlist_service() + + # Check if wishlist has tracks + count = wishlist_service.get_wishlist_count() + if count == 0: + print("โ„น๏ธ No tracks in wishlist for auto-processing.") + with wishlist_timer_lock: + wishlist_auto_processing = False + schedule_next_wishlist_processing() + return + + print(f"๐ŸŽต Found {count} tracks in wishlist, starting automatic processing...") + + # Check if wishlist processing is already active + playlist_id = "wishlist" + with tasks_lock: + for batch_id, batch_data in download_batches.items(): + if (batch_data.get('playlist_id') == playlist_id and + batch_data.get('phase') not in ['complete', 'error', 'cancelled']): + print("โš ๏ธ Wishlist processing already active in another batch, skipping automatic start") + with wishlist_timer_lock: + wishlist_auto_processing = False + schedule_next_wishlist_processing() + return + + # Get wishlist tracks for processing + wishlist_tracks = wishlist_service.get_wishlist_tracks_for_download() + if not wishlist_tracks: + print("โš ๏ธ No tracks returned from wishlist service.") + with wishlist_timer_lock: + wishlist_auto_processing = False + schedule_next_wishlist_processing() + return + + # Create batch for automatic processing + batch_id = str(uuid.uuid4()) + playlist_name = "Wishlist (Auto)" + + # Create task queue - convert wishlist tracks to expected format + with tasks_lock: + download_batches[batch_id] = { + 'phase': 'analysis', + 'playlist_id': playlist_id, + 'playlist_name': playlist_name, + 'queue': [], + 'active_count': 0, + 'max_concurrent': 3, + 'queue_index': 0, + 'analysis_total': len(wishlist_tracks), + 'analysis_processed': 0, + 'analysis_results': [], + # Track state management (replicating sync.py) + 'permanently_failed_tracks': [], + 'cancelled_tracks': set(), + # Mark as auto-initiated + 'auto_initiated': True, + 'auto_processing_timestamp': time.time() + } + + print(f"๐Ÿš€ Starting automatic wishlist batch {batch_id} with {len(wishlist_tracks)} tracks") + + # Submit the wishlist processing job using existing infrastructure + missing_download_executor.submit(_run_full_missing_tracks_process, batch_id, playlist_id, wishlist_tracks) + + # Don't mark auto_processing as False here - let completion handler do it + + except Exception as e: + print(f"โŒ Error in automatic wishlist processing: {e}") + import traceback + traceback.print_exc() + + with wishlist_timer_lock: + wishlist_auto_processing = False + schedule_next_wishlist_processing() + # =============================== # == DATABASE UPDATER API == # =============================== @@ -3879,6 +4019,49 @@ def _process_failed_tracks_to_wishlist_exact(batch_id): return {'tracks_added': 0, 'errors': 1, 'total_failed': 0} +def _process_failed_tracks_to_wishlist_exact_with_auto_completion(batch_id): + """ + Process failed tracks to wishlist for auto-initiated batches and handle auto-processing completion. + This extends the standard processing with automatic scheduling of the next cycle. + """ + global wishlist_auto_processing + + try: + print(f"๐Ÿค– [Auto-Wishlist] Processing completion for auto-initiated batch {batch_id}") + + # Run standard wishlist processing + completion_summary = _process_failed_tracks_to_wishlist_exact(batch_id) + + # Log auto-processing completion + tracks_added = completion_summary.get('tracks_added', 0) + total_failed = completion_summary.get('total_failed', 0) + print(f"๐ŸŽ‰ [Auto-Wishlist] Background processing complete: {tracks_added} added to wishlist, {total_failed} failed") + + # Mark auto-processing as complete + with wishlist_timer_lock: + wishlist_auto_processing = False + + # Schedule next automatic processing cycle + print("โฐ [Auto-Wishlist] Scheduling next automatic cycle in 10 minutes") + schedule_next_wishlist_processing() + + return completion_summary + + except Exception as e: + print(f"โŒ [Auto-Wishlist] Error in auto-completion processing: {e}") + import traceback + traceback.print_exc() + + # Ensure auto-processing flag is reset even on error + with wishlist_timer_lock: + wishlist_auto_processing = False + + # Schedule next cycle even after error to maintain continuity + print("โฐ [Auto-Wishlist] Scheduling next cycle after error (10 minutes)") + schedule_next_wishlist_processing() + + return {'tracks_added': 0, 'errors': 1, 'total_failed': 0} + def _on_download_completed(batch_id, task_id, success=True): """Called when a download completes to start the next one in queue""" with tasks_lock: @@ -3930,6 +4113,9 @@ def _on_download_completed(batch_id, task_id, success=True): if all_processed and no_active: print(f"๐ŸŽ‰ [Batch Manager] Batch {batch_id} fully complete - processing failed tracks to wishlist") + # Check if this is an auto-initiated batch + is_auto_batch = batch.get('auto_initiated', False) + # Mark batch as complete and process wishlist outside of lock to prevent deadlocks batch['phase'] = 'complete' @@ -3937,7 +4123,12 @@ def _on_download_completed(batch_id, task_id, success=True): download_monitor.stop_monitoring(batch_id) # Process wishlist outside of the lock to prevent threading issues - missing_download_executor.submit(_process_failed_tracks_to_wishlist_exact, batch_id) + if is_auto_batch: + # For auto-initiated batches, handle completion and schedule next cycle + missing_download_executor.submit(_process_failed_tracks_to_wishlist_exact_with_auto_completion, batch_id) + else: + # For manual batches, use standard wishlist processing + missing_download_executor.submit(_process_failed_tracks_to_wishlist_exact, batch_id) return # Don't start next batch if we're done # Start next downloads in queue @@ -4495,7 +4686,8 @@ def get_batch_download_status(batch_id): batch = download_batches[batch_id] response_data = { "phase": batch.get('phase', 'unknown'), - "error": batch.get('error') + "error": batch.get('error'), + "auto_initiated": batch.get('auto_initiated', False) } if response_data["phase"] == 'analysis': @@ -5370,4 +5562,9 @@ if __name__ == '__main__': start_simple_background_monitor() print("โœ… Simple background monitor started") + # Start automatic wishlist processing when server starts + print("๐Ÿ”ง Starting automatic wishlist processing...") + start_wishlist_auto_processing() + print("โœ… Automatic wishlist processing started") + app.run(host='0.0.0.0', port=5001, debug=True) diff --git a/webui/static/script.js b/webui/static/script.js index 18e89c76..672b312e 100644 --- a/webui/static/script.js +++ b/webui/static/script.js @@ -29,9 +29,6 @@ let dbStatsInterval = null; let dbUpdateStatusInterval = null; let wishlistCountInterval = null; -// --- Auto Wishlist Processing (Simple Timer System) --- -let autoWishlistTimer = null; - // --- Add these globals for the Sync Page --- let spotifyPlaylists = []; let selectedPlaylists = new Set(); @@ -278,7 +275,6 @@ async function loadPageData(pageId) { stopDbStatsPolling(); stopDbUpdatePolling(); stopWishlistCountPolling(); - stopAutoWishlistProcessing(); switch (pageId) { case 'dashboard': stopDownloadPolling(); @@ -1784,26 +1780,42 @@ async function checkForActiveProcesses() { } } -async function rehydrateModal(processInfo) { +async function rehydrateModal(processInfo, userRequested = false) { const { playlist_id, playlist_name, batch_id } = processInfo; - console.log(`๐Ÿ’ง Rehydrating modal for "${playlist_name}" (batch: ${batch_id})`); + console.log(`๐Ÿ’ง Rehydrating modal for "${playlist_name}" (batch: ${batch_id}) - User requested: ${userRequested}`); // Handle wishlist processes specially if (playlist_id === "wishlist") { + console.log(`๐Ÿ” Current activeDownloadProcesses keys: [${Object.keys(activeDownloadProcesses).join(', ')}]`); + await openDownloadMissingWishlistModal(); const process = activeDownloadProcesses[playlist_id]; - if (!process) return; + if (!process) { + console.error('โŒ Failed to create wishlist process in activeDownloadProcesses'); + return; + } + console.log(`โœ… Setting wishlist process state - batchId: ${batch_id}, status: running`); process.status = 'running'; process.batchId = batch_id; updateRefreshButtonState(); - document.getElementById(`begin-analysis-btn-${playlist_id}`).style.display = 'none'; - document.getElementById(`cancel-all-btn-${playlist_id}`).style.display = 'inline-block'; + const beginBtn = document.getElementById(`begin-analysis-btn-${playlist_id}`); + const cancelBtn = document.getElementById(`cancel-all-btn-${playlist_id}`); + + if (beginBtn) beginBtn.style.display = 'none'; + if (cancelBtn) cancelBtn.style.display = 'inline-block'; startModalDownloadPolling(playlist_id); - process.modalElement.style.display = 'none'; + // Only hide modal if this is background rehydration, not user-requested + if (!userRequested) { + process.modalElement.style.display = 'none'; + console.log('๐Ÿ” Hiding wishlist modal for background rehydration'); + } else { + process.modalElement.style.display = 'flex'; + console.log('๐Ÿ” Showing wishlist modal as requested by user'); + } return; } @@ -2668,8 +2680,8 @@ function startModalDownloadPolling(playlistId) { // Enhanced check for background auto-processing for wishlist const isWishlist = (playlistId === 'wishlist'); const isModalHidden = (process.modalElement && process.modalElement.style.display === 'none'); - const hasAutoTimer = (autoWishlistTimer !== null); - const isBackgroundWishlist = isWishlist && (isModalHidden || hasAutoTimer); + const isAutoInitiated = data.auto_initiated || false; // Server indicates if batch was auto-started + const isBackgroundWishlist = isWishlist && (isModalHidden || isAutoInitiated); if (data.phase === 'cancelled') { process.status = 'cancelled'; @@ -2690,7 +2702,7 @@ function startModalDownloadPolling(playlistId) { // Reset modal to idle state to prevent "complete" phase disruption setTimeout(() => { resetWishlistModalToIdleState(); - scheduleNextAutoWishlist(); // Schedule next cycle + // Server-side auto-processing will handle next cycle automatically }, 500); return; // Skip normal completion handling @@ -5274,124 +5286,7 @@ function stopWishlistCountPolling() { } } -// --- Auto Wishlist Processing (Simple Implementation) --- -function startAutoWishlistProcessing() { - console.log("๐Ÿ”„ Starting automatic wishlist processing system (1 minute initial delay)"); - stopAutoWishlistProcessing(); // Prevent duplicates - autoWishlistTimer = setTimeout(processWishlistAutomatically, 60000); // 1 minute -} - -function stopAutoWishlistProcessing() { - if (autoWishlistTimer) { - clearTimeout(autoWishlistTimer); - autoWishlistTimer = null; - console.log("โน๏ธ Stopped automatic wishlist processing"); - } -} - -function scheduleNextAutoWishlist() { - console.log("โฐ Scheduling next automatic wishlist processing in 10 minutes"); - autoWishlistTimer = setTimeout(processWishlistAutomatically, 600000); // 10 minutes -} - -async function processWishlistAutomatically() { - console.log("๐Ÿค– Starting automatic wishlist processing..."); - - try { - // Check if wishlist has tracks - const countResponse = await fetch('/api/wishlist/count'); - if (!countResponse.ok) { - console.warn("โš ๏ธ Could not fetch wishlist count, will retry next cycle"); - scheduleNextAutoWishlist(); - return; - } - - const countData = await countResponse.json(); - if (countData.count === 0) { - console.log("โ„น๏ธ Wishlist is empty, skipping automatic processing"); - scheduleNextAutoWishlist(); - return; - } - - console.log(`๐ŸŽต Found ${countData.count} tracks in wishlist, starting automatic processing...`); - - const playlistId = 'wishlist'; - - // Check if processing is already active - if (activeDownloadProcesses[playlistId] && activeDownloadProcesses[playlistId].status === 'running') { - console.log("โš ๏ธ Wishlist processing already active, skipping automatic start"); - scheduleNextAutoWishlist(); - return; - } - - // Check if modal is currently being viewed by user - const existingModal = document.getElementById(`download-missing-modal-${playlistId}`); - const userIsViewingModal = existingModal && existingModal.style.display === 'flex'; - - console.log("๐Ÿค– Setting up wishlist modal for automatic processing"); - console.log(`๐Ÿ” User currently viewing modal: ${userIsViewingModal}`); - - // Create modal if it doesn't exist, but keep it hidden for background processing - if (!existingModal) { - await openDownloadMissingWishlistModal(); - // Immediately hide the modal since this is background processing - const modal = document.getElementById(`download-missing-modal-${playlistId}`); - if (modal) { - modal.style.display = 'none'; - console.log("๐Ÿค– Modal created and hidden for background processing"); - } - } - - // Wait a moment for modal to be ready, then programmatically click "Begin Analysis" - setTimeout(() => { - const beginButton = document.getElementById(`begin-analysis-btn-${playlistId}`); - const modal = document.getElementById(`download-missing-modal-${playlistId}`); - - console.log(`๐Ÿ” Looking for button with ID: begin-analysis-btn-${playlistId}`); - console.log(`๐Ÿ” Button found:`, beginButton); - - if (beginButton) { - // Check if button is visible and clickable - const buttonStyle = window.getComputedStyle(beginButton); - console.log(`๐Ÿ” Button display style:`, buttonStyle.display); - console.log(`๐Ÿ” Button disabled:`, beginButton.disabled); - - if (buttonStyle.display === 'none' || beginButton.disabled) { - console.warn("โš ๏ธ Begin Analysis button is hidden or disabled, skipping automatic click"); - scheduleNextAutoWishlist(); - return; - } - - console.log("๐Ÿค– Programmatically clicking 'Begin Analysis' button"); - beginButton.click(); - - // Only hide modal if user wasn't actively viewing it - if (!userIsViewingModal) { - setTimeout(() => { - const modal = document.getElementById(`download-missing-modal-${playlistId}`); - if (modal && modal.style.display !== 'none') { - modal.style.display = 'none'; - console.log("๐Ÿค– Modal hidden - processing continues in background"); - } - }, 500); - } else { - console.log("๐Ÿค– User is viewing modal - keeping it visible with live updates"); - } - - } else { - console.warn("โš ๏ธ Could not find Begin Analysis button, will retry next cycle"); - console.log("๐Ÿ” Available buttons with 'begin-analysis' in ID:", - Array.from(document.querySelectorAll('[id*="begin-analysis"]')).map(el => el.id)); - scheduleNextAutoWishlist(); - } - }, 1000); - - } catch (error) { - console.error("โŒ Error in automatic wishlist processing:", error); - scheduleNextAutoWishlist(); - } -} function resetWishlistModalToIdleState() { // Reset wishlist modal to idle state after background processing completes @@ -5491,8 +5386,7 @@ async function loadDashboardData() { // Check for any active download processes that need rehydration await checkForActiveProcesses(); - // Start automatic wishlist processing (1 minute delay, then every 10 minutes) - startAutoWishlistProcessing(); + // Automatic wishlist processing now runs server-side } // --- Data Fetching and UI Updates --- @@ -5736,16 +5630,59 @@ async function handleDbUpdateButtonClick() { async function handleWishlistButtonClick() { try { - // First check if there are any tracks in the wishlist - const response = await fetch('/api/wishlist/count'); - const data = await response.json(); + const playlistId = 'wishlist'; - if (data.count === 0) { + // Always check server state first to detect any active wishlist processes + const response = await fetch('/api/active-processes'); + if (response.ok) { + const data = await response.json(); + const processes = data.active_processes || []; + + // Look for active wishlist process on server + const serverWishlistProcess = processes.find(p => p.playlist_id === playlistId); + const clientWishlistProcess = activeDownloadProcesses[playlistId]; + + if (serverWishlistProcess) { + console.log('Server has active wishlist process:', serverWishlistProcess.batch_id); + + // Check if client state is properly synchronized with server + const isClientStateMismatched = !clientWishlistProcess || + clientWishlistProcess.batchId !== serverWishlistProcess.batch_id || + clientWishlistProcess.status !== 'running'; + + const isModalElementMissing = !clientWishlistProcess?.modalElement || + !document.body.contains(clientWishlistProcess.modalElement); + + if (isClientStateMismatched || isModalElementMissing) { + console.log('Server/client wishlist state mismatch or missing modal, rehydrating...'); + console.log('Client batch ID:', clientWishlistProcess?.batchId, 'Server batch ID:', serverWishlistProcess.batch_id); + console.log('Client status:', clientWishlistProcess?.status, 'Modal exists:', !isModalElementMissing); + + await rehydrateModal(serverWishlistProcess, true); // Force rehydration with user request + return; + } else { + // Client state is properly synchronized, just show existing modal + console.log('Found properly synced wishlist process, showing modal'); + if (clientWishlistProcess.modalElement) { + clientWishlistProcess.modalElement.style.display = 'flex'; + return; + } + } + } + } + + // No active wishlist process on server, proceed with normal modal creation + // First check if there are any tracks in the wishlist + const countResponse = await fetch('/api/wishlist/count'); + const countData = await countResponse.json(); + + if (countData.count === 0) { showToast('Wishlist is empty. No tracks to download.', 'info'); return; } - // Open the wishlist download modal + // Open the wishlist download modal for new process + console.log('No active server process, creating new wishlist modal'); await openDownloadMissingWishlistModal(); } catch (error) { @@ -5812,7 +5749,4 @@ async function clearWishlist(playlistId) { } // --- Global Cleanup on Page Unload --- -window.addEventListener('beforeunload', function() { - console.log("๐Ÿงน Page unload - stopping auto wishlist processing"); - stopAutoWishlistProcessing(); -}); \ No newline at end of file +// Note: Automatic wishlist processing now runs server-side and continues even when browser is closed \ No newline at end of file