batch process

web_server.py

@@ -5616,21 +5616,11 @@ def get_active_processes():
     print(f"📊 Active processes check: {len([p for p in active_processes if p['type'] == 'batch'])} download batches, {len([p for p in active_processes if p['type'] == 'youtube_playlist'])} YouTube playlists")
     return jsonify({"active_processes": active_processes})
 
-@app.route('/api/playlists/<batch_id>/download_status', methods=['GET'])
+def _build_batch_status_data(batch_id, batch, live_transfers_lookup):
-def get_batch_download_status(batch_id):
     """
-    Returns real-time status for a batch, now including the
+    Helper function to build status data for a single batch.
-    current phase (analysis, downloading, etc.) and analysis progress.
+    Extracted from get_batch_download_status for reuse in batched endpoint.
     """
-    try:
-        # Use cached transfer data to reduce API calls with multiple concurrent modals
-        live_transfers_lookup = get_cached_transfer_data()
-
-        with tasks_lock:
-            if batch_id not in download_batches:
-                return jsonify({"error": "Batch not found"}), 404
-
-            batch = download_batches[batch_id]
     response_data = {
         "phase": batch.get('phase', 'unknown'),
         "error": batch.get('error'),
@@ -5679,8 +5669,8 @@ def get_batch_download_status(batch_id):
                         elif 'Failed' in state_str or 'Errored' in state_str: 
                             # Don't mark as failed immediately - trigger retry like GUI
                             batch_id_for_retry = None
-                                    for bid, batch in download_batches.items():
+                            for bid, batch_check in download_batches.items():
-                                        if task_id in batch.get('queue', []):
+                                if task_id in batch_check.get('queue', []):
                                     batch_id_for_retry = bid
                                     break
                             if batch_id_for_retry:
@@ -5707,6 +5697,24 @@ def get_batch_download_status(batch_id):
         if response_data["phase"] == 'complete' and 'wishlist_summary' in batch:
             response_data['wishlist_summary'] = batch['wishlist_summary']
 
+    return response_data
+
+@app.route('/api/playlists/<batch_id>/download_status', methods=['GET'])
+def get_batch_download_status(batch_id):
+    """
+    Returns real-time status for a single batch.
+    Now uses shared helper function for consistency with batched endpoint.
+    """
+    try:
+        # Use cached transfer data to reduce API calls with multiple concurrent modals
+        live_transfers_lookup = get_cached_transfer_data()
+
+        with tasks_lock:
+            if batch_id not in download_batches:
+                return jsonify({"error": "Batch not found"}), 404
+
+            batch = download_batches[batch_id]
+            response_data = _build_batch_status_data(batch_id, batch, live_transfers_lookup)
             return jsonify(response_data)
 
     except Exception as e:
@@ -5714,6 +5722,63 @@ def get_batch_download_status(batch_id):
         traceback.print_exc()
         return jsonify({"error": str(e)}), 500
 
+@app.route('/api/download_status/batch', methods=['GET'])
+def get_batched_download_statuses():
+    """
+    NEW: Returns status for multiple download batches in a single request.
+    Dramatically reduces API calls when multiple download modals are active.
+    
+    Query params:
+    - batch_ids: Optional list of specific batch IDs to include
+    - If no batch_ids provided, returns all active batches
+    """
+    try:
+        # Get optional batch ID filtering from query params
+        requested_batch_ids = request.args.getlist('batch_ids')
+        
+        # Use shared cached transfer data - single lookup for all batches
+        live_transfers_lookup = get_cached_transfer_data()
+        
+        response = {"batches": {}}
+        
+        with tasks_lock:
+            # Determine which batches to include
+            if requested_batch_ids:
+                # Filter to only requested batch IDs that exist
+                target_batches = {
+                    bid: batch for bid, batch in download_batches.items() 
+                    if bid in requested_batch_ids
+                }
+            else:
+                # Return all active batches
+                target_batches = download_batches.copy()
+            
+            # Build status data for each batch using shared helper
+            for batch_id, batch in target_batches.items():
+                try:
+                    response["batches"][batch_id] = _build_batch_status_data(
+                        batch_id, batch, live_transfers_lookup
+                    )
+                except Exception as batch_error:
+                    # Don't fail entire request if one batch has issues
+                    print(f"❌ Error processing batch {batch_id}: {batch_error}")
+                    response["batches"][batch_id] = {"error": str(batch_error)}
+        
+        # Add metadata for debugging/monitoring
+        response["metadata"] = {
+            "total_batches": len(response["batches"]),
+            "requested_batch_ids": requested_batch_ids,
+            "timestamp": time.time()
+        }
+        
+        print(f"📊 [Batched Status] Returning status for {len(response['batches'])} batches")
+        return jsonify(response)
+
+    except Exception as e:
+        import traceback
+        traceback.print_exc()
+        return jsonify({"error": str(e)}), 500
+
 @app.route('/api/downloads/cancel_task', methods=['POST'])
 def cancel_download_task():
     """

webui/static/script.js

@@ -2485,11 +2485,16 @@ async function cleanupDownloadProcess(playlistId) {
 
     // Stop any active polling first
     if (process.poller) {
-        console.log(`🛑 Stopping polling for ${playlistId}`);
+        console.log(`🛑 Stopping individual polling for ${playlistId}`);
         clearInterval(process.poller);
         process.poller = null;
     }
     
+    // Mark process as no longer running
+    if (process.status === 'running') {
+        process.status = 'complete';
+    }
+
     // If the process has a batchId, tell the server to clean it up.
     if (process.batchId) {
         try {
@@ -2514,6 +2519,9 @@ async function cleanupDownloadProcess(playlistId) {
     // Remove from client-side global state
     delete activeDownloadProcesses[playlistId];
 
+    // Check if global polling should be stopped
+    checkAndCleanupGlobalPolling();
+
     // Restore card UI (only for non-wishlist playlists)
     if (playlistId !== 'wishlist') {
         updatePlaylistCardUI(playlistId);
@@ -3409,28 +3417,96 @@ function updateTrackAnalysisResults(playlistId, results) {
 
 
 
-function startModalDownloadPolling(playlistId) {
+// ============================================================================
-    const process = activeDownloadProcesses[playlistId];
+// GLOBAL BATCHED POLLING SYSTEM - Optimized for multiple concurrent modals
-    if (!process || !process.batchId) return;
+// ============================================================================
-    if (process.poller) clearInterval(process.poller);
 
-    console.log(`🔄 [Polling] Starting status polling for playlistId: ${playlistId}, batchId: ${process.batchId}`);
+let globalDownloadStatusPoller = null;
 
-    process.poller = setInterval(async () => {
+function startGlobalDownloadPolling() {
-        if (!activeDownloadProcesses[playlistId]) {
+    if (globalDownloadStatusPoller) {
-            clearInterval(process.poller);
+        console.debug('🔄 [Global Polling] Already running, skipping start');
+        return; // Prevent duplicate pollers
+    }
+    
+    console.log('🔄 [Global Polling] Starting batched download status polling');
+    
+    globalDownloadStatusPoller = setInterval(async () => {
+        // Get all active processes that need polling
+        const activeBatchIds = [];
+        const batchToPlaylistMap = {};
+        
+        Object.entries(activeDownloadProcesses).forEach(([playlistId, process]) => {
+            if (process.batchId && process.status === 'running') {
+                activeBatchIds.push(process.batchId);
+                batchToPlaylistMap[process.batchId] = playlistId;
+            }
+        });
+        
+        if (activeBatchIds.length === 0) {
+            console.log('🛑 [Global Polling] No active processes, stopping global poller');
+            stopGlobalDownloadPolling();
             return;
         }
+        
         try {
-            const response = await fetch(`/api/playlists/${process.batchId}/download_status`);
+            // Single batched API call for all active processes
+            const queryParams = activeBatchIds.map(id => `batch_ids=${id}`).join('&');
+            const response = await fetch(`/api/download_status/batch?${queryParams}`);
+            
             if (!response.ok) {
                 throw new Error(`HTTP ${response.status}: ${response.statusText}`);
             }
             
             const data = await response.json();
-            if (data.error) throw new Error(data.error);
+            console.debug(`📊 [Global Polling] Received batched update for ${Object.keys(data.batches).length} processes`);
             
-            console.debug(`📊 [Polling] Status update for ${playlistId}: phase=${data.phase}, tasks=${(data.tasks || []).length}`);
+            // Process each batch's status data using existing logic
+            Object.entries(data.batches).forEach(([batchId, statusData]) => {
+                const playlistId = batchToPlaylistMap[batchId];
+                if (!playlistId || statusData.error) {
+                    if (statusData.error) {
+                        console.error(`❌ [Global Polling] Error for batch ${batchId}:`, statusData.error);
+                    }
+                    return;
+                }
+                
+                // Use existing modal update logic - zero changes needed!
+                processModalStatusUpdate(playlistId, statusData);
+            });
+            
+        } catch (error) {
+            console.error('❌ [Global Polling] Batched request failed:', error);
+            
+            // Fallback: If batched request fails, don't break individual modals
+            // Individual error handling will be preserved in processModalStatusUpdate
+        }
+    }, 1000); // 1 second polling (was 500ms individual = 2x less aggressive)
+}
+
+function stopGlobalDownloadPolling() {
+    if (globalDownloadStatusPoller) {
+        console.log('🛑 [Global Polling] Stopping batched download status polling');
+        clearInterval(globalDownloadStatusPoller);
+        globalDownloadStatusPoller = null;
+    }
+}
+
+function processModalStatusUpdate(playlistId, data) {
+    // This function contains ALL the existing polling logic from startModalDownloadPolling
+    // Extracted so it can be called from both individual and batched polling
+    const process = activeDownloadProcesses[playlistId];
+    if (!process) {
+        console.debug(`⚠️ [Status Update] No process found for ${playlistId}, skipping update`);
+        return;
+    }
+    
+    if (data.error) {
+        console.error(`❌ [Status Update] Error for ${playlistId}: ${data.error}`);
+        return;
+    }
+    
+    console.debug(`📊 [Status Update] Processing update for ${playlistId}: phase=${data.phase}, tasks=${(data.tasks || []).length}`);
     
     // Auto-show wishlist modal during active auto-processing
     const isWishlist = (playlistId === 'wishlist');
@@ -3438,7 +3514,7 @@ function startModalDownloadPolling(playlistId) {
     const isModalHidden = process.modalElement && process.modalElement.style.display === 'none';
     
     if (isWishlist && isAutoInitiated && isModalHidden && currentPage === 'dashboard' && !WishlistModalState.wasUserClosed()) {
-                console.log('🤖 [Polling] Auto-showing wishlist modal during active auto-processing');
+        console.log('🤖 [Status Update] Auto-showing wishlist modal during active auto-processing');
         process.modalElement.style.display = 'flex';
         WishlistModalState.setVisible();
     }
@@ -3538,7 +3614,7 @@ function startModalDownloadPolling(playlistId) {
             
             // Auto-show modal for wishlist auto-processing if user is on dashboard and hasn't closed it
             if (isWishlist && isAutoInitiated && isModalHidden && currentPage === 'dashboard' && !WishlistModalState.wasUserClosed()) {
-                        console.log('🤖 [Polling] Auto-showing wishlist modal for live updates during auto-processing');
+                console.log('🤖 [Status Update] Auto-showing wishlist modal for live updates during auto-processing');
                 process.modalElement.style.display = 'flex';
                 WishlistModalState.setVisible();
                 showToast('Auto-processing wishlist - showing live updates', 'info', 2000);
@@ -3591,9 +3667,6 @@ function startModalDownloadPolling(playlistId) {
                 if (isBackgroundWishlist) {
                     console.log(`🎉 Background wishlist processing complete: ${completedCount} downloaded, ${failedOrCancelledCount} failed`);
                     
-                            // Clean up polling first
-                            clearInterval(process.poller);
-                            
                     // Reset modal to idle state to prevent "complete" phase disruption
                     setTimeout(() => {
                         resetWishlistModalToIdleState();
@@ -3624,43 +3697,75 @@ function startModalDownloadPolling(playlistId) {
             }
             
             document.getElementById(`cancel-all-btn-${playlistId}`).style.display = 'none';
-                    clearInterval(process.poller);
-                    process.poller = null;
-                    updatePlaylistCardUI(playlistId);
-                }
-            }
-        } catch (error) {
-            console.error(`❌ [Polling] Error for ${playlistId} (batch: ${process.batchId}):`, error);
             
-            // Check for 404 or connection errors that indicate batch no longer exists
+            // Mark process as complete and trigger cleanup check
-            const is404Error = error.message.includes('404') || 
-                              error.message.includes('Batch not found') ||
-                              (error instanceof TypeError && error.message.includes('Failed to fetch'));
-            
-            if (is404Error) {
-                console.warn(`🛑 [Polling] Stopping polling for ${playlistId} - batch no longer exists`);
-                
-                // Immediately clear polling to prevent further requests
-                clearInterval(process.poller);
-                process.poller = null;
-                
-                // Mark process as complete to prevent further issues
-                if (process.status !== 'complete') {
             process.status = 'complete';
             updatePlaylistCardUI(playlistId);
+            
+            // Check if any other processes still need polling
+            checkAndCleanupGlobalPolling();
+        }
+    }
 }
 
-                // For artist downloads, ensure proper cleanup happens
+function checkAndCleanupGlobalPolling() {
-                if (playlistId.startsWith('artist_album_')) {
+    // Check if any processes still need polling
-                    console.log(`🧹 Cleaning up orphaned artist download: ${playlistId}`);
+    const hasActivePolling = Object.values(activeDownloadProcesses)
-                    // Trigger artist download status refresh to update UI
+        .some(p => p.batchId && p.status === 'running');
-                    updateArtistDownloadsSection();
+    
+    if (!hasActivePolling) {
+        console.log('🧹 [Cleanup] No more active processes, stopping global polling');
+        stopGlobalDownloadPolling();
+    }
 }
 
-                return; // Exit the polling function entirely
+// LEGACY FUNCTION: Keep for backward compatibility, but now uses global polling
+function startModalDownloadPolling(playlistId) {
+    const process = activeDownloadProcesses[playlistId];
+    if (!process || !process.batchId) return;
+    
+    console.log(`🔄 [Legacy Polling] Starting polling for ${playlistId}, delegating to global poller`);
+    
+    // Clear any existing individual poller (cleanup)
+    if (process.poller) {
+        clearInterval(process.poller);
+        process.poller = null;
     }
+    
+    // Mark process as running to be picked up by global poller
+    process.status = 'running';
+    
+    // Start global polling if not already running
+    startGlobalDownloadPolling();
+    
+    // Create dummy poller for backward compatibility with cleanup functions
+    ensureLegacyCompatibility(playlistId);
+}
+
+// For backward compatibility with cleanup functions that expect process.poller
+// Creates a dummy poller that will be cleaned up by the existing cleanup logic
+function createLegacyPoller(playlistId) {
+    const process = activeDownloadProcesses[playlistId];
+    if (!process) return;
+    
+    // Create a dummy interval that just checks if the process is still active
+    // This ensures existing cleanup logic that calls clearInterval(process.poller) works
+    process.poller = setInterval(() => {
+        // This dummy poller doesn't do anything - global poller handles updates
+        if (!activeDownloadProcesses[playlistId] || process.status === 'complete') {
+            clearInterval(process.poller);
+            process.poller = null;
+            return;
+        }
+    }, 5000); // Very infrequent check, just for cleanup compatibility
+}
+
+// Call this to create the legacy poller after starting global polling
+function ensureLegacyCompatibility(playlistId) {
+    const process = activeDownloadProcesses[playlistId];
+    if (process && !process.poller) {
+        createLegacyPoller(playlistId);
     }
-    }, 500);
 }
 async function updateModalWithLiveDownloadProgress() {
     try {