danvics/soulsync
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 1

move wishlist processing to backend

Browse source
This commit is contained in:
Broque Thomas 2025-08-30 21:18:10 -07:00
parent ab9be91a90
commit 697a255047
2 changed files with 274 additions and 143 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

201
web_server.py
View file

@ -120,6 +120,13 @@ download_tasks = {} # task_id -> task state dict
download_batches = {} # batch_id -> {queue, active_count, max_concurrent}
tasks_lock = threading.Lock()
# --- Automatic Wishlist Processing Infrastructure ---
# Server-side timer system for automatic wishlist processing (replaces client-side JavaScript timers)
wishlist_auto_processor = ThreadPoolExecutor(max_workers=1, thread_name_prefix="WishlistAutoProcessor")
wishlist_auto_timer = None # threading.Timer for scheduling next auto-processing
wishlist_auto_processing = False # Flag to prevent concurrent auto-processing
wishlist_timer_lock = threading.Lock() # Thread safety for timer operations
# --- Shared Transfer Data Cache ---
# Cache transfer data to avoid hammering the Soulseek API with multiple concurrent modals
transfer_data_cache = {
@ -3384,6 +3391,139 @@ def start_simple_background_monitor():
monitor_thread.daemon = True
monitor_thread.start()
# ===============================
# == AUTOMATIC WISHLIST PROCESSING ==
# ===============================
def start_wishlist_auto_processing():
"""Start automatic wishlist processing with 1-minute initial delay."""
global wishlist_auto_timer
with wishlist_timer_lock:
# Stop any existing timer to prevent duplicates
if wishlist_auto_timer is not None:
wishlist_auto_timer.cancel()
print("🔄 Starting automatic wishlist processing system (1 minute initial delay)")
wishlist_auto_timer = threading.Timer(60.0, _process_wishlist_automatically) # 1 minute
wishlist_auto_timer.daemon = True
wishlist_auto_timer.start()
def stop_wishlist_auto_processing():
"""Stop automatic wishlist processing and cleanup timer."""
global wishlist_auto_timer, wishlist_auto_processing
with wishlist_timer_lock:
if wishlist_auto_timer is not None:
wishlist_auto_timer.cancel()
wishlist_auto_timer = None
print("⏹️ Stopped automatic wishlist processing")
wishlist_auto_processing = False
def schedule_next_wishlist_processing():
"""Schedule next automatic wishlist processing in 10 minutes."""
global wishlist_auto_timer
with wishlist_timer_lock:
print("⏰ Scheduling next automatic wishlist processing in 10 minutes")
wishlist_auto_timer = threading.Timer(600.0, _process_wishlist_automatically) # 10 minutes
wishlist_auto_timer.daemon = True
wishlist_auto_timer.start()
def _process_wishlist_automatically():
"""Main automatic processing logic that runs in background thread."""
global wishlist_auto_processing
print("🤖 Starting automatic wishlist processing...")
try:
with wishlist_timer_lock:
if wishlist_auto_processing:
print("⚠️ Wishlist auto-processing already running, skipping.")
schedule_next_wishlist_processing()
return
wishlist_auto_processing = True
# Use app context for database operations
with app.app_context():
from core.wishlist_service import get_wishlist_service
wishlist_service = get_wishlist_service()
# Check if wishlist has tracks
count = wishlist_service.get_wishlist_count()
if count == 0:
print("ℹ️ No tracks in wishlist for auto-processing.")
with wishlist_timer_lock:
wishlist_auto_processing = False
schedule_next_wishlist_processing()
return
print(f"🎵 Found {count} tracks in wishlist, starting automatic processing...")
# Check if wishlist processing is already active
playlist_id = "wishlist"
with tasks_lock:
for batch_id, batch_data in download_batches.items():
if (batch_data.get('playlist_id') == playlist_id and
batch_data.get('phase') not in ['complete', 'error', 'cancelled']):
print("⚠️ Wishlist processing already active in another batch, skipping automatic start")
with wishlist_timer_lock:
wishlist_auto_processing = False
schedule_next_wishlist_processing()
return
# Get wishlist tracks for processing
wishlist_tracks = wishlist_service.get_wishlist_tracks_for_download()
if not wishlist_tracks:
print("⚠️ No tracks returned from wishlist service.")
with wishlist_timer_lock:
wishlist_auto_processing = False
schedule_next_wishlist_processing()
return
# Create batch for automatic processing
batch_id = str(uuid.uuid4())
playlist_name = "Wishlist (Auto)"
# Create task queue - convert wishlist tracks to expected format
with tasks_lock:
download_batches[batch_id] = {
'phase': 'analysis',
'playlist_id': playlist_id,
'playlist_name': playlist_name,
'queue': [],
'active_count': 0,
'max_concurrent': 3,
'queue_index': 0,
'analysis_total': len(wishlist_tracks),
'analysis_processed': 0,
'analysis_results': [],
# Track state management (replicating sync.py)
'permanently_failed_tracks': [],
'cancelled_tracks': set(),
# Mark as auto-initiated
'auto_initiated': True,
'auto_processing_timestamp': time.time()
}
print(f"🚀 Starting automatic wishlist batch {batch_id} with {len(wishlist_tracks)} tracks")
# Submit the wishlist processing job using existing infrastructure
missing_download_executor.submit(_run_full_missing_tracks_process, batch_id, playlist_id, wishlist_tracks)
# Don't mark auto_processing as False here - let completion handler do it
except Exception as e:
print(f"❌ Error in automatic wishlist processing: {e}")
import traceback
traceback.print_exc()
with wishlist_timer_lock:
wishlist_auto_processing = False
schedule_next_wishlist_processing()
# ===============================
# == DATABASE UPDATER API ==
# ===============================
@ -3879,6 +4019,49 @@ def _process_failed_tracks_to_wishlist_exact(batch_id):
return {'tracks_added': 0, 'errors': 1, 'total_failed': 0}
def _process_failed_tracks_to_wishlist_exact_with_auto_completion(batch_id):
"""
Process failed tracks to wishlist for auto-initiated batches and handle auto-processing completion.
This extends the standard processing with automatic scheduling of the next cycle.
"""
global wishlist_auto_processing
try:
print(f"🤖 [Auto-Wishlist] Processing completion for auto-initiated batch {batch_id}")
# Run standard wishlist processing
completion_summary = _process_failed_tracks_to_wishlist_exact(batch_id)
# Log auto-processing completion
tracks_added = completion_summary.get('tracks_added', 0)
total_failed = completion_summary.get('total_failed', 0)
print(f"🎉 [Auto-Wishlist] Background processing complete: {tracks_added} added to wishlist, {total_failed} failed")
# Mark auto-processing as complete
with wishlist_timer_lock:
wishlist_auto_processing = False
# Schedule next automatic processing cycle
print("⏰ [Auto-Wishlist] Scheduling next automatic cycle in 10 minutes")
schedule_next_wishlist_processing()
return completion_summary
except Exception as e:
print(f"❌ [Auto-Wishlist] Error in auto-completion processing: {e}")
import traceback
traceback.print_exc()
# Ensure auto-processing flag is reset even on error
with wishlist_timer_lock:
wishlist_auto_processing = False
# Schedule next cycle even after error to maintain continuity
print("⏰ [Auto-Wishlist] Scheduling next cycle after error (10 minutes)")
schedule_next_wishlist_processing()
return {'tracks_added': 0, 'errors': 1, 'total_failed': 0}
def _on_download_completed(batch_id, task_id, success=True):
"""Called when a download completes to start the next one in queue"""
with tasks_lock:
@ -3930,6 +4113,9 @@ def _on_download_completed(batch_id, task_id, success=True):
if all_processed and no_active:
print(f"🎉 [Batch Manager] Batch {batch_id} fully complete - processing failed tracks to wishlist")
# Check if this is an auto-initiated batch
is_auto_batch = batch.get('auto_initiated', False)
# Mark batch as complete and process wishlist outside of lock to prevent deadlocks
batch['phase'] = 'complete'
@ -3937,7 +4123,12 @@ def _on_download_completed(batch_id, task_id, success=True):
download_monitor.stop_monitoring(batch_id)
# Process wishlist outside of the lock to prevent threading issues
missing_download_executor.submit(_process_failed_tracks_to_wishlist_exact, batch_id)
if is_auto_batch:
# For auto-initiated batches, handle completion and schedule next cycle
missing_download_executor.submit(_process_failed_tracks_to_wishlist_exact_with_auto_completion, batch_id)
else:
# For manual batches, use standard wishlist processing
missing_download_executor.submit(_process_failed_tracks_to_wishlist_exact, batch_id)
return # Don't start next batch if we're done
# Start next downloads in queue
@ -4495,7 +4686,8 @@ def get_batch_download_status(batch_id):
batch = download_batches[batch_id]
response_data = {
"phase": batch.get('phase', 'unknown'),
"error": batch.get('error')
"error": batch.get('error'),
"auto_initiated": batch.get('auto_initiated', False)
}
if response_data["phase"] == 'analysis':
@ -5370,4 +5562,9 @@ if __name__ == '__main__':
start_simple_background_monitor()
print("✅ Simple background monitor started")
# Start automatic wishlist processing when server starts
print("🔧 Starting automatic wishlist processing...")
start_wishlist_auto_processing()
print("✅ Automatic wishlist processing started")
app.run(host='0.0.0.0', port=5001, debug=True)

216
webui/static/script.js
View file

@ -29,9 +29,6 @@ let dbStatsInterval = null;
let dbUpdateStatusInterval = null;
let wishlistCountInterval = null;
// --- Auto Wishlist Processing (Simple Timer System) ---
let autoWishlistTimer = null;
// --- Add these globals for the Sync Page ---
let spotifyPlaylists = [];
let selectedPlaylists = new Set();
@ -278,7 +275,6 @@ async function loadPageData(pageId) {
stopDbStatsPolling();
stopDbUpdatePolling();
stopWishlistCountPolling();
stopAutoWishlistProcessing();
switch (pageId) {
case 'dashboard':
stopDownloadPolling();
@ -1784,26 +1780,42 @@ async function checkForActiveProcesses() {
}
}
async function rehydrateModal(processInfo) {
async function rehydrateModal(processInfo, userRequested = false) {
const { playlist_id, playlist_name, batch_id } = processInfo;
console.log(`💧 Rehydrating modal for "${playlist_name}" (batch: ${batch_id})`);
console.log(`💧 Rehydrating modal for "${playlist_name}" (batch: ${batch_id}) - User requested: ${userRequested}`);
// Handle wishlist processes specially
if (playlist_id === "wishlist") {
console.log(`🔍 Current activeDownloadProcesses keys: [${Object.keys(activeDownloadProcesses).join(', ')}]`);
await openDownloadMissingWishlistModal();
const process = activeDownloadProcesses[playlist_id];
if (!process) return;
if (!process) {
console.error('❌ Failed to create wishlist process in activeDownloadProcesses');
return;
}
console.log(`✅ Setting wishlist process state - batchId: ${batch_id}, status: running`);
process.status = 'running';
process.batchId = batch_id;
updateRefreshButtonState();
document.getElementById(`begin-analysis-btn-${playlist_id}`).style.display = 'none';
document.getElementById(`cancel-all-btn-${playlist_id}`).style.display = 'inline-block';
const beginBtn = document.getElementById(`begin-analysis-btn-${playlist_id}`);
const cancelBtn = document.getElementById(`cancel-all-btn-${playlist_id}`);
if (beginBtn) beginBtn.style.display = 'none';
if (cancelBtn) cancelBtn.style.display = 'inline-block';
startModalDownloadPolling(playlist_id);
process.modalElement.style.display = 'none';
// Only hide modal if this is background rehydration, not user-requested
if (!userRequested) {
process.modalElement.style.display = 'none';
console.log('🔍 Hiding wishlist modal for background rehydration');
} else {
process.modalElement.style.display = 'flex';
console.log('🔍 Showing wishlist modal as requested by user');
}
return;
}
@ -2668,8 +2680,8 @@ function startModalDownloadPolling(playlistId) {
// Enhanced check for background auto-processing for wishlist
const isWishlist = (playlistId === 'wishlist');
const isModalHidden = (process.modalElement && process.modalElement.style.display === 'none');
const hasAutoTimer = (autoWishlistTimer !== null);
const isBackgroundWishlist = isWishlist && (isModalHidden || hasAutoTimer);
const isAutoInitiated = data.auto_initiated || false; // Server indicates if batch was auto-started
const isBackgroundWishlist = isWishlist && (isModalHidden || isAutoInitiated);
if (data.phase === 'cancelled') {
process.status = 'cancelled';
@ -2690,7 +2702,7 @@ function startModalDownloadPolling(playlistId) {
// Reset modal to idle state to prevent "complete" phase disruption
setTimeout(() => {
resetWishlistModalToIdleState();
scheduleNextAutoWishlist(); // Schedule next cycle
// Server-side auto-processing will handle next cycle automatically
}, 500);
return; // Skip normal completion handling
@ -5274,124 +5286,7 @@ function stopWishlistCountPolling() {
}
}
// --- Auto Wishlist Processing (Simple Implementation) ---
function startAutoWishlistProcessing() {
console.log("🔄 Starting automatic wishlist processing system (1 minute initial delay)");
stopAutoWishlistProcessing(); // Prevent duplicates
autoWishlistTimer = setTimeout(processWishlistAutomatically, 60000); // 1 minute
}
function stopAutoWishlistProcessing() {
if (autoWishlistTimer) {
clearTimeout(autoWishlistTimer);
autoWishlistTimer = null;
console.log("⏹️ Stopped automatic wishlist processing");
}
}
function scheduleNextAutoWishlist() {
console.log("⏰ Scheduling next automatic wishlist processing in 10 minutes");
autoWishlistTimer = setTimeout(processWishlistAutomatically, 600000); // 10 minutes
}
async function processWishlistAutomatically() {
console.log("🤖 Starting automatic wishlist processing...");
try {
// Check if wishlist has tracks
const countResponse = await fetch('/api/wishlist/count');
if (!countResponse.ok) {
console.warn("⚠️ Could not fetch wishlist count, will retry next cycle");
scheduleNextAutoWishlist();
return;
}
const countData = await countResponse.json();
if (countData.count === 0) {
console.log("ℹ️ Wishlist is empty, skipping automatic processing");
scheduleNextAutoWishlist();
return;
}
console.log(`🎵 Found ${countData.count} tracks in wishlist, starting automatic processing...`);
const playlistId = 'wishlist';
// Check if processing is already active
if (activeDownloadProcesses[playlistId] && activeDownloadProcesses[playlistId].status === 'running') {
console.log("⚠️ Wishlist processing already active, skipping automatic start");
scheduleNextAutoWishlist();
return;
}
// Check if modal is currently being viewed by user
const existingModal = document.getElementById(`download-missing-modal-${playlistId}`);
const userIsViewingModal = existingModal && existingModal.style.display === 'flex';
console.log("🤖 Setting up wishlist modal for automatic processing");
console.log(`🔍 User currently viewing modal: ${userIsViewingModal}`);
// Create modal if it doesn't exist, but keep it hidden for background processing
if (!existingModal) {
await openDownloadMissingWishlistModal();
// Immediately hide the modal since this is background processing
const modal = document.getElementById(`download-missing-modal-${playlistId}`);
if (modal) {
modal.style.display = 'none';
console.log("🤖 Modal created and hidden for background processing");
}
}
// Wait a moment for modal to be ready, then programmatically click "Begin Analysis"
setTimeout(() => {
const beginButton = document.getElementById(`begin-analysis-btn-${playlistId}`);
const modal = document.getElementById(`download-missing-modal-${playlistId}`);
console.log(`🔍 Looking for button with ID: begin-analysis-btn-${playlistId}`);
console.log(`🔍 Button found:`, beginButton);
if (beginButton) {
// Check if button is visible and clickable
const buttonStyle = window.getComputedStyle(beginButton);
console.log(`🔍 Button display style:`, buttonStyle.display);
console.log(`🔍 Button disabled:`, beginButton.disabled);
if (buttonStyle.display === 'none' || beginButton.disabled) {
console.warn("⚠️ Begin Analysis button is hidden or disabled, skipping automatic click");
scheduleNextAutoWishlist();
return;
}
console.log("🤖 Programmatically clicking 'Begin Analysis' button");
beginButton.click();
// Only hide modal if user wasn't actively viewing it
if (!userIsViewingModal) {
setTimeout(() => {
const modal = document.getElementById(`download-missing-modal-${playlistId}`);
if (modal && modal.style.display !== 'none') {
modal.style.display = 'none';
console.log("🤖 Modal hidden - processing continues in background");
}
}, 500);
} else {
console.log("🤖 User is viewing modal - keeping it visible with live updates");
}
} else {
console.warn("⚠️ Could not find Begin Analysis button, will retry next cycle");
console.log("🔍 Available buttons with 'begin-analysis' in ID:",
Array.from(document.querySelectorAll('[id*="begin-analysis"]')).map(el => el.id));
scheduleNextAutoWishlist();
}
}, 1000);
} catch (error) {
console.error("❌ Error in automatic wishlist processing:", error);
scheduleNextAutoWishlist();
}
}
function resetWishlistModalToIdleState() {
// Reset wishlist modal to idle state after background processing completes
@ -5491,8 +5386,7 @@ async function loadDashboardData() {
// Check for any active download processes that need rehydration
await checkForActiveProcesses();
// Start automatic wishlist processing (1 minute delay, then every 10 minutes)
startAutoWishlistProcessing();
// Automatic wishlist processing now runs server-side
}
// --- Data Fetching and UI Updates ---
@ -5736,16 +5630,59 @@ async function handleDbUpdateButtonClick() {
async function handleWishlistButtonClick() {
try {
// First check if there are any tracks in the wishlist
const response = await fetch('/api/wishlist/count');
const data = await response.json();
const playlistId = 'wishlist';
if (data.count === 0) {
// Always check server state first to detect any active wishlist processes
const response = await fetch('/api/active-processes');
if (response.ok) {
const data = await response.json();
const processes = data.active_processes || [];
// Look for active wishlist process on server
const serverWishlistProcess = processes.find(p => p.playlist_id === playlistId);
const clientWishlistProcess = activeDownloadProcesses[playlistId];
if (serverWishlistProcess) {
console.log('Server has active wishlist process:', serverWishlistProcess.batch_id);
// Check if client state is properly synchronized with server
const isClientStateMismatched = !clientWishlistProcess ||
clientWishlistProcess.batchId !== serverWishlistProcess.batch_id ||
clientWishlistProcess.status !== 'running';
const isModalElementMissing = !clientWishlistProcess?.modalElement ||
!document.body.contains(clientWishlistProcess.modalElement);
if (isClientStateMismatched || isModalElementMissing) {
console.log('Server/client wishlist state mismatch or missing modal, rehydrating...');
console.log('Client batch ID:', clientWishlistProcess?.batchId, 'Server batch ID:', serverWishlistProcess.batch_id);
console.log('Client status:', clientWishlistProcess?.status, 'Modal exists:', !isModalElementMissing);
await rehydrateModal(serverWishlistProcess, true); // Force rehydration with user request
return;
} else {
// Client state is properly synchronized, just show existing modal
console.log('Found properly synced wishlist process, showing modal');
if (clientWishlistProcess.modalElement) {
clientWishlistProcess.modalElement.style.display = 'flex';
return;
}
}
}
}
// No active wishlist process on server, proceed with normal modal creation
// First check if there are any tracks in the wishlist
const countResponse = await fetch('/api/wishlist/count');
const countData = await countResponse.json();
if (countData.count === 0) {
showToast('Wishlist is empty. No tracks to download.', 'info');
return;
}
// Open the wishlist download modal
// Open the wishlist download modal for new process
console.log('No active server process, creating new wishlist modal');
await openDownloadMissingWishlistModal();
} catch (error) {
@ -5812,7 +5749,4 @@ async function clearWishlist(playlistId) {
}
// --- Global Cleanup on Page Unload ---
window.addEventListener('beforeunload', function() {
console.log("🧹 Page unload - stopping auto wishlist processing");
stopAutoWishlistProcessing();
});
// Note: Automatic wishlist processing now runs server-side and continues even when browser is closed