danvics/soulsync
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 1

Merge origin/fix/batch-queue-system into dev

Browse source
This commit is contained in:
JohnBaumb 2026-04-23 11:13:34 -07:00
commit ddae75f0f9
3 changed files with 65 additions and 18 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

62
web_server.py
View file

@ -2545,6 +2545,8 @@ def _get_file_lock(file_path):
# --- Download Missing Tracks Modal State Management ---
# Thread-safe state tracking for modal download functionality with batch management
missing_download_executor = ThreadPoolExecutor(max_workers=3, thread_name_prefix="MissingTrackWorker")
# Separate executor for analysis to prevent starvation when download workers are busy
analysis_executor = ThreadPoolExecutor(max_workers=2, thread_name_prefix="AnalysisWorker")
download_tasks = {} # task_id -> task state dict
download_batches = {} # batch_id -> {queue, active_count, max_concurrent}
tasks_lock = threading.Lock()
@ -3914,6 +3916,7 @@ def _shutdown_runtime_components():
(retag_executor, "retag executor"),
(sync_executor, "sync executor"),
(missing_download_executor, "missing download executor"),
(analysis_executor, "analysis executor"),
(tidal_discovery_executor, "tidal discovery executor"),
(deezer_discovery_executor, "deezer discovery executor"),
(spotify_public_discovery_executor, "spotify public discovery executor"),
@ -24867,7 +24870,7 @@ def _process_wishlist_automatically(automation_id=None):
log_line=f'Started batch: {len(wishlist_tracks)} {current_cycle}', log_type='success')
# Submit the wishlist processing job using existing infrastructure
missing_download_executor.submit(_run_full_missing_tracks_process, batch_id, playlist_id, wishlist_tracks)
_submit_or_queue_batch(batch_id, playlist_id, wishlist_tracks)
# Don't mark auto_processing as False here - let completion handler do it
@ -26065,7 +26068,7 @@ def start_wishlist_missing_downloads():
}
# Submit the wishlist processing job using the same processing function
missing_download_executor.submit(_run_full_missing_tracks_process, batch_id, playlist_id, wishlist_tracks)
_submit_or_queue_batch(batch_id, playlist_id, wishlist_tracks)
return jsonify({
"success": True,
@ -28988,6 +28991,43 @@ def _on_download_completed(batch_id, task_id, success=True):
logger.info(f"[Batch Manager] Starting next batch for {batch_id}")
_start_next_batch_of_downloads(batch_id)
def _submit_or_queue_batch(batch_id, playlist_id, tracks):
"""Submit a batch for analysis, or queue it if 3 analysis slots are full."""
with tasks_lock:
active_analysis_count = sum(1 for b in download_batches.values()
if b.get('phase') == 'analysis')
if active_analysis_count >= 3:
download_batches[batch_id]['phase'] = 'queued'
download_batches[batch_id]['_queued_tracks'] = tracks
download_batches[batch_id]['_queued_playlist_id'] = playlist_id
name = download_batches[batch_id].get('playlist_name', batch_id)
logger.info(f"[Queue] Batch {batch_id} ('{name}') queued — {active_analysis_count} analysis already running")
return
analysis_executor.submit(_run_full_missing_tracks_process, batch_id, playlist_id, tracks)
def _promote_queued_batches():
"""Check for queued batches and promote them to analysis if capacity allows."""
with tasks_lock:
active_analysis_count = sum(1 for b in download_batches.values()
if b.get('phase') == 'analysis')
if active_analysis_count >= 3:
return
# Find batches waiting in queue, ordered by creation (dict insertion order)
for bid, batch in list(download_batches.items()):
if batch.get('phase') != 'queued':
continue
queued_tracks = batch.pop('_queued_tracks', None)
queued_pid = batch.pop('_queued_playlist_id', None)
if queued_tracks and queued_pid:
logger.info(f"[Queue] Promoting batch {bid} ('{batch.get('playlist_name')}') from queued -> analysis")
analysis_executor.submit(_run_full_missing_tracks_process, bid, queued_pid, queued_tracks)
active_analysis_count += 1
if active_analysis_count >= 3:
break
def _run_full_missing_tracks_process(batch_id, playlist_id, tracks_json):
"""
A master worker that handles the entire missing tracks process:
@ -29377,6 +29417,12 @@ def _run_full_missing_tracks_process(batch_id, playlist_id, tracks_json):
download_batches[batch_id]['phase'] = 'downloading'
# Analysis slot freed — promote any queued batches
_promote_queued_batches()
with tasks_lock:
if batch_id not in download_batches: return
# Store album pre-flight results on batch for source reuse
if preflight_source and preflight_tracks:
download_batches[batch_id]['last_good_source'] = preflight_source
@ -33425,16 +33471,6 @@ def start_missing_tracks_process(playlist_id):
if playlist_folder_mode:
logger.info(f"[Playlist Folder] Enabled for playlist: '{playlist_name}'")
# Limit concurrent analysis processes to prevent resource exhaustion
with tasks_lock:
active_analysis_count = sum(1 for batch in download_batches.values()
if batch.get('phase') == 'analysis')
if active_analysis_count >= 3: # Allow max 3 concurrent analysis processes
return jsonify({
"success": False,
"error": "Too many analysis processes running. Please wait for one to complete."
}), 429
batch_id = str(uuid.uuid4())
with tasks_lock:
@ -33519,7 +33555,7 @@ def start_missing_tracks_process(playlist_id):
if '_original_index' not in track:
track['_original_index'] = i
missing_download_executor.submit(_run_full_missing_tracks_process, batch_id, playlist_id, tracks)
_submit_or_queue_batch(batch_id, playlist_id, tracks)
return jsonify({
"success": True,

13
webui/static/pages-extra.js
View file

@ -2163,14 +2163,14 @@ let _adlFilterBatchId = null; // When set, main list shows only this batch
const _batchColorMap = {};
const _batchCompletedAt = {}; // batch_id -> timestamp when first seen as complete
let _batchColorNext = 0;
const _BATCH_COLOR_COUNT = 16;
function _getBatchColor(batchId) {
if (!batchId) return -1;
if (_batchColorMap[batchId] === undefined) {
// Deterministic color from batch_id hash for consistency across reloads
let hash = 0;
for (let i = 0; i < batchId.length; i++) hash = ((hash << 5) - hash + batchId.charCodeAt(i)) | 0;
_batchColorMap[batchId] = Math.abs(hash) % 8;
// Assign colors sequentially so no duplicates until all 16 are used
_batchColorMap[batchId] = _batchColorNext % _BATCH_COLOR_COUNT;
_batchColorNext++;
}
return _batchColorMap[batchId];
}
@ -2507,7 +2507,10 @@ function _adlRenderBatchPanel() {
// Phase label with icon
let phaseText = '';
let phaseIcon = '';
if (batch.phase === 'analysis') {
if (batch.phase === 'queued') {
phaseText = 'Queued';
phaseIcon = '<span style="color:#eab308;margin-right:4px">⏳</span>';
} else if (batch.phase === 'analysis') {
phaseText = 'Analyzing...';
phaseIcon = '<span class="adl-spinner" style="margin-right:4px"></span>';
} else if (batch.phase === 'downloading') {

8
webui/static/style.css
View file

@ -56371,6 +56371,14 @@ body.reduce-effects *::after {
--batch-color-5: 239, 68, 68; /* red */
--batch-color-6: 6, 182, 212; /* cyan */
--batch-color-7: 168, 85, 247; /* purple */
--batch-color-8: 234, 179, 8; /* yellow */
--batch-color-9: 249, 115, 22; /* orange */
--batch-color-10: 34, 197, 94; /* green */
--batch-color-11: 99, 102, 241; /* indigo */
--batch-color-12: 244, 63, 94; /* rose */
--batch-color-13: 20, 184, 166; /* teal */
--batch-color-14: 217, 70, 239; /* fuchsia */
--batch-color-15: 56, 189, 248; /* sky */
}
.adl-layout {