Fix auto-import rescan race condition, coverage penalty, and UI
Race condition: scanner re-scanned folders while post-processing was still moving files, causing partial matches and ghost failures. Now tracks in-progress paths and skips them on subsequent scans. Coverage penalty fix: individual tracks that match at 80%+ confidence now auto-import even when overall album coverage is low (e.g. 2 of 18 tracks present). Previously low coverage killed the entire import. Import page: stats bar, filter pills, Scan Now, Approve All, Clear History (clears imported + failed), live scan progress.
This commit is contained in:
Broque Thomas
parent
a2e3ce8000
commit
bbf5af1ce1
5 changed files with 335 additions and 11 deletions
|
|
@ -135,6 +135,7 @@ class AutoImportWorker:
|
|||
|
||||
# State
|
||||
self._folder_snapshots: Dict[str, float] = {} # path -> mtime_sum
|
||||
self._processing_paths: set = set() # Paths currently being processed (skip on rescan)
|
||||
self._current_folder = ''
|
||||
self._current_status = 'idle'
|
||||
self._stats = {'scanned': 0, 'auto_processed': 0, 'pending_review': 0, 'failed': 0}
|
||||
|
|
@ -237,6 +238,11 @@ class AutoImportWorker:
|
|||
|
||||
self._current_folder = candidate.name
|
||||
|
||||
# Skip folders currently being processed by a previous scan cycle
|
||||
if candidate.path in self._processing_paths:
|
||||
logger.debug(f"[Auto-Import] Skipping {candidate.name} — still processing from previous cycle")
|
||||
continue
|
||||
|
||||
# Check if already processed
|
||||
if self._is_already_processed(candidate.folder_hash):
|
||||
continue
|
||||
|
|
@ -248,6 +254,8 @@ class AutoImportWorker:
|
|||
self._stats['scanned'] += 1
|
||||
logger.info(f"[Auto-Import] Processing folder: {candidate.name} ({len(candidate.audio_files)} files)")
|
||||
|
||||
# Mark as in-progress so next scan cycle skips this folder
|
||||
self._processing_paths.add(candidate.path)
|
||||
try:
|
||||
# Phase 3: Identify
|
||||
identification = self._identify_folder(candidate)
|
||||
|
|
@ -272,11 +280,21 @@ class AutoImportWorker:
|
|||
confidence = match_result['confidence']
|
||||
status = 'matched'
|
||||
|
||||
if confidence >= threshold and auto_process:
|
||||
# Phase 5: Auto-process
|
||||
logger.info(f"[Auto-Import] High confidence ({confidence:.0%}) — auto-processing {candidate.name}")
|
||||
# Check if individual track matches are strong even if overall confidence
|
||||
# is low (e.g. only 2 of 18 album tracks present → low coverage kills
|
||||
# overall score, but the 2 tracks match perfectly and should still import)
|
||||
high_conf_matches = [m for m in match_result.get('matches', []) if m['confidence'] >= 0.8]
|
||||
has_strong_individual_matches = len(high_conf_matches) > 0
|
||||
|
||||
if (confidence >= threshold or has_strong_individual_matches) and auto_process:
|
||||
# Phase 5: Auto-process — process all tracks that matched
|
||||
effective_conf = max(confidence, min(m['confidence'] for m in high_conf_matches) if high_conf_matches else 0)
|
||||
logger.info(f"[Auto-Import] Processing {candidate.name} — "
|
||||
f"overall: {confidence:.0%}, {len(high_conf_matches)} strong matches, "
|
||||
f"{match_result.get('matched_count', 0)}/{match_result.get('total_tracks', '?')} tracks")
|
||||
success = self._process_matches(candidate, identification, match_result)
|
||||
status = 'completed' if success else 'failed'
|
||||
confidence = max(confidence, effective_conf)
|
||||
if success:
|
||||
self._stats['auto_processed'] += 1
|
||||
else:
|
||||
|
|
@ -302,6 +320,8 @@ class AutoImportWorker:
|
|||
logger.error(f"[Auto-Import] Error processing {candidate.name}: {e}")
|
||||
self._record_result(candidate, 'failed', 0.0, error_message=str(e))
|
||||
self._stats['failed'] += 1
|
||||
finally:
|
||||
self._processing_paths.discard(candidate.path)
|
||||
|
||||
# Rate limit between folders
|
||||
if self._interruptible_sleep(2):
|
||||
|
|
|
|||
|
|
@ -53267,6 +53267,53 @@ def auto_import_reject(item_id):
|
|||
return jsonify(auto_import_worker.reject_item(item_id))
|
||||
|
||||
|
||||
@app.route('/api/auto-import/scan-now', methods=['POST'])
|
||||
def auto_import_scan_now():
|
||||
"""Trigger an immediate scan cycle."""
|
||||
if not auto_import_worker:
|
||||
return jsonify({"success": False, "error": "Auto-import not available"}), 500
|
||||
if not auto_import_worker.running:
|
||||
return jsonify({"success": False, "error": "Auto-import is not running"}), 400
|
||||
# Run scan in background thread
|
||||
import threading
|
||||
threading.Thread(target=auto_import_worker._scan_cycle, daemon=True).start()
|
||||
return jsonify({"success": True})
|
||||
|
||||
|
||||
@app.route('/api/auto-import/approve-all', methods=['POST'])
|
||||
def auto_import_approve_all():
|
||||
"""Approve all pending review items."""
|
||||
if not auto_import_worker:
|
||||
return jsonify({"success": False, "error": "Auto-import not available"}), 500
|
||||
try:
|
||||
results = auto_import_worker.get_results(status_filter='pending_review', limit=200)
|
||||
count = 0
|
||||
for r in results:
|
||||
result = auto_import_worker.approve_item(r['id'])
|
||||
if result.get('success'):
|
||||
count += 1
|
||||
return jsonify({"success": True, "count": count})
|
||||
except Exception as e:
|
||||
return jsonify({"success": False, "error": str(e)}), 500
|
||||
|
||||
|
||||
@app.route('/api/auto-import/clear-completed', methods=['POST'])
|
||||
def auto_import_clear_completed():
|
||||
"""Remove completed/imported items from history."""
|
||||
if not auto_import_worker:
|
||||
return jsonify({"success": False, "error": "Auto-import not available"}), 500
|
||||
try:
|
||||
db = get_database()
|
||||
with db._get_connection() as conn:
|
||||
cursor = conn.cursor()
|
||||
cursor.execute("DELETE FROM auto_import_history WHERE status IN ('completed', 'approved', 'failed', 'needs_identification', 'rejected')")
|
||||
count = cursor.rowcount
|
||||
conn.commit()
|
||||
return jsonify({"success": True, "count": count})
|
||||
except Exception as e:
|
||||
return jsonify({"success": False, "error": str(e)}), 500
|
||||
|
||||
|
||||
@app.route('/api/import/staging/suggestions', methods=['GET'])
|
||||
def import_staging_suggestions():
|
||||
"""Return cached import suggestions. If cache isn't built yet, returns partial/empty with a flag."""
|
||||
|
|
|
|||
|
|
@ -6048,6 +6048,10 @@
|
|||
<span>Auto-Import</span>
|
||||
</label>
|
||||
<span class="auto-import-status" id="auto-import-status-text">Disabled</span>
|
||||
<button class="auto-import-scan-now-btn" id="auto-import-scan-now" onclick="_autoImportScanNow()" title="Scan staging folder now" style="display:none">
|
||||
<svg width="14" height="14" viewBox="0 0 16 16" fill="currentColor"><path d="M13.65 2.35A8 8 0 1 0 16 8h-2a6 6 0 1 1-1.76-4.24L10 6h6V0l-2.35 2.35z"/></svg>
|
||||
Scan Now
|
||||
</button>
|
||||
</div>
|
||||
<div class="auto-import-settings-row" id="auto-import-settings-row" style="display:none;">
|
||||
<label>Confidence: <input type="range" id="auto-import-confidence" min="50" max="100" value="90" oninput="document.getElementById('auto-import-conf-val').textContent=this.value+'%'"> <span id="auto-import-conf-val">90%</span></label>
|
||||
|
|
@ -6059,11 +6063,32 @@
|
|||
</select></label>
|
||||
<button class="watchlist-action-btn watchlist-action-secondary" onclick="_autoImportSaveSettings()">Save</button>
|
||||
</div>
|
||||
<!-- Live scan progress -->
|
||||
<div class="auto-import-progress" id="auto-import-progress" style="display:none">
|
||||
<div class="auto-import-progress-text" id="auto-import-progress-text">Scanning...</div>
|
||||
<div class="auto-import-progress-bar"><div class="auto-import-progress-fill" id="auto-import-progress-fill"></div></div>
|
||||
</div>
|
||||
</div>
|
||||
<!-- Stats summary -->
|
||||
<div class="auto-import-stats" id="auto-import-stats" style="display:none">
|
||||
<span class="auto-import-stat" id="auto-import-stat-imported">0 imported</span>
|
||||
<span class="auto-import-stat auto-import-stat-review" id="auto-import-stat-review">0 review</span>
|
||||
<span class="auto-import-stat auto-import-stat-failed" id="auto-import-stat-failed">0 failed</span>
|
||||
</div>
|
||||
<!-- Filter pills -->
|
||||
<div class="auto-import-filters" id="auto-import-filters" style="display:none">
|
||||
<button class="adl-pill active" data-filter="all" onclick="_autoImportSetFilter('all')">All</button>
|
||||
<button class="adl-pill" data-filter="pending" onclick="_autoImportSetFilter('pending')">Needs Review</button>
|
||||
<button class="adl-pill" data-filter="imported" onclick="_autoImportSetFilter('imported')">Imported</button>
|
||||
<button class="adl-pill" data-filter="failed" onclick="_autoImportSetFilter('failed')">Failed</button>
|
||||
<div style="flex:1"></div>
|
||||
<button class="auto-import-batch-btn" id="auto-import-approve-all" onclick="_autoImportApproveAll()" style="display:none">Approve All</button>
|
||||
<button class="auto-import-batch-btn auto-import-clear-btn" id="auto-import-clear-completed" onclick="_autoImportClearCompleted()" style="display:none">Clear History</button>
|
||||
</div>
|
||||
<div class="auto-import-results" id="auto-import-results">
|
||||
<div class="auto-import-empty">
|
||||
<p>Enable auto-import to watch your staging folder for new music.</p>
|
||||
<p style="opacity:0.5;font-size:12px;">Drop album folders into your staging directory and SoulSync will identify, match, and import them automatically.</p>
|
||||
<p style="opacity:0.5;font-size:12px;">Drop album folders or single tracks into your staging directory and SoulSync will identify, match, and import them automatically.</p>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
|
|
|||
|
|
@ -66449,6 +66449,7 @@ function importPageSwitchTab(tab) {
|
|||
|
||||
// ── Auto-Import Tab ──
|
||||
let _autoImportPollInterval = null;
|
||||
let _autoImportFilter = 'all';
|
||||
|
||||
function _autoImportStartPolling() {
|
||||
_autoImportStopPolling();
|
||||
|
|
@ -66499,14 +66500,43 @@ async function _autoImportLoadStatus() {
|
|||
const toggle = document.getElementById('auto-import-enabled');
|
||||
const statusText = document.getElementById('auto-import-status-text');
|
||||
const settingsRow = document.getElementById('auto-import-settings-row');
|
||||
const scanNowBtn = document.getElementById('auto-import-scan-now');
|
||||
const progressEl = document.getElementById('auto-import-progress');
|
||||
const progressText = document.getElementById('auto-import-progress-text');
|
||||
|
||||
if (toggle) toggle.checked = data.running;
|
||||
if (settingsRow) settingsRow.style.display = data.running ? '' : 'none';
|
||||
if (scanNowBtn) scanNowBtn.style.display = data.running ? '' : 'none';
|
||||
|
||||
// Live scan progress
|
||||
if (progressEl) {
|
||||
if (data.current_status === 'scanning') {
|
||||
progressEl.style.display = '';
|
||||
if (progressText) {
|
||||
const stats = data.stats || {};
|
||||
progressText.textContent = `Scanning: ${data.current_folder || '...'} (${stats.scanned || 0} processed)`;
|
||||
}
|
||||
} else {
|
||||
progressEl.style.display = 'none';
|
||||
}
|
||||
}
|
||||
|
||||
if (statusText) {
|
||||
if (data.paused) statusText.textContent = 'Paused';
|
||||
else if (data.current_status === 'scanning') statusText.textContent = `Scanning: ${data.current_folder || '...'}`;
|
||||
else if (data.running) statusText.textContent = 'Watching';
|
||||
else statusText.textContent = 'Disabled';
|
||||
else if (data.current_status === 'scanning') statusText.textContent = 'Scanning...';
|
||||
else if (data.running) {
|
||||
// Show last scan time
|
||||
let watchText = 'Watching';
|
||||
if (data.last_scan_time) {
|
||||
try {
|
||||
const lastScan = new Date(data.last_scan_time);
|
||||
const diffS = Math.floor((Date.now() - lastScan) / 1000);
|
||||
if (diffS < 60) watchText = `Watching (scanned ${diffS}s ago)`;
|
||||
else if (diffS < 3600) watchText = `Watching (scanned ${Math.floor(diffS / 60)}m ago)`;
|
||||
} catch (e) {}
|
||||
}
|
||||
statusText.textContent = watchText;
|
||||
} else statusText.textContent = 'Disabled';
|
||||
statusText.className = 'auto-import-status ' + (data.running ? (data.current_status === 'scanning' ? 'scanning' : 'active') : 'disabled');
|
||||
}
|
||||
} catch (e) {}
|
||||
|
|
@ -66516,19 +66546,61 @@ async function _autoImportLoadResults() {
|
|||
const container = document.getElementById('auto-import-results');
|
||||
if (!container) return;
|
||||
try {
|
||||
const res = await fetch('/api/auto-import/results?limit=30');
|
||||
const res = await fetch('/api/auto-import/results?limit=100');
|
||||
const data = await res.json();
|
||||
if (!data.success || !data.results || data.results.length === 0) {
|
||||
// Keep empty state if no results
|
||||
if (!container.querySelector('.auto-import-card')) {
|
||||
container.innerHTML = `<div class="auto-import-empty">
|
||||
<p>No imports yet. Drop album folders into your staging directory.</p>
|
||||
<p>No imports yet. Drop album folders or single tracks into your staging directory.</p>
|
||||
</div>`;
|
||||
}
|
||||
// Hide stats and filters
|
||||
const statsEl = document.getElementById('auto-import-stats');
|
||||
const filtersEl = document.getElementById('auto-import-filters');
|
||||
if (statsEl) statsEl.style.display = 'none';
|
||||
if (filtersEl) filtersEl.style.display = 'none';
|
||||
return;
|
||||
}
|
||||
|
||||
container.innerHTML = data.results.map((r, idx) => {
|
||||
// Compute stats
|
||||
const allResults = data.results;
|
||||
const importedCount = allResults.filter(r => r.status === 'completed' || r.status === 'approved').length;
|
||||
const reviewCount = allResults.filter(r => r.status === 'pending_review').length;
|
||||
const failedCount = allResults.filter(r => r.status === 'failed' || r.status === 'needs_identification').length;
|
||||
|
||||
// Update stats
|
||||
const statsEl = document.getElementById('auto-import-stats');
|
||||
if (statsEl) {
|
||||
statsEl.style.display = '';
|
||||
document.getElementById('auto-import-stat-imported').textContent = `${importedCount} imported`;
|
||||
document.getElementById('auto-import-stat-review').textContent = `${reviewCount} review`;
|
||||
document.getElementById('auto-import-stat-failed').textContent = `${failedCount} failed`;
|
||||
}
|
||||
|
||||
// Show filters
|
||||
const filtersEl = document.getElementById('auto-import-filters');
|
||||
if (filtersEl) {
|
||||
filtersEl.style.display = '';
|
||||
// Show batch action buttons when applicable
|
||||
const approveAllBtn = document.getElementById('auto-import-approve-all');
|
||||
const clearBtn = document.getElementById('auto-import-clear-completed');
|
||||
if (approveAllBtn) approveAllBtn.style.display = reviewCount > 0 ? '' : 'none';
|
||||
if (clearBtn) clearBtn.style.display = (importedCount + failedCount) > 0 ? '' : 'none';
|
||||
}
|
||||
|
||||
// Apply filter
|
||||
let filtered = allResults;
|
||||
if (_autoImportFilter === 'pending') filtered = allResults.filter(r => r.status === 'pending_review');
|
||||
else if (_autoImportFilter === 'imported') filtered = allResults.filter(r => r.status === 'completed' || r.status === 'approved');
|
||||
else if (_autoImportFilter === 'failed') filtered = allResults.filter(r => r.status === 'failed' || r.status === 'needs_identification');
|
||||
|
||||
if (filtered.length === 0) {
|
||||
const filterName = _autoImportFilter === 'pending' ? 'pending review' : _autoImportFilter;
|
||||
container.innerHTML = `<div class="auto-import-empty"><p>No ${filterName} items.</p></div>`;
|
||||
return;
|
||||
}
|
||||
|
||||
container.innerHTML = filtered.map((r, idx) => {
|
||||
const confPct = Math.round((r.confidence || 0) * 100);
|
||||
const confClass = confPct >= 90 ? 'high' : confPct >= 70 ? 'medium' : 'low';
|
||||
const statusLabels = {
|
||||
|
|
@ -66652,6 +66724,58 @@ async function _autoImportSaveSettings() {
|
|||
} catch (e) { showToast('Error', 'error'); }
|
||||
}
|
||||
|
||||
function _autoImportSetFilter(filter) {
|
||||
_autoImportFilter = filter;
|
||||
document.querySelectorAll('#auto-import-filters .adl-pill').forEach(p =>
|
||||
p.classList.toggle('active', p.dataset.filter === filter));
|
||||
_autoImportLoadResults();
|
||||
}
|
||||
|
||||
async function _autoImportScanNow() {
|
||||
try {
|
||||
const res = await fetch('/api/auto-import/scan-now', { method: 'POST' });
|
||||
const data = await res.json();
|
||||
if (data.success) {
|
||||
showToast('Scan triggered', 'success');
|
||||
_autoImportLoadStatus();
|
||||
} else {
|
||||
showToast(data.error || 'Failed to trigger scan', 'error');
|
||||
}
|
||||
} catch (e) { showToast('Error: ' + e.message, 'error'); }
|
||||
}
|
||||
|
||||
async function _autoImportApproveAll() {
|
||||
const confirmed = await showConfirmDialog({
|
||||
title: 'Approve All',
|
||||
message: 'Approve and import all pending review items?',
|
||||
confirmText: 'Approve All',
|
||||
});
|
||||
if (!confirmed) return;
|
||||
try {
|
||||
const res = await fetch('/api/auto-import/approve-all', { method: 'POST' });
|
||||
const data = await res.json();
|
||||
if (data.success) {
|
||||
showToast(`Approved ${data.count || 0} items`, 'success');
|
||||
_autoImportLoadResults();
|
||||
} else {
|
||||
showToast(data.error || 'Failed', 'error');
|
||||
}
|
||||
} catch (e) { showToast('Error: ' + e.message, 'error'); }
|
||||
}
|
||||
|
||||
async function _autoImportClearCompleted() {
|
||||
try {
|
||||
const res = await fetch('/api/auto-import/clear-completed', { method: 'POST' });
|
||||
const data = await res.json();
|
||||
if (data.success) {
|
||||
showToast(`Cleared ${data.count || 0} imported items`, 'success');
|
||||
_autoImportLoadResults();
|
||||
} else {
|
||||
showToast(data.error || 'Failed', 'error');
|
||||
}
|
||||
} catch (e) { showToast('Error: ' + e.message, 'error'); }
|
||||
}
|
||||
|
||||
function _autoImportToggleDetail(idx) {
|
||||
const trackList = document.getElementById(`auto-import-tracks-${idx}`);
|
||||
if (trackList) {
|
||||
|
|
@ -66659,6 +66783,10 @@ function _autoImportToggleDetail(idx) {
|
|||
}
|
||||
}
|
||||
window._autoImportToggleDetail = _autoImportToggleDetail;
|
||||
window._autoImportSetFilter = _autoImportSetFilter;
|
||||
window._autoImportScanNow = _autoImportScanNow;
|
||||
window._autoImportApproveAll = _autoImportApproveAll;
|
||||
window._autoImportClearCompleted = _autoImportClearCompleted;
|
||||
|
||||
async function _autoImportApprove(id) {
|
||||
try {
|
||||
|
|
|
|||
|
|
@ -58313,6 +58313,110 @@ body.reduce-effects *::after {
|
|||
font-size: 10px; font-weight: 600; color: rgba(255,255,255,0.5);
|
||||
}
|
||||
|
||||
/* Scan Now button */
|
||||
.auto-import-scan-now-btn {
|
||||
background: rgba(255,255,255,0.06);
|
||||
border: 1px solid rgba(255,255,255,0.1);
|
||||
color: rgba(255,255,255,0.6);
|
||||
font-size: 11px;
|
||||
padding: 4px 12px;
|
||||
border-radius: 6px;
|
||||
cursor: pointer;
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 5px;
|
||||
transition: all 0.15s;
|
||||
margin-left: auto;
|
||||
}
|
||||
|
||||
.auto-import-scan-now-btn:hover {
|
||||
background: rgba(255,255,255,0.1);
|
||||
color: rgba(255,255,255,0.9);
|
||||
}
|
||||
|
||||
/* Live progress */
|
||||
.auto-import-progress {
|
||||
margin-top: 8px;
|
||||
padding: 8px 12px;
|
||||
background: rgba(var(--accent-rgb), 0.04);
|
||||
border: 1px solid rgba(var(--accent-rgb), 0.1);
|
||||
border-radius: 8px;
|
||||
}
|
||||
|
||||
.auto-import-progress-text {
|
||||
font-size: 11px;
|
||||
color: rgba(255,255,255,0.6);
|
||||
margin-bottom: 4px;
|
||||
}
|
||||
|
||||
.auto-import-progress-bar {
|
||||
height: 3px;
|
||||
background: rgba(255,255,255,0.06);
|
||||
border-radius: 2px;
|
||||
overflow: hidden;
|
||||
}
|
||||
|
||||
.auto-import-progress-fill {
|
||||
height: 100%;
|
||||
background: rgba(var(--accent-rgb), 0.6);
|
||||
border-radius: 2px;
|
||||
width: 100%;
|
||||
animation: adlPulse 1.5s ease-in-out infinite;
|
||||
}
|
||||
|
||||
/* Stats summary */
|
||||
.auto-import-stats {
|
||||
display: flex;
|
||||
gap: 16px;
|
||||
padding: 8px 0;
|
||||
margin-bottom: 4px;
|
||||
}
|
||||
|
||||
.auto-import-stat {
|
||||
font-size: 12px;
|
||||
color: rgba(255,255,255,0.5);
|
||||
font-weight: 500;
|
||||
}
|
||||
|
||||
.auto-import-stat-review { color: #fbbf24; }
|
||||
.auto-import-stat-failed { color: #f87171; }
|
||||
|
||||
/* Filter pills */
|
||||
.auto-import-filters {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 6px;
|
||||
padding: 6px 0;
|
||||
margin-bottom: 8px;
|
||||
}
|
||||
|
||||
/* Batch action buttons */
|
||||
.auto-import-batch-btn {
|
||||
font-size: 11px;
|
||||
padding: 4px 12px;
|
||||
border-radius: 6px;
|
||||
border: 1px solid rgba(var(--accent-rgb), 0.3);
|
||||
background: rgba(var(--accent-rgb), 0.08);
|
||||
color: rgba(var(--accent-rgb), 1);
|
||||
cursor: pointer;
|
||||
transition: all 0.15s;
|
||||
}
|
||||
|
||||
.auto-import-batch-btn:hover {
|
||||
background: rgba(var(--accent-rgb), 0.15);
|
||||
}
|
||||
|
||||
.auto-import-clear-btn {
|
||||
border-color: rgba(255,255,255,0.1);
|
||||
background: rgba(255,255,255,0.04);
|
||||
color: rgba(255,255,255,0.5);
|
||||
}
|
||||
|
||||
.auto-import-clear-btn:hover {
|
||||
background: rgba(255,255,255,0.08);
|
||||
color: rgba(255,255,255,0.8);
|
||||
}
|
||||
|
||||
.auto-import-card-meta {
|
||||
display: flex; gap: 8px; align-items: center;
|
||||
font-size: 10px; color: rgba(255,255,255,0.3); margin-top: 3px;
|
||||
|
|
|
|||
Loading…
Reference in a new issue