From aa09b0ba5ba0758a009994e737a535e6b8bfa644 Mon Sep 17 00:00:00 2001 From: Broque Thomas Date: Mon, 10 Nov 2025 19:36:36 -0800 Subject: [PATCH] duplicate cleaner tool --- web_server.py | 246 +++++++++++++++++++++++++++++++++++++++++ webui/index.html | 33 ++++++ webui/static/script.js | 144 ++++++++++++++++++++++++ 3 files changed, 423 insertions(+) diff --git a/web_server.py b/web_server.py index 9d5f79b4..e465a115 100644 --- a/web_server.py +++ b/web_server.py @@ -171,6 +171,21 @@ quality_scanner_state = { quality_scanner_lock = threading.Lock() quality_scanner_executor = ThreadPoolExecutor(max_workers=1, thread_name_prefix="QualityScanner") +# Duplicate Cleaner state +duplicate_cleaner_state = { + "status": "idle", # idle, running, finished, error + "phase": "Ready to scan", + "progress": 0, + "files_scanned": 0, + "total_files": 0, + "duplicates_found": 0, + "deleted": 0, + "space_freed": 0, # in bytes + "error_message": "" +} +duplicate_cleaner_lock = threading.Lock() +duplicate_cleaner_executor = ThreadPoolExecutor(max_workers=1, thread_name_prefix="DuplicateCleaner") + # --- Sync Page Globals --- sync_executor = ThreadPoolExecutor(max_workers=3, thread_name_prefix="SyncWorker") active_sync_workers = {} # Key: playlist_id, Value: Future object @@ -8389,6 +8404,190 @@ def _run_quality_scanner(scope='watchlist'): quality_scanner_state["error_message"] = str(e) quality_scanner_state["phase"] = f"Error: {str(e)}" + +def _run_duplicate_cleaner(): + """Main duplicate cleaner worker function - scans Transfer folder for duplicate files""" + import os + import shutil + from collections import defaultdict + from pathlib import Path + + try: + with duplicate_cleaner_lock: + duplicate_cleaner_state["status"] = "running" + duplicate_cleaner_state["phase"] = "Initializing scan..." + duplicate_cleaner_state["progress"] = 0 + duplicate_cleaner_state["files_scanned"] = 0 + duplicate_cleaner_state["total_files"] = 0 + duplicate_cleaner_state["duplicates_found"] = 0 + duplicate_cleaner_state["deleted"] = 0 + duplicate_cleaner_state["space_freed"] = 0 + duplicate_cleaner_state["error_message"] = "" + + print(f"๐Ÿงน [Duplicate Cleaner] Starting duplicate scan...") + + # Get Transfer folder path from config + transfer_folder = docker_resolve_path(config_manager.get('soulseek.transfer_path', './Transfer')) + if not transfer_folder or not os.path.exists(transfer_folder): + with duplicate_cleaner_lock: + duplicate_cleaner_state["status"] = "error" + duplicate_cleaner_state["phase"] = "Transfer folder not configured or does not exist" + duplicate_cleaner_state["error_message"] = "Please configure Transfer folder in settings" + print(f"โŒ [Duplicate Cleaner] Transfer folder not found: {transfer_folder}") + return + + # Create deleted folder if it doesn't exist + deleted_folder = os.path.join(transfer_folder, 'deleted') + os.makedirs(deleted_folder, exist_ok=True) + print(f"๐Ÿ“ [Duplicate Cleaner] Deleted folder: {deleted_folder}") + + # Phase 1: Count total files for progress tracking + with duplicate_cleaner_lock: + duplicate_cleaner_state["phase"] = "Counting files..." + + total_files = 0 + for root, dirs, files in os.walk(transfer_folder): + # Skip the deleted folder itself + if 'deleted' in dirs: + dirs.remove('deleted') + total_files += len(files) + + print(f"๐Ÿ“Š [Duplicate Cleaner] Found {total_files} total files to scan") + + with duplicate_cleaner_lock: + duplicate_cleaner_state["total_files"] = total_files + duplicate_cleaner_state["phase"] = f"Scanning {total_files} files..." + + # Phase 2: Scan and group files by directory and filename + # Structure: {directory_path: {filename_without_ext: [full_file_paths]}} + files_by_dir_and_name = defaultdict(lambda: defaultdict(list)) + files_scanned = 0 + + # Audio file extensions to consider + audio_extensions = {'.flac', '.mp3', '.m4a', '.aac', '.opus', '.ogg', '.wav', '.ape', '.wma', '.alac', '.aiff', '.aif', '.dsf', '.dff'} + + for root, dirs, files in os.walk(transfer_folder): + # Skip the deleted folder + if 'deleted' in dirs: + dirs.remove('deleted') + + for file in files: + files_scanned += 1 + + # Update progress + with duplicate_cleaner_lock: + duplicate_cleaner_state["files_scanned"] = files_scanned + duplicate_cleaner_state["progress"] = (files_scanned / total_files) * 100 if total_files > 0 else 0 + duplicate_cleaner_state["phase"] = f"Scanning: {file}" + + # Get file extension + file_path = os.path.join(root, file) + file_name, file_ext = os.path.splitext(file) + file_ext_lower = file_ext.lower() + + # Only process audio files + if file_ext_lower not in audio_extensions: + continue + + # Group by directory and filename (without extension) + files_by_dir_and_name[root][file_name].append({ + 'full_path': file_path, + 'extension': file_ext_lower, + 'size': os.path.getsize(file_path) + }) + + # Phase 3: Process duplicates + with duplicate_cleaner_lock: + duplicate_cleaner_state["phase"] = "Processing duplicates..." + + # Quality priority: FLAC > OPUS/OGG > M4A/AAC > MP3/WMA + format_priority = { + '.flac': 1, '.ape': 1, '.wav': 1, '.alac': 1, '.aiff': 1, '.aif': 1, '.dsf': 1, '.dff': 1, # Lossless + '.opus': 2, '.ogg': 2, # High quality lossy + '.m4a': 3, '.aac': 3, # Standard lossy + '.mp3': 4, '.wma': 4 # Lower quality lossy + } + + duplicates_found = 0 + deleted_count = 0 + space_freed = 0 + + for directory, files_by_name in files_by_dir_and_name.items(): + for filename, file_versions in files_by_name.items(): + # Only process if we have duplicates (more than one version) + if len(file_versions) <= 1: + continue + + duplicates_found += len(file_versions) - 1 # Count all but the one we keep + print(f"๐Ÿ” [Duplicate Cleaner] Found {len(file_versions)} versions of '{filename}' in {directory}") + + # Sort by priority: best format first, then largest size + def sort_key(f): + priority = format_priority.get(f['extension'], 999) + size = f['size'] + return (priority, -size) # Negative size for descending order + + sorted_versions = sorted(file_versions, key=sort_key) + + # Keep the first one (best quality), delete the rest + best_version = sorted_versions[0] + print(f"โœ… [Duplicate Cleaner] Keeping: {os.path.basename(best_version['full_path'])} " + f"({best_version['extension']}, {best_version['size']} bytes)") + + for duplicate_file in sorted_versions[1:]: + try: + # Move to deleted folder with relative path preserved + relative_path = os.path.relpath(duplicate_file['full_path'], transfer_folder) + deleted_path = os.path.join(deleted_folder, relative_path) + + # Create subdirectories in deleted folder if needed + os.makedirs(os.path.dirname(deleted_path), exist_ok=True) + + # Move the file + shutil.move(duplicate_file['full_path'], deleted_path) + + # Track stats + deleted_count += 1 + space_freed += duplicate_file['size'] + + print(f"๐Ÿ—‘๏ธ [Duplicate Cleaner] Moved to deleted: {os.path.basename(duplicate_file['full_path'])} " + f"({duplicate_file['extension']}, {duplicate_file['size']} bytes)") + + # Update stats + with duplicate_cleaner_lock: + duplicate_cleaner_state["deleted"] = deleted_count + duplicate_cleaner_state["space_freed"] = space_freed + duplicate_cleaner_state["duplicates_found"] = duplicates_found + + except Exception as e: + print(f"โŒ [Duplicate Cleaner] Error moving file {duplicate_file['full_path']}: {e}") + continue + + # Scan complete + with duplicate_cleaner_lock: + duplicate_cleaner_state["status"] = "finished" + duplicate_cleaner_state["progress"] = 100 + duplicate_cleaner_state["phase"] = "Cleaning complete" + + space_mb = space_freed / (1024 * 1024) + print(f"โœ… [Duplicate Cleaner] Scan complete: {files_scanned} files scanned, " + f"{duplicates_found} duplicates found, {deleted_count} files moved to deleted folder, " + f"{space_mb:.2f} MB freed") + + # Add activity + add_activity_item("๐Ÿงน", "Duplicate Cleaner Complete", + f"{deleted_count} files removed, {space_mb:.1f} MB freed", "Now") + + except Exception as e: + print(f"โŒ [Duplicate Cleaner] Critical error: {e}") + import traceback + traceback.print_exc() + + with duplicate_cleaner_lock: + duplicate_cleaner_state["status"] = "error" + duplicate_cleaner_state["error_message"] = str(e) + duplicate_cleaner_state["phase"] = f"Error: {str(e)}" + @app.route('/api/quality-scanner/start', methods=['POST']) def start_quality_scan(): """Start the quality scanner""" @@ -8437,6 +8636,53 @@ def stop_quality_scan(): else: return jsonify({"success": False, "error": "No scan is currently running"}), 404 +@app.route('/api/duplicate-cleaner/start', methods=['POST']) +def start_duplicate_cleaner(): + """Start the duplicate cleaner""" + with duplicate_cleaner_lock: + if duplicate_cleaner_state["status"] == "running": + return jsonify({"success": False, "error": "A scan is already in progress"}), 409 + + print(f"๐Ÿงน [Duplicate Cleaner API] Starting duplicate cleaner...") + + # Reset state + duplicate_cleaner_state["status"] = "running" + duplicate_cleaner_state["phase"] = "Initializing..." + duplicate_cleaner_state["progress"] = 0 + duplicate_cleaner_state["files_scanned"] = 0 + duplicate_cleaner_state["total_files"] = 0 + duplicate_cleaner_state["duplicates_found"] = 0 + duplicate_cleaner_state["deleted"] = 0 + duplicate_cleaner_state["space_freed"] = 0 + duplicate_cleaner_state["error_message"] = "" + + # Submit worker + duplicate_cleaner_executor.submit(_run_duplicate_cleaner) + + add_activity_item("๐Ÿงน", "Duplicate Cleaner Started", "Scanning Transfer folder", "Now") + + return jsonify({"success": True, "message": "Duplicate cleaner started"}) + +@app.route('/api/duplicate-cleaner/status', methods=['GET']) +def get_duplicate_cleaner_status(): + """Get current duplicate cleaner status""" + with duplicate_cleaner_lock: + # Convert space_freed from bytes to MB for display + state_copy = duplicate_cleaner_state.copy() + state_copy["space_freed_mb"] = duplicate_cleaner_state["space_freed"] / (1024 * 1024) + return jsonify(state_copy) + +@app.route('/api/duplicate-cleaner/stop', methods=['POST']) +def stop_duplicate_cleaner(): + """Stop the duplicate cleaner (sets a stop flag)""" + with duplicate_cleaner_lock: + if duplicate_cleaner_state["status"] == "running": + duplicate_cleaner_state["status"] = "finished" + duplicate_cleaner_state["phase"] = "Scan stopped by user" + return jsonify({"success": True, "message": "Stop request sent"}) + else: + return jsonify({"success": False, "error": "No scan is currently running"}), 404 + # =============================== # == DOWNLOAD MISSING TRACKS == # =============================== diff --git a/webui/index.html b/webui/index.html index 8d095605..a6fcf3d9 100644 --- a/webui/index.html +++ b/webui/index.html @@ -337,6 +337,39 @@

0 / 0 tracks scanned (0.0%)

+ +
+

Duplicate Cleaner

+

Detect and remove duplicate tracks in Transfer folder

+
+
+ Files Scanned: + 0 +
+
+ Duplicates Found: + 0 +
+
+ Deleted: + 0 +
+
+ Space Freed: + 0 MB +
+
+
+ +
+
+

Ready to scan

+
+
+
+

0 files scanned (0.0%)

+
+
diff --git a/webui/static/script.js b/webui/static/script.js index d8495b0b..94cd4579 100644 --- a/webui/static/script.js +++ b/webui/static/script.js @@ -28,6 +28,7 @@ let searchAbortController = null; let dbStatsInterval = null; let dbUpdateStatusInterval = null; let qualityScannerStatusInterval = null; +let duplicateCleanerStatusInterval = null; let wishlistCountInterval = null; // --- Add these globals for the Sync Page --- @@ -8914,6 +8915,140 @@ function stopQualityScannerPolling() { } } +// ============================================ +// == DUPLICATE CLEANER FUNCTIONS == +// ============================================ + +async function handleDuplicateCleanButtonClick() { + const button = document.getElementById('duplicate-clean-button'); + const currentAction = button.textContent; + + if (currentAction === 'Clean Duplicates') { + try { + button.disabled = true; + button.textContent = 'Starting...'; + const response = await fetch('/api/duplicate-cleaner/start', { + method: 'POST', + headers: { 'Content-Type': 'application/json' } + }); + + if (response.ok) { + showToast('Duplicate cleaner started!', 'success'); + // Start polling immediately to get live status + checkAndUpdateDuplicateCleanProgress(); + } else { + const errorData = await response.json(); + showToast(`Error: ${errorData.error}`, 'error'); + button.disabled = false; + button.textContent = 'Clean Duplicates'; + } + } catch (error) { + showToast('Failed to start duplicate cleaner.', 'error'); + button.disabled = false; + button.textContent = 'Clean Duplicates'; + } + + } else { // "Stop Cleaning" + try { + const response = await fetch('/api/duplicate-cleaner/stop', { method: 'POST' }); + if (response.ok) { + showToast('Stop request sent.', 'info'); + } else { + showToast('Failed to send stop request.', 'error'); + } + } catch (error) { + showToast('Error sending stop request.', 'error'); + } + } +} + +async function checkAndUpdateDuplicateCleanProgress() { + try { + const response = await fetch('/api/duplicate-cleaner/status', { + signal: AbortSignal.timeout(10000) // 10 second timeout + }); + if (!response.ok) return; + + const state = await response.json(); + console.debug('๐Ÿงน Duplicate Cleaner Status:', state.status, `${state.files_scanned}/${state.total_files}`, `${state.progress.toFixed(1)}%`); + updateDuplicateCleanProgressUI(state); + + // Start polling only if not already polling and status is running + if (state.status === 'running' && !duplicateCleanerStatusInterval) { + console.log('๐Ÿ”„ Starting duplicate cleaner polling (1 second interval)'); + duplicateCleanerStatusInterval = setInterval(checkAndUpdateDuplicateCleanProgress, 1000); + } + + } catch (error) { + console.warn('Could not fetch duplicate cleaner status:', error); + // Don't stop polling on network errors - keep trying + } +} + +function updateDuplicateCleanProgressUI(state) { + const button = document.getElementById('duplicate-clean-button'); + const phaseLabel = document.getElementById('duplicate-phase-label'); + const progressLabel = document.getElementById('duplicate-progress-label'); + const progressBar = document.getElementById('duplicate-progress-bar'); + + // Stats + const scannedStat = document.getElementById('duplicate-stat-scanned'); + const foundStat = document.getElementById('duplicate-stat-found'); + const deletedStat = document.getElementById('duplicate-stat-deleted'); + const spaceStat = document.getElementById('duplicate-stat-space'); + + if (!button || !phaseLabel || !progressLabel || !progressBar) return; + + // Update stats + if (scannedStat) scannedStat.textContent = state.files_scanned || 0; + if (foundStat) foundStat.textContent = state.duplicates_found || 0; + if (deletedStat) deletedStat.textContent = state.deleted || 0; + if (spaceStat) { + const spaceMB = state.space_freed_mb || 0; + if (spaceMB >= 1024) { + spaceStat.textContent = `${(spaceMB / 1024).toFixed(2)} GB`; + } else { + spaceStat.textContent = `${spaceMB.toFixed(2)} MB`; + } + } + + if (state.status === 'running') { + button.textContent = 'Stop Cleaning'; + button.disabled = false; + + phaseLabel.textContent = state.phase || 'Scanning...'; + progressLabel.textContent = `${state.files_scanned} / ${state.total_files} files scanned (${state.progress.toFixed(1)}%)`; + progressBar.style.width = `${state.progress}%`; + } else { // idle, finished, or error + stopDuplicateCleanerPolling(); + button.textContent = 'Clean Duplicates'; + button.disabled = false; + + if (state.status === 'error') { + phaseLabel.textContent = `Error: ${state.error_message}`; + progressBar.style.backgroundColor = '#ff4444'; // Red for error + } else { + phaseLabel.textContent = state.phase || 'Ready to scan'; + progressBar.style.backgroundColor = '#1db954'; // Green for normal + } + + if (state.status === 'finished') { + // Show completion toast with results + const spaceMB = state.space_freed_mb || 0; + const spaceDisplay = spaceMB >= 1024 ? `${(spaceMB / 1024).toFixed(2)} GB` : `${spaceMB.toFixed(1)} MB`; + showToast(`Cleaning complete! ${state.deleted} files removed, ${spaceDisplay} freed`, 'success'); + } + } +} + +function stopDuplicateCleanerPolling() { + if (duplicateCleanerStatusInterval) { + console.log('โน๏ธ Stopping duplicate cleaner polling'); + clearInterval(duplicateCleanerStatusInterval); + duplicateCleanerStatusInterval = null; + } +} + function stopWishlistCountPolling() { if (wishlistCountInterval) { clearInterval(wishlistCountInterval); @@ -9019,6 +9154,12 @@ async function loadDashboardData() { qualityScanButton.addEventListener('click', handleQualityScanButtonClick); } + // Attach event listener for the duplicate cleaner tool + const duplicateCleanButton = document.getElementById('duplicate-clean-button'); + if (duplicateCleanButton) { + duplicateCleanButton.addEventListener('click', handleDuplicateCleanButtonClick); + } + // Attach event listener for the wishlist button const wishlistButton = document.getElementById('wishlist-button'); if (wishlistButton) { @@ -9062,6 +9203,9 @@ async function loadDashboardData() { // Check for any ongoing quality scanner when the page loads await checkAndUpdateQualityScanProgress(); + // Check for any ongoing duplicate cleaner when the page loads + await checkAndUpdateDuplicateCleanProgress(); + // Check for any active download processes that need rehydration await checkForActiveProcesses();