duplicate cleaner tool
This commit is contained in:
Broque Thomas
parent
37c9de0007
commit
aa09b0ba5b
3 changed files with 423 additions and 0 deletions
246
web_server.py
246
web_server.py
|
|
@ -171,6 +171,21 @@ quality_scanner_state = {
|
|||
quality_scanner_lock = threading.Lock()
|
||||
quality_scanner_executor = ThreadPoolExecutor(max_workers=1, thread_name_prefix="QualityScanner")
|
||||
|
||||
# Duplicate Cleaner state
|
||||
duplicate_cleaner_state = {
|
||||
"status": "idle", # idle, running, finished, error
|
||||
"phase": "Ready to scan",
|
||||
"progress": 0,
|
||||
"files_scanned": 0,
|
||||
"total_files": 0,
|
||||
"duplicates_found": 0,
|
||||
"deleted": 0,
|
||||
"space_freed": 0, # in bytes
|
||||
"error_message": ""
|
||||
}
|
||||
duplicate_cleaner_lock = threading.Lock()
|
||||
duplicate_cleaner_executor = ThreadPoolExecutor(max_workers=1, thread_name_prefix="DuplicateCleaner")
|
||||
|
||||
# --- Sync Page Globals ---
|
||||
sync_executor = ThreadPoolExecutor(max_workers=3, thread_name_prefix="SyncWorker")
|
||||
active_sync_workers = {} # Key: playlist_id, Value: Future object
|
||||
|
|
@ -8389,6 +8404,190 @@ def _run_quality_scanner(scope='watchlist'):
|
|||
quality_scanner_state["error_message"] = str(e)
|
||||
quality_scanner_state["phase"] = f"Error: {str(e)}"
|
||||
|
||||
|
||||
def _run_duplicate_cleaner():
|
||||
"""Main duplicate cleaner worker function - scans Transfer folder for duplicate files"""
|
||||
import os
|
||||
import shutil
|
||||
from collections import defaultdict
|
||||
from pathlib import Path
|
||||
|
||||
try:
|
||||
with duplicate_cleaner_lock:
|
||||
duplicate_cleaner_state["status"] = "running"
|
||||
duplicate_cleaner_state["phase"] = "Initializing scan..."
|
||||
duplicate_cleaner_state["progress"] = 0
|
||||
duplicate_cleaner_state["files_scanned"] = 0
|
||||
duplicate_cleaner_state["total_files"] = 0
|
||||
duplicate_cleaner_state["duplicates_found"] = 0
|
||||
duplicate_cleaner_state["deleted"] = 0
|
||||
duplicate_cleaner_state["space_freed"] = 0
|
||||
duplicate_cleaner_state["error_message"] = ""
|
||||
|
||||
print(f"🧹 [Duplicate Cleaner] Starting duplicate scan...")
|
||||
|
||||
# Get Transfer folder path from config
|
||||
transfer_folder = docker_resolve_path(config_manager.get('soulseek.transfer_path', './Transfer'))
|
||||
if not transfer_folder or not os.path.exists(transfer_folder):
|
||||
with duplicate_cleaner_lock:
|
||||
duplicate_cleaner_state["status"] = "error"
|
||||
duplicate_cleaner_state["phase"] = "Transfer folder not configured or does not exist"
|
||||
duplicate_cleaner_state["error_message"] = "Please configure Transfer folder in settings"
|
||||
print(f"❌ [Duplicate Cleaner] Transfer folder not found: {transfer_folder}")
|
||||
return
|
||||
|
||||
# Create deleted folder if it doesn't exist
|
||||
deleted_folder = os.path.join(transfer_folder, 'deleted')
|
||||
os.makedirs(deleted_folder, exist_ok=True)
|
||||
print(f"📁 [Duplicate Cleaner] Deleted folder: {deleted_folder}")
|
||||
|
||||
# Phase 1: Count total files for progress tracking
|
||||
with duplicate_cleaner_lock:
|
||||
duplicate_cleaner_state["phase"] = "Counting files..."
|
||||
|
||||
total_files = 0
|
||||
for root, dirs, files in os.walk(transfer_folder):
|
||||
# Skip the deleted folder itself
|
||||
if 'deleted' in dirs:
|
||||
dirs.remove('deleted')
|
||||
total_files += len(files)
|
||||
|
||||
print(f"📊 [Duplicate Cleaner] Found {total_files} total files to scan")
|
||||
|
||||
with duplicate_cleaner_lock:
|
||||
duplicate_cleaner_state["total_files"] = total_files
|
||||
duplicate_cleaner_state["phase"] = f"Scanning {total_files} files..."
|
||||
|
||||
# Phase 2: Scan and group files by directory and filename
|
||||
# Structure: {directory_path: {filename_without_ext: [full_file_paths]}}
|
||||
files_by_dir_and_name = defaultdict(lambda: defaultdict(list))
|
||||
files_scanned = 0
|
||||
|
||||
# Audio file extensions to consider
|
||||
audio_extensions = {'.flac', '.mp3', '.m4a', '.aac', '.opus', '.ogg', '.wav', '.ape', '.wma', '.alac', '.aiff', '.aif', '.dsf', '.dff'}
|
||||
|
||||
for root, dirs, files in os.walk(transfer_folder):
|
||||
# Skip the deleted folder
|
||||
if 'deleted' in dirs:
|
||||
dirs.remove('deleted')
|
||||
|
||||
for file in files:
|
||||
files_scanned += 1
|
||||
|
||||
# Update progress
|
||||
with duplicate_cleaner_lock:
|
||||
duplicate_cleaner_state["files_scanned"] = files_scanned
|
||||
duplicate_cleaner_state["progress"] = (files_scanned / total_files) * 100 if total_files > 0 else 0
|
||||
duplicate_cleaner_state["phase"] = f"Scanning: {file}"
|
||||
|
||||
# Get file extension
|
||||
file_path = os.path.join(root, file)
|
||||
file_name, file_ext = os.path.splitext(file)
|
||||
file_ext_lower = file_ext.lower()
|
||||
|
||||
# Only process audio files
|
||||
if file_ext_lower not in audio_extensions:
|
||||
continue
|
||||
|
||||
# Group by directory and filename (without extension)
|
||||
files_by_dir_and_name[root][file_name].append({
|
||||
'full_path': file_path,
|
||||
'extension': file_ext_lower,
|
||||
'size': os.path.getsize(file_path)
|
||||
})
|
||||
|
||||
# Phase 3: Process duplicates
|
||||
with duplicate_cleaner_lock:
|
||||
duplicate_cleaner_state["phase"] = "Processing duplicates..."
|
||||
|
||||
# Quality priority: FLAC > OPUS/OGG > M4A/AAC > MP3/WMA
|
||||
format_priority = {
|
||||
'.flac': 1, '.ape': 1, '.wav': 1, '.alac': 1, '.aiff': 1, '.aif': 1, '.dsf': 1, '.dff': 1, # Lossless
|
||||
'.opus': 2, '.ogg': 2, # High quality lossy
|
||||
'.m4a': 3, '.aac': 3, # Standard lossy
|
||||
'.mp3': 4, '.wma': 4 # Lower quality lossy
|
||||
}
|
||||
|
||||
duplicates_found = 0
|
||||
deleted_count = 0
|
||||
space_freed = 0
|
||||
|
||||
for directory, files_by_name in files_by_dir_and_name.items():
|
||||
for filename, file_versions in files_by_name.items():
|
||||
# Only process if we have duplicates (more than one version)
|
||||
if len(file_versions) <= 1:
|
||||
continue
|
||||
|
||||
duplicates_found += len(file_versions) - 1 # Count all but the one we keep
|
||||
print(f"🔍 [Duplicate Cleaner] Found {len(file_versions)} versions of '{filename}' in {directory}")
|
||||
|
||||
# Sort by priority: best format first, then largest size
|
||||
def sort_key(f):
|
||||
priority = format_priority.get(f['extension'], 999)
|
||||
size = f['size']
|
||||
return (priority, -size) # Negative size for descending order
|
||||
|
||||
sorted_versions = sorted(file_versions, key=sort_key)
|
||||
|
||||
# Keep the first one (best quality), delete the rest
|
||||
best_version = sorted_versions[0]
|
||||
print(f"✅ [Duplicate Cleaner] Keeping: {os.path.basename(best_version['full_path'])} "
|
||||
f"({best_version['extension']}, {best_version['size']} bytes)")
|
||||
|
||||
for duplicate_file in sorted_versions[1:]:
|
||||
try:
|
||||
# Move to deleted folder with relative path preserved
|
||||
relative_path = os.path.relpath(duplicate_file['full_path'], transfer_folder)
|
||||
deleted_path = os.path.join(deleted_folder, relative_path)
|
||||
|
||||
# Create subdirectories in deleted folder if needed
|
||||
os.makedirs(os.path.dirname(deleted_path), exist_ok=True)
|
||||
|
||||
# Move the file
|
||||
shutil.move(duplicate_file['full_path'], deleted_path)
|
||||
|
||||
# Track stats
|
||||
deleted_count += 1
|
||||
space_freed += duplicate_file['size']
|
||||
|
||||
print(f"🗑️ [Duplicate Cleaner] Moved to deleted: {os.path.basename(duplicate_file['full_path'])} "
|
||||
f"({duplicate_file['extension']}, {duplicate_file['size']} bytes)")
|
||||
|
||||
# Update stats
|
||||
with duplicate_cleaner_lock:
|
||||
duplicate_cleaner_state["deleted"] = deleted_count
|
||||
duplicate_cleaner_state["space_freed"] = space_freed
|
||||
duplicate_cleaner_state["duplicates_found"] = duplicates_found
|
||||
|
||||
except Exception as e:
|
||||
print(f"❌ [Duplicate Cleaner] Error moving file {duplicate_file['full_path']}: {e}")
|
||||
continue
|
||||
|
||||
# Scan complete
|
||||
with duplicate_cleaner_lock:
|
||||
duplicate_cleaner_state["status"] = "finished"
|
||||
duplicate_cleaner_state["progress"] = 100
|
||||
duplicate_cleaner_state["phase"] = "Cleaning complete"
|
||||
|
||||
space_mb = space_freed / (1024 * 1024)
|
||||
print(f"✅ [Duplicate Cleaner] Scan complete: {files_scanned} files scanned, "
|
||||
f"{duplicates_found} duplicates found, {deleted_count} files moved to deleted folder, "
|
||||
f"{space_mb:.2f} MB freed")
|
||||
|
||||
# Add activity
|
||||
add_activity_item("🧹", "Duplicate Cleaner Complete",
|
||||
f"{deleted_count} files removed, {space_mb:.1f} MB freed", "Now")
|
||||
|
||||
except Exception as e:
|
||||
print(f"❌ [Duplicate Cleaner] Critical error: {e}")
|
||||
import traceback
|
||||
traceback.print_exc()
|
||||
|
||||
with duplicate_cleaner_lock:
|
||||
duplicate_cleaner_state["status"] = "error"
|
||||
duplicate_cleaner_state["error_message"] = str(e)
|
||||
duplicate_cleaner_state["phase"] = f"Error: {str(e)}"
|
||||
|
||||
@app.route('/api/quality-scanner/start', methods=['POST'])
|
||||
def start_quality_scan():
|
||||
"""Start the quality scanner"""
|
||||
|
|
@ -8437,6 +8636,53 @@ def stop_quality_scan():
|
|||
else:
|
||||
return jsonify({"success": False, "error": "No scan is currently running"}), 404
|
||||
|
||||
@app.route('/api/duplicate-cleaner/start', methods=['POST'])
|
||||
def start_duplicate_cleaner():
|
||||
"""Start the duplicate cleaner"""
|
||||
with duplicate_cleaner_lock:
|
||||
if duplicate_cleaner_state["status"] == "running":
|
||||
return jsonify({"success": False, "error": "A scan is already in progress"}), 409
|
||||
|
||||
print(f"🧹 [Duplicate Cleaner API] Starting duplicate cleaner...")
|
||||
|
||||
# Reset state
|
||||
duplicate_cleaner_state["status"] = "running"
|
||||
duplicate_cleaner_state["phase"] = "Initializing..."
|
||||
duplicate_cleaner_state["progress"] = 0
|
||||
duplicate_cleaner_state["files_scanned"] = 0
|
||||
duplicate_cleaner_state["total_files"] = 0
|
||||
duplicate_cleaner_state["duplicates_found"] = 0
|
||||
duplicate_cleaner_state["deleted"] = 0
|
||||
duplicate_cleaner_state["space_freed"] = 0
|
||||
duplicate_cleaner_state["error_message"] = ""
|
||||
|
||||
# Submit worker
|
||||
duplicate_cleaner_executor.submit(_run_duplicate_cleaner)
|
||||
|
||||
add_activity_item("🧹", "Duplicate Cleaner Started", "Scanning Transfer folder", "Now")
|
||||
|
||||
return jsonify({"success": True, "message": "Duplicate cleaner started"})
|
||||
|
||||
@app.route('/api/duplicate-cleaner/status', methods=['GET'])
|
||||
def get_duplicate_cleaner_status():
|
||||
"""Get current duplicate cleaner status"""
|
||||
with duplicate_cleaner_lock:
|
||||
# Convert space_freed from bytes to MB for display
|
||||
state_copy = duplicate_cleaner_state.copy()
|
||||
state_copy["space_freed_mb"] = duplicate_cleaner_state["space_freed"] / (1024 * 1024)
|
||||
return jsonify(state_copy)
|
||||
|
||||
@app.route('/api/duplicate-cleaner/stop', methods=['POST'])
|
||||
def stop_duplicate_cleaner():
|
||||
"""Stop the duplicate cleaner (sets a stop flag)"""
|
||||
with duplicate_cleaner_lock:
|
||||
if duplicate_cleaner_state["status"] == "running":
|
||||
duplicate_cleaner_state["status"] = "finished"
|
||||
duplicate_cleaner_state["phase"] = "Scan stopped by user"
|
||||
return jsonify({"success": True, "message": "Stop request sent"})
|
||||
else:
|
||||
return jsonify({"success": False, "error": "No scan is currently running"}), 404
|
||||
|
||||
# ===============================
|
||||
# == DOWNLOAD MISSING TRACKS ==
|
||||
# ===============================
|
||||
|
|
|
|||
|
|
@ -337,6 +337,39 @@
|
|||
<p class="progress-details-label" id="quality-progress-label">0 / 0 tracks scanned (0.0%)</p>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="tool-card" id="duplicate-cleaner-card">
|
||||
<h4 class="tool-card-title">Duplicate Cleaner</h4>
|
||||
<p class="tool-card-info">Detect and remove duplicate tracks in Transfer folder</p>
|
||||
<div class="tool-card-stats">
|
||||
<div class="stat-item">
|
||||
<span class="stat-item-label">Files Scanned:</span>
|
||||
<span class="stat-item-value" id="duplicate-stat-scanned">0</span>
|
||||
</div>
|
||||
<div class="stat-item">
|
||||
<span class="stat-item-label">Duplicates Found:</span>
|
||||
<span class="stat-item-value" id="duplicate-stat-found">0</span>
|
||||
</div>
|
||||
<div class="stat-item">
|
||||
<span class="stat-item-label">Deleted:</span>
|
||||
<span class="stat-item-value" id="duplicate-stat-deleted">0</span>
|
||||
</div>
|
||||
<div class="stat-item">
|
||||
<span class="stat-item-label">Space Freed:</span>
|
||||
<span class="stat-item-value" id="duplicate-stat-space">0 MB</span>
|
||||
</div>
|
||||
</div>
|
||||
<div class="tool-card-controls">
|
||||
<button id="duplicate-clean-button">Clean Duplicates</button>
|
||||
</div>
|
||||
<div class="tool-card-progress-section">
|
||||
<p class="progress-phase-label" id="duplicate-phase-label">Ready to scan</p>
|
||||
<div class="progress-bar-container">
|
||||
<div class="progress-bar-fill" id="duplicate-progress-bar" style="width: 0%;"></div>
|
||||
</div>
|
||||
<p class="progress-details-label" id="duplicate-progress-label">0 files scanned (0.0%)</p>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
|
|
|
|||
|
|
@ -28,6 +28,7 @@ let searchAbortController = null;
|
|||
let dbStatsInterval = null;
|
||||
let dbUpdateStatusInterval = null;
|
||||
let qualityScannerStatusInterval = null;
|
||||
let duplicateCleanerStatusInterval = null;
|
||||
let wishlistCountInterval = null;
|
||||
|
||||
// --- Add these globals for the Sync Page ---
|
||||
|
|
@ -8914,6 +8915,140 @@ function stopQualityScannerPolling() {
|
|||
}
|
||||
}
|
||||
|
||||
// ============================================
|
||||
// == DUPLICATE CLEANER FUNCTIONS ==
|
||||
// ============================================
|
||||
|
||||
async function handleDuplicateCleanButtonClick() {
|
||||
const button = document.getElementById('duplicate-clean-button');
|
||||
const currentAction = button.textContent;
|
||||
|
||||
if (currentAction === 'Clean Duplicates') {
|
||||
try {
|
||||
button.disabled = true;
|
||||
button.textContent = 'Starting...';
|
||||
const response = await fetch('/api/duplicate-cleaner/start', {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' }
|
||||
});
|
||||
|
||||
if (response.ok) {
|
||||
showToast('Duplicate cleaner started!', 'success');
|
||||
// Start polling immediately to get live status
|
||||
checkAndUpdateDuplicateCleanProgress();
|
||||
} else {
|
||||
const errorData = await response.json();
|
||||
showToast(`Error: ${errorData.error}`, 'error');
|
||||
button.disabled = false;
|
||||
button.textContent = 'Clean Duplicates';
|
||||
}
|
||||
} catch (error) {
|
||||
showToast('Failed to start duplicate cleaner.', 'error');
|
||||
button.disabled = false;
|
||||
button.textContent = 'Clean Duplicates';
|
||||
}
|
||||
|
||||
} else { // "Stop Cleaning"
|
||||
try {
|
||||
const response = await fetch('/api/duplicate-cleaner/stop', { method: 'POST' });
|
||||
if (response.ok) {
|
||||
showToast('Stop request sent.', 'info');
|
||||
} else {
|
||||
showToast('Failed to send stop request.', 'error');
|
||||
}
|
||||
} catch (error) {
|
||||
showToast('Error sending stop request.', 'error');
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async function checkAndUpdateDuplicateCleanProgress() {
|
||||
try {
|
||||
const response = await fetch('/api/duplicate-cleaner/status', {
|
||||
signal: AbortSignal.timeout(10000) // 10 second timeout
|
||||
});
|
||||
if (!response.ok) return;
|
||||
|
||||
const state = await response.json();
|
||||
console.debug('🧹 Duplicate Cleaner Status:', state.status, `${state.files_scanned}/${state.total_files}`, `${state.progress.toFixed(1)}%`);
|
||||
updateDuplicateCleanProgressUI(state);
|
||||
|
||||
// Start polling only if not already polling and status is running
|
||||
if (state.status === 'running' && !duplicateCleanerStatusInterval) {
|
||||
console.log('🔄 Starting duplicate cleaner polling (1 second interval)');
|
||||
duplicateCleanerStatusInterval = setInterval(checkAndUpdateDuplicateCleanProgress, 1000);
|
||||
}
|
||||
|
||||
} catch (error) {
|
||||
console.warn('Could not fetch duplicate cleaner status:', error);
|
||||
// Don't stop polling on network errors - keep trying
|
||||
}
|
||||
}
|
||||
|
||||
function updateDuplicateCleanProgressUI(state) {
|
||||
const button = document.getElementById('duplicate-clean-button');
|
||||
const phaseLabel = document.getElementById('duplicate-phase-label');
|
||||
const progressLabel = document.getElementById('duplicate-progress-label');
|
||||
const progressBar = document.getElementById('duplicate-progress-bar');
|
||||
|
||||
// Stats
|
||||
const scannedStat = document.getElementById('duplicate-stat-scanned');
|
||||
const foundStat = document.getElementById('duplicate-stat-found');
|
||||
const deletedStat = document.getElementById('duplicate-stat-deleted');
|
||||
const spaceStat = document.getElementById('duplicate-stat-space');
|
||||
|
||||
if (!button || !phaseLabel || !progressLabel || !progressBar) return;
|
||||
|
||||
// Update stats
|
||||
if (scannedStat) scannedStat.textContent = state.files_scanned || 0;
|
||||
if (foundStat) foundStat.textContent = state.duplicates_found || 0;
|
||||
if (deletedStat) deletedStat.textContent = state.deleted || 0;
|
||||
if (spaceStat) {
|
||||
const spaceMB = state.space_freed_mb || 0;
|
||||
if (spaceMB >= 1024) {
|
||||
spaceStat.textContent = `${(spaceMB / 1024).toFixed(2)} GB`;
|
||||
} else {
|
||||
spaceStat.textContent = `${spaceMB.toFixed(2)} MB`;
|
||||
}
|
||||
}
|
||||
|
||||
if (state.status === 'running') {
|
||||
button.textContent = 'Stop Cleaning';
|
||||
button.disabled = false;
|
||||
|
||||
phaseLabel.textContent = state.phase || 'Scanning...';
|
||||
progressLabel.textContent = `${state.files_scanned} / ${state.total_files} files scanned (${state.progress.toFixed(1)}%)`;
|
||||
progressBar.style.width = `${state.progress}%`;
|
||||
} else { // idle, finished, or error
|
||||
stopDuplicateCleanerPolling();
|
||||
button.textContent = 'Clean Duplicates';
|
||||
button.disabled = false;
|
||||
|
||||
if (state.status === 'error') {
|
||||
phaseLabel.textContent = `Error: ${state.error_message}`;
|
||||
progressBar.style.backgroundColor = '#ff4444'; // Red for error
|
||||
} else {
|
||||
phaseLabel.textContent = state.phase || 'Ready to scan';
|
||||
progressBar.style.backgroundColor = '#1db954'; // Green for normal
|
||||
}
|
||||
|
||||
if (state.status === 'finished') {
|
||||
// Show completion toast with results
|
||||
const spaceMB = state.space_freed_mb || 0;
|
||||
const spaceDisplay = spaceMB >= 1024 ? `${(spaceMB / 1024).toFixed(2)} GB` : `${spaceMB.toFixed(1)} MB`;
|
||||
showToast(`Cleaning complete! ${state.deleted} files removed, ${spaceDisplay} freed`, 'success');
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function stopDuplicateCleanerPolling() {
|
||||
if (duplicateCleanerStatusInterval) {
|
||||
console.log('⏹️ Stopping duplicate cleaner polling');
|
||||
clearInterval(duplicateCleanerStatusInterval);
|
||||
duplicateCleanerStatusInterval = null;
|
||||
}
|
||||
}
|
||||
|
||||
function stopWishlistCountPolling() {
|
||||
if (wishlistCountInterval) {
|
||||
clearInterval(wishlistCountInterval);
|
||||
|
|
@ -9019,6 +9154,12 @@ async function loadDashboardData() {
|
|||
qualityScanButton.addEventListener('click', handleQualityScanButtonClick);
|
||||
}
|
||||
|
||||
// Attach event listener for the duplicate cleaner tool
|
||||
const duplicateCleanButton = document.getElementById('duplicate-clean-button');
|
||||
if (duplicateCleanButton) {
|
||||
duplicateCleanButton.addEventListener('click', handleDuplicateCleanButtonClick);
|
||||
}
|
||||
|
||||
// Attach event listener for the wishlist button
|
||||
const wishlistButton = document.getElementById('wishlist-button');
|
||||
if (wishlistButton) {
|
||||
|
|
@ -9062,6 +9203,9 @@ async function loadDashboardData() {
|
|||
// Check for any ongoing quality scanner when the page loads
|
||||
await checkAndUpdateQualityScanProgress();
|
||||
|
||||
// Check for any ongoing duplicate cleaner when the page loads
|
||||
await checkAndUpdateDuplicateCleanProgress();
|
||||
|
||||
// Check for any active download processes that need rehydration
|
||||
await checkForActiveProcesses();
|
||||
|
||||
|
|
|
|||
Loading…
Reference in a new issue