database updater
This commit is contained in:
Broque Thomas
parent
facc45e6f4
commit
2f535d8222
4 changed files with 384 additions and 1 deletions
129
web_server.py
129
web_server.py
|
|
@ -24,6 +24,8 @@ from core.jellyfin_client import JellyfinClient
|
|||
from core.soulseek_client import SoulseekClient
|
||||
from core.tidal_client import TidalClient # Added import for Tidal
|
||||
from core.matching_engine import MusicMatchingEngine
|
||||
from core.database_update_worker import DatabaseUpdateWorker, DatabaseStatsWorker
|
||||
from database.music_database import get_database
|
||||
|
||||
# --- Flask App Setup ---
|
||||
base_dir = os.path.abspath(os.path.dirname(__file__))
|
||||
|
|
@ -81,6 +83,19 @@ stream_lock = threading.Lock() # Prevent race conditions
|
|||
stream_background_task = None
|
||||
stream_executor = ThreadPoolExecutor(max_workers=1) # Only one stream at a time
|
||||
|
||||
db_update_executor = ThreadPoolExecutor(max_workers=1, thread_name_prefix="DBUpdate")
|
||||
db_update_worker = None
|
||||
db_update_state = {
|
||||
"status": "idle", # idle, running, finished, error
|
||||
"phase": "Idle",
|
||||
"progress": 0,
|
||||
"current_item": "",
|
||||
"processed": 0,
|
||||
"total": 0,
|
||||
"error_message": ""
|
||||
}
|
||||
db_update_lock = threading.Lock()
|
||||
|
||||
# --- Global Matched Downloads Context Management ---
|
||||
# Thread-safe storage for matched download contexts
|
||||
# Key: slskd download ID, Value: dict containing Spotify artist/album data
|
||||
|
|
@ -2278,6 +2293,120 @@ def start_simple_background_monitor():
|
|||
import traceback
|
||||
traceback.print_exc()
|
||||
time.sleep(10)
|
||||
|
||||
|
||||
# ===============================
|
||||
# == DATABASE UPDATER API ==
|
||||
# ===============================
|
||||
|
||||
def _db_update_progress_callback(current_item, processed, total, percentage):
|
||||
with db_update_lock:
|
||||
db_update_state.update({
|
||||
"current_item": current_item,
|
||||
"processed": processed,
|
||||
"total": total,
|
||||
"progress": percentage
|
||||
})
|
||||
|
||||
def _db_update_phase_callback(phase):
|
||||
with db_update_lock:
|
||||
db_update_state["phase"] = phase
|
||||
|
||||
def _db_update_finished_callback(total_artists, total_albums, total_tracks, successful, failed):
|
||||
with db_update_lock:
|
||||
db_update_state["status"] = "finished"
|
||||
db_update_state["phase"] = f"Completed: {successful} successful, {failed} failed."
|
||||
|
||||
def _db_update_error_callback(error_message):
|
||||
with db_update_lock:
|
||||
db_update_state["status"] = "error"
|
||||
db_update_state["error_message"] = error_message
|
||||
|
||||
def _run_db_update_task(full_refresh, server_type):
|
||||
"""The actual function that runs in the background thread."""
|
||||
global db_update_worker
|
||||
media_client = None
|
||||
|
||||
if server_type == "plex":
|
||||
media_client = plex_client
|
||||
elif server_type == "jellyfin":
|
||||
media_client = jellyfin_client
|
||||
|
||||
if not media_client:
|
||||
_db_update_error_callback(f"Media client for '{server_type}' not available.")
|
||||
return
|
||||
|
||||
with db_update_lock:
|
||||
db_update_worker = DatabaseUpdateWorker(
|
||||
media_client=media_client,
|
||||
full_refresh=full_refresh,
|
||||
server_type=server_type
|
||||
)
|
||||
# Connect signals to callbacks
|
||||
db_update_worker.progress_updated.connect(_db_update_progress_callback)
|
||||
db_update_worker.phase_changed.connect(_db_update_phase_callback)
|
||||
db_update_worker.finished.connect(_db_update_finished_callback)
|
||||
db_update_worker.error.connect(_db_update_error_callback)
|
||||
|
||||
# This is a blocking call that runs the QThread's logic
|
||||
db_update_worker.run()
|
||||
|
||||
|
||||
@app.route('/api/database/stats', methods=['GET'])
|
||||
def get_database_stats():
|
||||
"""Endpoint to get current database statistics."""
|
||||
try:
|
||||
# This logic is adapted from DatabaseStatsWorker
|
||||
db = get_database()
|
||||
stats = db.get_database_info_for_server()
|
||||
return jsonify(stats)
|
||||
except Exception as e:
|
||||
print(f"Error getting database stats: {e}")
|
||||
return jsonify({"error": str(e)}), 500
|
||||
|
||||
@app.route('/api/database/update', methods=['POST'])
|
||||
def start_database_update():
|
||||
"""Endpoint to start the database update process."""
|
||||
global db_update_worker
|
||||
with db_update_lock:
|
||||
if db_update_state["status"] == "running":
|
||||
return jsonify({"success": False, "error": "An update is already in progress."}), 409
|
||||
|
||||
data = request.get_json()
|
||||
full_refresh = data.get('full_refresh', False)
|
||||
active_server = config_manager.get_active_media_server()
|
||||
|
||||
db_update_state.update({
|
||||
"status": "running",
|
||||
"phase": "Initializing...",
|
||||
"progress": 0, "current_item": "", "processed": 0, "total": 0, "error_message": ""
|
||||
})
|
||||
|
||||
# Submit the worker function to the executor
|
||||
db_update_executor.submit(_run_db_update_task, full_refresh, active_server)
|
||||
|
||||
return jsonify({"success": True, "message": "Database update started."})
|
||||
|
||||
@app.route('/api/database/update/status', methods=['GET'])
|
||||
def get_database_update_status():
|
||||
"""Endpoint to poll for the current update status."""
|
||||
with db_update_lock:
|
||||
return jsonify(db_update_state)
|
||||
|
||||
@app.route('/api/database/update/stop', methods=['POST'])
|
||||
def stop_database_update():
|
||||
"""Endpoint to stop the current database update."""
|
||||
global db_update_worker
|
||||
with db_update_lock:
|
||||
if db_update_worker and db_update_state["status"] == "running":
|
||||
db_update_worker.stop()
|
||||
db_update_state["status"] = "finished"
|
||||
db_update_state["phase"] = "Update stopped by user."
|
||||
return jsonify({"success": True, "message": "Stop request sent."})
|
||||
else:
|
||||
return jsonify({"success": False, "error": "No update is currently running."}), 404
|
||||
|
||||
|
||||
|
||||
monitor_thread = threading.Thread(target=simple_monitor)
|
||||
monitor_thread.daemon = True
|
||||
|
|
|
|||
|
|
@ -222,6 +222,24 @@
|
|||
<div class="tool-card" id="db-updater-card">
|
||||
<h4 class="tool-card-title">Database Updater</h4>
|
||||
<p class="tool-card-info">Last Full Refresh: <span id="db-last-refresh">Never</span></p>
|
||||
<div class="tool-card-stats">
|
||||
<div class="stat-item">
|
||||
<span class="stat-item-label">Artists:</span>
|
||||
<span class="stat-item-value" id="db-stat-artists">0</span>
|
||||
</div>
|
||||
<div class="stat-item">
|
||||
<span class="stat-item-label">Albums:</span>
|
||||
<span class="stat-item-value" id="db-stat-albums">0</span>
|
||||
</div>
|
||||
<div class="stat-item">
|
||||
<span class="stat-item-label">Tracks:</span>
|
||||
<span class="stat-item-value" id="db-stat-tracks">0</span>
|
||||
</div>
|
||||
<div class="stat-item">
|
||||
<span class="stat-item-label">Size:</span>
|
||||
<span class="stat-item-value" id="db-stat-size">0.0 MB</span>
|
||||
</div>
|
||||
</div>
|
||||
<div class="tool-card-controls">
|
||||
<select id="db-refresh-type">
|
||||
<option value="incremental">Incremental Update</option>
|
||||
|
|
|
|||
|
|
@ -22,6 +22,8 @@ let currentFilterFormat = 'all';
|
|||
let currentSortBy = 'quality_score';
|
||||
let isSortReversed = false;
|
||||
let searchAbortController = null;
|
||||
let dbStatsInterval = null;
|
||||
let dbUpdateStatusInterval = null;
|
||||
|
||||
// API endpoints
|
||||
const API = {
|
||||
|
|
@ -117,6 +119,9 @@ function navigateToPage(pageId) {
|
|||
|
||||
async function loadPageData(pageId) {
|
||||
try {
|
||||
// Stop any active polling when navigating away
|
||||
stopDbStatsPolling();
|
||||
stopDbUpdatePolling();
|
||||
switch (pageId) {
|
||||
case 'dashboard':
|
||||
stopDownloadPolling();
|
||||
|
|
@ -127,7 +132,6 @@ async function loadPageData(pageId) {
|
|||
await loadSyncData();
|
||||
break;
|
||||
case 'downloads':
|
||||
// --- FIX: Initialize first, THEN load data. This is the correct order. ---
|
||||
initializeSearch();
|
||||
initializeFilters();
|
||||
await loadDownloadsData();
|
||||
|
|
@ -3208,4 +3212,205 @@ function matchedDownloadAlbumTrack(albumIndex, trackIndex) {
|
|||
openMatchingModal(trackData, false, albumData);
|
||||
}
|
||||
|
||||
// ===========================================
|
||||
// == DASHBOARD DATABASE UPDATER FUNCTIONALITY ==
|
||||
// ===========================================
|
||||
|
||||
// --- State and Polling Management ---
|
||||
|
||||
function stopDbStatsPolling() {
|
||||
if (dbStatsInterval) {
|
||||
clearInterval(dbStatsInterval);
|
||||
dbStatsInterval = null;
|
||||
}
|
||||
}
|
||||
|
||||
function stopDbUpdatePolling() {
|
||||
if (dbUpdateStatusInterval) {
|
||||
clearInterval(dbUpdateStatusInterval);
|
||||
dbUpdateStatusInterval = null;
|
||||
}
|
||||
}
|
||||
|
||||
async function loadDashboardData() {
|
||||
// Attach event listeners for the DB updater tool
|
||||
const updateButton = document.getElementById('db-update-button');
|
||||
if (updateButton) {
|
||||
updateButton.addEventListener('click', handleDbUpdateButtonClick);
|
||||
}
|
||||
|
||||
// Initial load of stats
|
||||
await fetchAndUpdateDbStats();
|
||||
|
||||
// Start periodic refresh of stats (every 30 seconds)
|
||||
stopDbStatsPolling(); // Ensure no duplicates
|
||||
dbStatsInterval = setInterval(fetchAndUpdateDbStats, 30000);
|
||||
|
||||
// Also check the status of any ongoing update when the page loads
|
||||
await checkAndUpdateDbProgress();
|
||||
}
|
||||
|
||||
// --- Data Fetching and UI Updates ---
|
||||
|
||||
async function fetchAndUpdateDbStats() {
|
||||
try {
|
||||
const response = await fetch('/api/database/stats');
|
||||
if (!response.ok) return;
|
||||
|
||||
const stats = await response.json();
|
||||
|
||||
// This function updates the stat cards in the top grid
|
||||
updateDashboardStatCards(stats);
|
||||
|
||||
// This function updates the info within the DB Updater tool card
|
||||
updateDbUpdaterCardInfo(stats);
|
||||
|
||||
} catch (error) {
|
||||
console.warn('Could not fetch DB stats:', error);
|
||||
}
|
||||
}
|
||||
|
||||
function updateDashboardStatCards(stats) {
|
||||
// You can expand this later to update the main stat cards
|
||||
// For now, we focus on the updater tool itself.
|
||||
}
|
||||
|
||||
|
||||
|
||||
function updateDbUpdaterCardInfo(stats) {
|
||||
// Update the detailed stats within the DB Updater tool card
|
||||
const lastRefreshEl = document.getElementById('db-last-refresh');
|
||||
const artistsStatEl = document.getElementById('db-stat-artists');
|
||||
const albumsStatEl = document.getElementById('db-stat-albums');
|
||||
const tracksStatEl = document.getElementById('db-stat-tracks');
|
||||
const sizeStatEl = document.getElementById('db-stat-size');
|
||||
|
||||
if (lastRefreshEl) {
|
||||
if (stats.last_full_refresh) {
|
||||
const date = new Date(stats.last_full_refresh);
|
||||
lastRefreshEl.textContent = date.toLocaleString();
|
||||
} else {
|
||||
lastRefreshEl.textContent = 'Never';
|
||||
}
|
||||
}
|
||||
|
||||
if (artistsStatEl) artistsStatEl.textContent = stats.artists.toLocaleString() || '0';
|
||||
if (albumsStatEl) albumsStatEl.textContent = stats.albums.toLocaleString() || '0';
|
||||
if (tracksStatEl) tracksStatEl.textContent = stats.tracks.toLocaleString() || '0';
|
||||
if (sizeStatEl) sizeStatEl.textContent = `${stats.database_size_mb.toFixed(2)} MB`;
|
||||
|
||||
// Update the title of the tool card to show which server is active
|
||||
const toolCardTitle = document.querySelector('#db-updater-card .tool-card-title');
|
||||
if (toolCardTitle && stats.server_source) {
|
||||
const serverName = stats.server_source.charAt(0).toUpperCase() + stats.server_source.slice(1);
|
||||
toolCardTitle.textContent = `${serverName} Database Updater`;
|
||||
}
|
||||
}
|
||||
|
||||
async function checkAndUpdateDbProgress() {
|
||||
try {
|
||||
const response = await fetch('/api/database/update/status');
|
||||
if (!response.ok) return;
|
||||
|
||||
const state = await response.json();
|
||||
updateDbProgressUI(state);
|
||||
|
||||
if (state.status === 'running') {
|
||||
// If an update is running, start polling for progress
|
||||
stopDbUpdatePolling();
|
||||
dbUpdateStatusInterval = setInterval(checkAndUpdateDbProgress, 1000);
|
||||
}
|
||||
|
||||
} catch (error) {
|
||||
console.warn('Could not fetch DB update status:', error);
|
||||
}
|
||||
}
|
||||
|
||||
function updateDbProgressUI(state) {
|
||||
const button = document.getElementById('db-update-button');
|
||||
const phaseLabel = document.getElementById('db-phase-label');
|
||||
const progressLabel = document.getElementById('db-progress-label');
|
||||
const progressBar = document.getElementById('db-progress-bar');
|
||||
const refreshSelect = document.getElementById('db-refresh-type');
|
||||
|
||||
if (!button || !phaseLabel || !progressLabel || !progressBar || !refreshSelect) return;
|
||||
|
||||
if (state.status === 'running') {
|
||||
button.textContent = 'Stop Update';
|
||||
button.disabled = false;
|
||||
refreshSelect.disabled = true;
|
||||
|
||||
phaseLabel.textContent = state.phase || 'Processing...';
|
||||
progressLabel.textContent = `${state.processed} / ${state.total} artists (${state.progress.toFixed(1)}%)`;
|
||||
progressBar.style.width = `${state.progress}%`;
|
||||
} else { // idle, finished, or error
|
||||
stopDbUpdatePolling();
|
||||
button.textContent = 'Update Database';
|
||||
button.disabled = false;
|
||||
refreshSelect.disabled = false;
|
||||
|
||||
if (state.status === 'error') {
|
||||
phaseLabel.textContent = `Error: ${state.error_message}`;
|
||||
progressBar.style.backgroundColor = '#ff4444'; // Red for error
|
||||
} else {
|
||||
phaseLabel.textContent = state.phase || 'Idle';
|
||||
progressBar.style.backgroundColor = '#1db954'; // Green for normal
|
||||
}
|
||||
|
||||
if (state.status === 'finished' || state.status === 'error') {
|
||||
// Final stats refresh after completion/error
|
||||
setTimeout(fetchAndUpdateDbStats, 500);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
// --- Event Handlers ---
|
||||
|
||||
async function handleDbUpdateButtonClick() {
|
||||
const button = document.getElementById('db-update-button');
|
||||
const currentAction = button.textContent;
|
||||
|
||||
if (currentAction === 'Update Database') {
|
||||
const refreshSelect = document.getElementById('db-refresh-type');
|
||||
const isFullRefresh = refreshSelect.value === 'full';
|
||||
|
||||
if (isFullRefresh) {
|
||||
const confirmed = confirm("⚠️ Full Refresh Warning!\n\nThis will clear and rebuild the database for the active server. It can take a long time. Are you sure you want to proceed?");
|
||||
if (!confirmed) return;
|
||||
}
|
||||
|
||||
try {
|
||||
const response = await fetch('/api/database/update', {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({ full_refresh: isFullRefresh })
|
||||
});
|
||||
|
||||
if (response.ok) {
|
||||
showToast('Database update started!', 'success');
|
||||
// Start polling immediately
|
||||
stopDbUpdatePolling();
|
||||
dbUpdateStatusInterval = setInterval(checkAndUpdateDbProgress, 1000);
|
||||
} else {
|
||||
const errorData = await response.json();
|
||||
showToast(`Error: ${errorData.error}`, 'error');
|
||||
}
|
||||
} catch (error) {
|
||||
showToast('Failed to start update process.', 'error');
|
||||
}
|
||||
|
||||
} else { // "Stop Update"
|
||||
try {
|
||||
const response = await fetch('/api/database/update/stop', { method: 'POST' });
|
||||
if (response.ok) {
|
||||
showToast('Stop request sent.', 'info');
|
||||
} else {
|
||||
showToast('Failed to send stop request.', 'error');
|
||||
}
|
||||
} catch (error) {
|
||||
showToast('Error sending stop request.', 'error');
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -3770,3 +3770,34 @@ body {
|
|||
100% { left: 100%; }
|
||||
}
|
||||
|
||||
|
||||
/* Styles for Stats inside the DB Updater Tool Card */
|
||||
.tool-card-stats {
|
||||
display: grid;
|
||||
grid-template-columns: 1fr 1fr;
|
||||
gap: 8px;
|
||||
padding: 10px;
|
||||
background: rgba(0, 0, 0, 0.2);
|
||||
border-radius: 6px;
|
||||
margin-top: 5px; /* Add some space above */
|
||||
margin-bottom: 5px; /* Add some space below */
|
||||
}
|
||||
|
||||
.stat-item {
|
||||
display: flex;
|
||||
justify-content: space-between;
|
||||
align-items: center;
|
||||
font-size: 12px;
|
||||
}
|
||||
|
||||
.stat-item-label {
|
||||
color: #b3b3b3;
|
||||
}
|
||||
|
||||
.stat-item-value {
|
||||
color: #ffffff;
|
||||
font-weight: 600;
|
||||
background-color: rgba(29, 185, 84, 0.1);
|
||||
border-radius: 4px;
|
||||
padding: 2px 6px;
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in a new issue