video Tools: match music Database Updater (stats, cancel, real progress)
The scan tool now behaves like music's, not just looks like it: - Card matches: help '?' button, 'Last Scan' line, and the Movies/Shows/ Episodes/Size stats grid (populated from /api/video/dashboard on show + after a scan). Same .tool-card-stats markup. - Real progress bar: scanner fetches item totals up front (Plex section. totalSize / Jellyfin TotalRecordCount) and reports a true percent as it processes; the bar actually moves (movies → shows) instead of sitting at 100%. - Cancel: the Scan button toggles to 'Cancel' mid-scan and POSTs /api/video/scan/stop; the scanner checks a cancel flag between items and ends in a 'cancelled' state. Mirrors music's stop affordance. Tests: percent reported, cancel stops midway + saves only processed items, stop route registered, tool-card structure. 117 video/integrity tests green.
This commit is contained in:
BoulderBadgeDad
parent
da3f89314b
commit
b5d0b76fe6
8 changed files with 191 additions and 16 deletions
|
|
@ -32,3 +32,9 @@ def register_routes(bp):
|
|||
from . import get_video_db
|
||||
from core.video.scanner import get_video_scanner
|
||||
return jsonify(get_video_scanner(get_video_db()).get_status())
|
||||
|
||||
@bp.route("/scan/stop", methods=["POST"])
|
||||
def video_scan_stop():
|
||||
from . import get_video_db
|
||||
from core.video.scanner import get_video_scanner
|
||||
return jsonify(get_video_scanner(get_video_db()).cancel())
|
||||
|
|
|
|||
|
|
@ -39,6 +39,7 @@ class VideoLibraryScanner:
|
|||
self._lock = threading.Lock()
|
||||
self._status = {"state": "idle"}
|
||||
self._thread = None
|
||||
self._cancel = False
|
||||
|
||||
def get_status(self) -> dict:
|
||||
with self._lock:
|
||||
|
|
@ -48,6 +49,15 @@ class VideoLibraryScanner:
|
|||
with self._lock:
|
||||
self._status.update(kw)
|
||||
|
||||
def cancel(self) -> dict:
|
||||
"""Request the running scan to stop after the current item."""
|
||||
with self._lock:
|
||||
if self._status.get("state") == "scanning":
|
||||
self._cancel = True
|
||||
self._status["phase"] = "cancelling"
|
||||
return {"status": "cancelling"}
|
||||
return {"status": "idle"}
|
||||
|
||||
@staticmethod
|
||||
def _norm_mode(mode) -> str:
|
||||
return mode if mode in VALID_MODES else "full"
|
||||
|
|
@ -59,8 +69,9 @@ class VideoLibraryScanner:
|
|||
with self._lock:
|
||||
if self._status.get("state") == "scanning":
|
||||
return {"status": "in_progress"}
|
||||
self._cancel = False
|
||||
self._status = {"state": "scanning", "phase": "starting", "mode": mode,
|
||||
"started_at": time.time(),
|
||||
"started_at": time.time(), "percent": None,
|
||||
"movies": 0, "shows": 0, "episodes": 0}
|
||||
self._thread = threading.Thread(
|
||||
target=self._run, args=(source_factory, mode), daemon=True)
|
||||
|
|
@ -71,12 +82,18 @@ class VideoLibraryScanner:
|
|||
"""Run a scan inline (used by tests / callers that want to block)."""
|
||||
mode = self._norm_mode(mode)
|
||||
with self._lock:
|
||||
self._cancel = False
|
||||
self._status = {"state": "scanning", "phase": "starting", "mode": mode,
|
||||
"started_at": time.time(),
|
||||
"started_at": time.time(), "percent": None,
|
||||
"movies": 0, "shows": 0, "episodes": 0}
|
||||
self._run(source_factory, mode)
|
||||
return self.get_status()
|
||||
|
||||
def _finish_cancelled(self, movies, shows, episodes) -> None:
|
||||
self._set(state="cancelled", phase="cancelled", finished_at=time.time(),
|
||||
movies=movies, shows=shows, episodes=episodes)
|
||||
logger.info("Video scan cancelled at %d movies, %d shows", movies, shows)
|
||||
|
||||
def _run(self, source_factory, mode: str = "full") -> None:
|
||||
try:
|
||||
source = source_factory()
|
||||
|
|
@ -88,17 +105,33 @@ class VideoLibraryScanner:
|
|||
incremental = mode == "incremental"
|
||||
do_prune = mode == "deep"
|
||||
|
||||
# Totals up front so the progress bar shows a REAL percentage
|
||||
# (movies + shows are the unit; episodes ride along under each show).
|
||||
total = 0
|
||||
try:
|
||||
c = source.counts(incremental=incremental) or {}
|
||||
total = int(c.get("movies", 0) or 0) + int(c.get("shows", 0) or 0)
|
||||
except Exception:
|
||||
logger.debug("video scan: counts() unavailable; progress will be indeterminate")
|
||||
processed = 0
|
||||
|
||||
def pct():
|
||||
return round(processed / total * 100) if total else None
|
||||
|
||||
# ── Movies ──
|
||||
self._set(phase="scanning movies")
|
||||
self._set(phase="scanning movies", total=total, percent=pct())
|
||||
seen_movies: set[str] = set()
|
||||
movies = 0
|
||||
for item in source.iter_movies(incremental=incremental):
|
||||
if self._cancel:
|
||||
return self._finish_cancelled(movies, 0, 0)
|
||||
self.db.upsert_movie(server, item)
|
||||
seen_movies.add(str(item["server_id"]))
|
||||
movies += 1
|
||||
if movies % 25 == 0:
|
||||
self._set(movies=movies)
|
||||
self._set(movies=movies)
|
||||
processed += 1
|
||||
if movies % 10 == 0:
|
||||
self._set(movies=movies, percent=pct())
|
||||
self._set(movies=movies, percent=pct())
|
||||
# Prune ONLY on a deep scan, and only when we actually saw items —
|
||||
# so a transient empty response can never wipe the library.
|
||||
removed_m = (self.db.prune_missing("movies", server, seen_movies)
|
||||
|
|
@ -110,16 +143,19 @@ class VideoLibraryScanner:
|
|||
shows = 0
|
||||
episodes = 0
|
||||
for show in source.iter_shows(incremental=incremental):
|
||||
if self._cancel:
|
||||
return self._finish_cancelled(movies, shows, episodes)
|
||||
self.db.upsert_show_tree(server, show)
|
||||
seen_shows.add(str(show["server_id"]))
|
||||
shows += 1
|
||||
episodes += sum(len(s.get("episodes", [])) for s in show.get("seasons", []))
|
||||
self._set(shows=shows, episodes=episodes)
|
||||
processed += 1
|
||||
self._set(shows=shows, episodes=episodes, percent=pct())
|
||||
removed_s = (self.db.prune_missing("shows", server, seen_shows)
|
||||
if do_prune and seen_shows else 0)
|
||||
|
||||
self._set(state="done", phase="complete", finished_at=time.time(),
|
||||
movies=movies, shows=shows, episodes=episodes,
|
||||
movies=movies, shows=shows, episodes=episodes, percent=100,
|
||||
removed=removed_m + removed_s)
|
||||
logger.info("Video scan (%s) complete: %d movies, %d shows, %d episodes (%d pruned)",
|
||||
mode, movies, shows, episodes, removed_m + removed_s)
|
||||
|
|
|
|||
|
|
@ -112,6 +112,14 @@ class PlexVideoSource:
|
|||
"tv": [{"title": s.title} for s in self._sections("show")],
|
||||
}
|
||||
|
||||
def counts(self, incremental=False) -> dict:
|
||||
"""Cheap item totals (no full fetch) for the progress bar."""
|
||||
m = sum(int(getattr(s, "totalSize", 0) or 0) for s in self._sections("movie", self._movies_lib))
|
||||
sh = sum(int(getattr(s, "totalSize", 0) or 0) for s in self._sections("show", self._tv_lib))
|
||||
if incremental:
|
||||
m, sh = min(m, 100), min(sh, 50)
|
||||
return {"movies": m, "shows": sh}
|
||||
|
||||
def iter_movies(self, incremental=False):
|
||||
for section in self._sections("movie", self._movies_lib):
|
||||
items = section.search(sort="addedAt:desc", maxresults=100) if incremental else section.all()
|
||||
|
|
@ -235,6 +243,18 @@ class JellyfinVideoSource:
|
|||
"tv": [{"title": v.get("Name")} for v in self._views("tvshows")],
|
||||
}
|
||||
|
||||
def counts(self, incremental=False) -> dict:
|
||||
def total(view, itype):
|
||||
resp = self._req(f"/Users/{self.uid}/Items", {
|
||||
"ParentId": view["Id"], "IncludeItemTypes": itype,
|
||||
"Recursive": "true", "Limit": "0"}) or {}
|
||||
return int(resp.get("TotalRecordCount", 0) or 0)
|
||||
m = sum(total(v, "Movie") for v in self._views("movies", self._movies_lib))
|
||||
sh = sum(total(v, "Series") for v in self._views("tvshows", self._tv_lib))
|
||||
if incremental:
|
||||
m, sh = min(m, 100), min(sh, 50)
|
||||
return {"movies": m, "shows": sh}
|
||||
|
||||
def _paged(self, path, params, page_size=500):
|
||||
"""Yield items across pages so large libraries aren't capped/truncated."""
|
||||
start = 0
|
||||
|
|
|
|||
|
|
@ -31,6 +31,7 @@ def test_blueprint_exposes_dashboard_route():
|
|||
assert "/api/video/dashboard" in rules
|
||||
assert "/api/video/scan/request" in rules
|
||||
assert "/api/video/scan/status" in rules
|
||||
assert "/api/video/scan/stop" in rules
|
||||
assert "/api/video/library" in rules
|
||||
assert "/api/video/libraries" in rules
|
||||
assert any(r.startswith("/api/video/poster/") for r in rules)
|
||||
|
|
|
|||
|
|
@ -87,6 +87,42 @@ def test_scan_sync_no_source_reports_error(db):
|
|||
assert st["state"] == "error" and "error" in st
|
||||
|
||||
|
||||
def test_scan_reports_percent_from_counts(db):
|
||||
class S:
|
||||
server_name = "plex"
|
||||
def counts(self, incremental=False):
|
||||
return {"movies": 4, "shows": 0}
|
||||
def iter_movies(self, incremental=False):
|
||||
return iter([{"server_id": "m%d" % i, "title": str(i)} for i in range(4)])
|
||||
def iter_shows(self, incremental=False):
|
||||
return iter([])
|
||||
st = VideoLibraryScanner(db).scan_sync(lambda: S())
|
||||
assert st["state"] == "done"
|
||||
assert st["percent"] == 100
|
||||
|
||||
|
||||
def test_scan_cancel_stops_midway(db):
|
||||
scanner = VideoLibraryScanner(db)
|
||||
|
||||
def gen():
|
||||
yield {"server_id": "m1", "title": "A"}
|
||||
scanner.cancel() # request stop after the first item
|
||||
yield {"server_id": "m2", "title": "B"}
|
||||
|
||||
class S:
|
||||
server_name = "plex"
|
||||
def counts(self, incremental=False):
|
||||
return {"movies": 2, "shows": 0}
|
||||
def iter_movies(self, incremental=False):
|
||||
return gen()
|
||||
def iter_shows(self, incremental=False):
|
||||
return iter([])
|
||||
|
||||
st = scanner.scan_sync(lambda: S())
|
||||
assert st["state"] == "cancelled"
|
||||
assert db.dashboard_stats()["library"]["movies"] == 1 # only the first was saved
|
||||
|
||||
|
||||
def test_core_video_imports_nothing_from_music():
|
||||
base = Path(__file__).resolve().parent.parent / "core" / "video"
|
||||
for py in base.glob("*.py"):
|
||||
|
|
|
|||
|
|
@ -168,6 +168,10 @@ def test_video_tools_page_has_three_scan_modes():
|
|||
assert "data-video-scan-run" in block and "data-video-scan-select" in block
|
||||
assert "data-video-scan-phase" in block # reuses music progress markup
|
||||
assert "tool-card" in block and "tool-card-controls" in block # reuses music classes
|
||||
# Matches the music Database Updater card: help button, last-scan line, stats grid.
|
||||
assert "tool-help-button" in block
|
||||
assert "data-video-scan-last" in block
|
||||
assert "tool-card-stats" in block and 'data-video-scan-stat="movies"' in block
|
||||
assert "onclick" not in block
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -762,8 +762,27 @@
|
|||
<div class="tool-card">
|
||||
<div class="tool-card-header">
|
||||
<h4 class="tool-card-title">Library Scan</h4>
|
||||
<button class="tool-help-button" data-tool="video-scan" title="Learn more about this tool">?</button>
|
||||
</div>
|
||||
<p class="tool-card-info">Last Scan: <span data-video-scan-last>Never</span></p>
|
||||
<div class="tool-card-stats">
|
||||
<div class="stat-item">
|
||||
<span class="stat-item-label">Movies:</span>
|
||||
<span class="stat-item-value" data-video-scan-stat="movies">0</span>
|
||||
</div>
|
||||
<div class="stat-item">
|
||||
<span class="stat-item-label">Shows:</span>
|
||||
<span class="stat-item-value" data-video-scan-stat="shows">0</span>
|
||||
</div>
|
||||
<div class="stat-item">
|
||||
<span class="stat-item-label">Episodes:</span>
|
||||
<span class="stat-item-value" data-video-scan-stat="episodes">0</span>
|
||||
</div>
|
||||
<div class="stat-item">
|
||||
<span class="stat-item-label">Size:</span>
|
||||
<span class="stat-item-value" data-video-scan-stat="size">0.0 MB</span>
|
||||
</div>
|
||||
</div>
|
||||
<p class="tool-card-info">Pull movies & shows from your media server. Incremental = recently-added only; Full Refresh = re-read everything; Deep Scan = also remove items no longer on the server.</p>
|
||||
<div class="tool-card-controls">
|
||||
<select data-video-scan-select>
|
||||
<option value="incremental">Incremental Update</option>
|
||||
|
|
|
|||
|
|
@ -40,25 +40,70 @@
|
|||
return (s.movies || 0) + ' movies, ' + (s.shows || 0) + ' shows';
|
||||
}
|
||||
|
||||
function formatSize(bytes) {
|
||||
bytes = Number(bytes) || 0;
|
||||
var mb = bytes / 1048576;
|
||||
if (mb >= 1048576) return (mb / 1048576).toFixed(2) + ' TB';
|
||||
if (mb >= 1024) return (mb / 1024).toFixed(2) + ' GB';
|
||||
return mb.toFixed(1) + ' MB';
|
||||
}
|
||||
|
||||
// Populate the Tools card stat grid (Movies/Shows/Episodes/Size) from the
|
||||
// same dashboard endpoint — reuses existing data, no new API.
|
||||
function loadToolStats() {
|
||||
if (!document.querySelector('[data-video-scan-stat]')) return;
|
||||
fetch('/api/video/dashboard', { headers: { 'Accept': 'application/json' } })
|
||||
.then(function (r) { return r.ok ? r.json() : null; })
|
||||
.then(function (d) {
|
||||
if (!d || d.error) return;
|
||||
var lib = d.library || {};
|
||||
var set = function (k, v) {
|
||||
var n = document.querySelector('[data-video-scan-stat="' + k + '"]');
|
||||
if (n) n.textContent = v;
|
||||
};
|
||||
set('movies', lib.movies || 0);
|
||||
set('shows', lib.shows || 0);
|
||||
set('episodes', lib.episodes || 0);
|
||||
set('size', formatSize(lib.size_bytes));
|
||||
})
|
||||
.catch(function () { /* leave defaults */ });
|
||||
}
|
||||
|
||||
function setRunLabel(text) {
|
||||
var run = document.querySelector('[data-video-scan-run]');
|
||||
if (run) run.textContent = text;
|
||||
}
|
||||
|
||||
function reflectProgress(s) {
|
||||
var phase = (s.phase || 'scanning');
|
||||
setText('[data-video-scan-phase]', phase.charAt(0).toUpperCase() + phase.slice(1));
|
||||
setText('[data-video-scan-detail]', counts(s));
|
||||
setBar(100);
|
||||
var detail = counts(s);
|
||||
if (s.percent != null) detail += ' · ' + s.percent + '%';
|
||||
setText('[data-video-scan-detail]', detail);
|
||||
// Real percentage when we know the total; otherwise a full bar as a
|
||||
// "working" indicator.
|
||||
setBar(s.percent != null ? s.percent : 100);
|
||||
setRunLabel('Cancel'); // button doubles as cancel while scanning
|
||||
}
|
||||
|
||||
function reflectDone(s) {
|
||||
var run = document.querySelector('[data-video-scan-run]');
|
||||
if (run) run.disabled = false;
|
||||
setRunLabel('Scan Library');
|
||||
if (s.state === 'error') {
|
||||
setText('[data-video-scan-phase]', 'Failed');
|
||||
setText('[data-video-scan-detail]', s.error || 'Scan failed');
|
||||
setBar(0);
|
||||
} else if (s.state === 'cancelled') {
|
||||
setText('[data-video-scan-phase]', 'Cancelled');
|
||||
setText('[data-video-scan-detail]', counts(s) + ' (cancelled)');
|
||||
setBar(0);
|
||||
loadToolStats();
|
||||
} else {
|
||||
setText('[data-video-scan-phase]', 'Complete');
|
||||
setText('[data-video-scan-detail]',
|
||||
counts(s) + (s.removed ? ', ' + s.removed + ' removed' : ''));
|
||||
setBar(100);
|
||||
loadToolStats();
|
||||
try { setText('[data-video-scan-last]', new Date().toLocaleString()); } catch (e) { /* ignore */ }
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -86,9 +131,7 @@
|
|||
function start(mode) {
|
||||
if (scanning) return;
|
||||
scanning = true;
|
||||
var run = document.querySelector('[data-video-scan-run]');
|
||||
if (run) run.disabled = true;
|
||||
var pending = { state: 'scanning', phase: 'starting', mode: mode, movies: 0, shows: 0 };
|
||||
var pending = { state: 'scanning', phase: 'starting', mode: mode, movies: 0, shows: 0, percent: 0 };
|
||||
reflectProgress(pending);
|
||||
emit('soulsync:video-scan-progress', pending);
|
||||
fetch(REQUEST_URL, {
|
||||
|
|
@ -104,6 +147,12 @@
|
|||
});
|
||||
}
|
||||
|
||||
function stop() {
|
||||
setText('[data-video-scan-phase]', 'Cancelling…');
|
||||
fetch('/api/video/scan/stop', { method: 'POST', headers: { 'Accept': 'application/json' } })
|
||||
.catch(function () { /* poll will reconcile */ });
|
||||
}
|
||||
|
||||
function init() {
|
||||
var triggers = document.querySelectorAll('[data-video-scan-mode],[data-video-scan]');
|
||||
for (var i = 0; i < triggers.length; i++) {
|
||||
|
|
@ -118,6 +167,7 @@
|
|||
if (run) {
|
||||
run.addEventListener('click', function (e) {
|
||||
e.preventDefault();
|
||||
if (scanning) { stop(); return; } // button doubles as Cancel mid-scan
|
||||
var sel = document.querySelector('[data-video-scan-select]');
|
||||
start(sel ? sel.value : 'full');
|
||||
});
|
||||
|
|
@ -125,6 +175,9 @@
|
|||
document.addEventListener('soulsync:video-scan-start', function (e) {
|
||||
start((e.detail && e.detail.mode) || 'full');
|
||||
});
|
||||
document.addEventListener('soulsync:video-page-shown', function (e) {
|
||||
if (e && e.detail === 'video-tools') loadToolStats();
|
||||
});
|
||||
}
|
||||
|
||||
if (document.readyState === 'loading') {
|
||||
|
|
|
|||
Loading…
Reference in a new issue