video scan: harden library scope + pause ALL enrichers during any scan
1) Scan only the MAPPED libraries — never fall back to 'all'. The scan path used _sections/_views with the selected name, but an empty name returned ALL sections of that type — so a missing/unreadable selection silently scanned every library (how the 4K movie + 'YouTube' TV libraries leaked in as movies/shows). New _scan_sections / _scan_views return [] when a kind isn't mapped; available_libraries still lists all (for the Settings dropdown). Now an unmapped kind scans NOTHING. 2) A library scan (full/incremental/deep) now pauses EVERY enricher, including the YouTube date enricher — a separate singleton outside engine.workers that kept running through scans. pause_for_scan/resume_after_scan pause+resume it too, only if it wasn't already manually paused (never override the user). kettui: source-scope tests (mapped-only / unmapped-scans-nothing / listing still shows all) + engine pause tests (pauses+resumes YT / preserves manual pause). 123 scanner/source/enrichment tests + isolation guard green.
This commit is contained in:
parent
8349fa78dd
commit
7cd289e7b5
3 changed files with 147 additions and 10 deletions
|
|
@ -105,6 +105,19 @@ class VideoEnrichmentEngine:
|
||||||
if not w.paused:
|
if not w.paused:
|
||||||
w.pause(persist=False)
|
w.pause(persist=False)
|
||||||
self._scan_paused.add(service)
|
self._scan_paused.add(service)
|
||||||
|
# The YouTube date enricher is a separate singleton (not in self.workers) —
|
||||||
|
# pause it too so a scan stops EVERY enricher, not just the matcher/backfill
|
||||||
|
# workers. Only if it wasn't already paused (never override a manual pause).
|
||||||
|
self._scan_paused_yt = False
|
||||||
|
try:
|
||||||
|
from core.video.youtube_enrichment import get_youtube_date_enricher
|
||||||
|
yt = get_youtube_date_enricher()
|
||||||
|
if yt and not getattr(yt, "_paused", False):
|
||||||
|
yt.pause()
|
||||||
|
self._scan_paused_yt = True
|
||||||
|
self._scan_paused.add("youtube")
|
||||||
|
except Exception:
|
||||||
|
logger.debug("video enrichment: could not pause YouTube date enricher for scan", exc_info=True)
|
||||||
if self._scan_paused:
|
if self._scan_paused:
|
||||||
logger.info("video enrichment: paused %s for library scan",
|
logger.info("video enrichment: paused %s for library scan",
|
||||||
", ".join(sorted(self._scan_paused)))
|
", ".join(sorted(self._scan_paused)))
|
||||||
|
|
@ -115,6 +128,15 @@ class VideoEnrichmentEngine:
|
||||||
w = self.workers.get(service)
|
w = self.workers.get(service)
|
||||||
if w:
|
if w:
|
||||||
w.resume(persist=False)
|
w.resume(persist=False)
|
||||||
|
if getattr(self, "_scan_paused_yt", False):
|
||||||
|
try:
|
||||||
|
from core.video.youtube_enrichment import get_youtube_date_enricher
|
||||||
|
yt = get_youtube_date_enricher()
|
||||||
|
if yt:
|
||||||
|
yt.resume()
|
||||||
|
except Exception:
|
||||||
|
logger.debug("video enrichment: could not resume YouTube date enricher", exc_info=True)
|
||||||
|
self._scan_paused_yt = False
|
||||||
if getattr(self, "_scan_paused", None):
|
if getattr(self, "_scan_paused", None):
|
||||||
logger.info("video enrichment: resumed %s after library scan",
|
logger.info("video enrichment: resumed %s after library scan",
|
||||||
", ".join(sorted(self._scan_paused)))
|
", ".join(sorted(self._scan_paused)))
|
||||||
|
|
|
||||||
|
|
@ -308,6 +308,12 @@ class PlexVideoSource:
|
||||||
secs = [s for s in secs if s.title == name]
|
secs = [s for s in secs if s.title == name]
|
||||||
return secs
|
return secs
|
||||||
|
|
||||||
|
def _scan_sections(self, kind: str, name):
|
||||||
|
"""Sections to SCAN for a kind. UNLIKE _sections, an empty name means
|
||||||
|
'this kind isn't mapped' → scan NOTHING (never fall back to all sections).
|
||||||
|
Prevents a missing selection from silently pulling every library."""
|
||||||
|
return self._sections(kind, name) if name else []
|
||||||
|
|
||||||
def available_libraries(self) -> dict:
|
def available_libraries(self) -> dict:
|
||||||
return {
|
return {
|
||||||
"movies": [{"title": s.title} for s in self._sections("movie")],
|
"movies": [{"title": s.title} for s in self._sections("movie")],
|
||||||
|
|
@ -316,14 +322,14 @@ class PlexVideoSource:
|
||||||
|
|
||||||
def counts(self, incremental=False) -> dict:
|
def counts(self, incremental=False) -> dict:
|
||||||
"""Cheap item totals (no full fetch) for the progress bar."""
|
"""Cheap item totals (no full fetch) for the progress bar."""
|
||||||
m = sum(int(getattr(s, "totalSize", 0) or 0) for s in self._sections("movie", self._movies_lib))
|
m = sum(int(getattr(s, "totalSize", 0) or 0) for s in self._scan_sections("movie", self._movies_lib))
|
||||||
sh = sum(int(getattr(s, "totalSize", 0) or 0) for s in self._sections("show", self._tv_lib))
|
sh = sum(int(getattr(s, "totalSize", 0) or 0) for s in self._scan_sections("show", self._tv_lib))
|
||||||
if incremental:
|
if incremental:
|
||||||
m, sh = min(m, 100), min(sh, 50)
|
m, sh = min(m, 100), min(sh, 50)
|
||||||
return {"movies": m, "shows": sh}
|
return {"movies": m, "shows": sh}
|
||||||
|
|
||||||
def iter_movies(self, incremental=False):
|
def iter_movies(self, incremental=False):
|
||||||
for section in self._sections("movie", self._movies_lib):
|
for section in self._scan_sections("movie", self._movies_lib):
|
||||||
items = section.search(sort="addedAt:desc", maxresults=100) if incremental else section.all()
|
items = section.search(sort="addedAt:desc", maxresults=100) if incremental else section.all()
|
||||||
for m in items:
|
for m in items:
|
||||||
try:
|
try:
|
||||||
|
|
@ -332,7 +338,7 @@ class PlexVideoSource:
|
||||||
logger.exception("Plex: skipping movie %s", getattr(m, "title", "?"))
|
logger.exception("Plex: skipping movie %s", getattr(m, "title", "?"))
|
||||||
|
|
||||||
def iter_shows(self, incremental=False):
|
def iter_shows(self, incremental=False):
|
||||||
for section in self._sections("show", self._tv_lib):
|
for section in self._scan_sections("show", self._tv_lib):
|
||||||
items = section.search(sort="addedAt:desc", maxresults=50) if incremental else section.all()
|
items = section.search(sort="addedAt:desc", maxresults=50) if incremental else section.all()
|
||||||
for sh in items:
|
for sh in items:
|
||||||
try:
|
try:
|
||||||
|
|
@ -345,7 +351,7 @@ class PlexVideoSource:
|
||||||
get indexed. (plexapi ``section.update()`` triggers the library scan.)"""
|
get indexed. (plexapi ``section.update()`` triggers the library scan.)"""
|
||||||
n = 0
|
n = 0
|
||||||
for kind, name in (("movie", self._movies_lib), ("show", self._tv_lib)):
|
for kind, name in (("movie", self._movies_lib), ("show", self._tv_lib)):
|
||||||
for s in self._sections(kind, name):
|
for s in self._scan_sections(kind, name):
|
||||||
try:
|
try:
|
||||||
s.update()
|
s.update()
|
||||||
n += 1
|
n += 1
|
||||||
|
|
@ -508,6 +514,12 @@ class JellyfinVideoSource:
|
||||||
views = [v for v in views if v.get("Name") == name]
|
views = [v for v in views if v.get("Name") == name]
|
||||||
return views
|
return views
|
||||||
|
|
||||||
|
def _scan_views(self, collection_type: str, name):
|
||||||
|
"""Views to SCAN. An empty name means this kind isn't mapped → scan NOTHING
|
||||||
|
(never fall back to all views), so a missing selection can't pull every
|
||||||
|
library. (available_libraries still lists all via _views.)"""
|
||||||
|
return self._views(collection_type, name) if name else []
|
||||||
|
|
||||||
def available_libraries(self) -> dict:
|
def available_libraries(self) -> dict:
|
||||||
return {
|
return {
|
||||||
"movies": [{"title": v.get("Name")} for v in self._views("movies")],
|
"movies": [{"title": v.get("Name")} for v in self._views("movies")],
|
||||||
|
|
@ -522,7 +534,7 @@ class JellyfinVideoSource:
|
||||||
if not base:
|
if not base:
|
||||||
return {"ok": False, "sections": 0}
|
return {"ok": False, "sections": 0}
|
||||||
headers = {"X-Emby-Token": self._c.api_key or ""}
|
headers = {"X-Emby-Token": self._c.api_key or ""}
|
||||||
views = list(self._views("movies", self._movies_lib)) + list(self._views("tvshows", self._tv_lib))
|
views = list(self._scan_views("movies", self._movies_lib)) + list(self._scan_views("tvshows", self._tv_lib))
|
||||||
n = 0
|
n = 0
|
||||||
for v in views:
|
for v in views:
|
||||||
vid = v.get("Id")
|
vid = v.get("Id")
|
||||||
|
|
@ -543,8 +555,8 @@ class JellyfinVideoSource:
|
||||||
"ParentId": view["Id"], "IncludeItemTypes": itype,
|
"ParentId": view["Id"], "IncludeItemTypes": itype,
|
||||||
"Recursive": "true", "Limit": "0"}) or {}
|
"Recursive": "true", "Limit": "0"}) or {}
|
||||||
return int(resp.get("TotalRecordCount", 0) or 0)
|
return int(resp.get("TotalRecordCount", 0) or 0)
|
||||||
m = sum(total(v, "Movie") for v in self._views("movies", self._movies_lib))
|
m = sum(total(v, "Movie") for v in self._scan_views("movies", self._movies_lib))
|
||||||
sh = sum(total(v, "Series") for v in self._views("tvshows", self._tv_lib))
|
sh = sum(total(v, "Series") for v in self._scan_views("tvshows", self._tv_lib))
|
||||||
if incremental:
|
if incremental:
|
||||||
m, sh = min(m, 100), min(sh, 50)
|
m, sh = min(m, 100), min(sh, 50)
|
||||||
return {"movies": m, "shows": sh}
|
return {"movies": m, "shows": sh}
|
||||||
|
|
@ -589,7 +601,7 @@ class JellyfinVideoSource:
|
||||||
|
|
||||||
def iter_movies(self, incremental=False):
|
def iter_movies(self, incremental=False):
|
||||||
path = f"/Users/{self.uid}/Items"
|
path = f"/Users/{self.uid}/Items"
|
||||||
for view in self._views("movies", self._movies_lib):
|
for view in self._scan_views("movies", self._movies_lib):
|
||||||
params = {"ParentId": view["Id"], "IncludeItemTypes": "Movie",
|
params = {"ParentId": view["Id"], "IncludeItemTypes": "Movie",
|
||||||
"Recursive": "true", "Fields": _JF_MOVIE_FIELDS}
|
"Recursive": "true", "Fields": _JF_MOVIE_FIELDS}
|
||||||
if incremental:
|
if incremental:
|
||||||
|
|
@ -631,7 +643,7 @@ class JellyfinVideoSource:
|
||||||
|
|
||||||
def iter_shows(self, incremental=False):
|
def iter_shows(self, incremental=False):
|
||||||
path = f"/Users/{self.uid}/Items"
|
path = f"/Users/{self.uid}/Items"
|
||||||
for view in self._views("tvshows", self._tv_lib):
|
for view in self._scan_views("tvshows", self._tv_lib):
|
||||||
params = {"ParentId": view["Id"], "IncludeItemTypes": "Series",
|
params = {"ParentId": view["Id"], "IncludeItemTypes": "Series",
|
||||||
"Recursive": "true", "Fields": _JF_SHOW_FIELDS}
|
"Recursive": "true", "Fields": _JF_SHOW_FIELDS}
|
||||||
if incremental:
|
if incremental:
|
||||||
|
|
|
||||||
103
tests/test_video_scan_scope.py
Normal file
103
tests/test_video_scan_scope.py
Normal file
|
|
@ -0,0 +1,103 @@
|
||||||
|
"""Video scan SCOPE + worker-pause coupling.
|
||||||
|
|
||||||
|
1. The scan must read ONLY the user-mapped Movies/TV libraries — and when a kind
|
||||||
|
isn't mapped, scan NOTHING (never silently fall back to all libraries, which is
|
||||||
|
how YouTube/4K libraries leaked in).
|
||||||
|
2. A library scan (any mode) pauses EVERY enricher — including the YouTube date
|
||||||
|
enricher, which is a separate singleton outside engine.workers.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from core.video.sources import PlexVideoSource
|
||||||
|
from database.video_database import VideoDatabase
|
||||||
|
from core.video.enrichment.engine import VideoEnrichmentEngine
|
||||||
|
import core.video.youtube_enrichment as yt_mod
|
||||||
|
|
||||||
|
|
||||||
|
# ── 1. scan scope ──────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
class _Sec:
|
||||||
|
def __init__(self, type_, title):
|
||||||
|
self.type = type_
|
||||||
|
self.title = title
|
||||||
|
|
||||||
|
|
||||||
|
class _Lib:
|
||||||
|
def __init__(self, secs):
|
||||||
|
self._secs = secs
|
||||||
|
|
||||||
|
def sections(self):
|
||||||
|
return self._secs
|
||||||
|
|
||||||
|
|
||||||
|
class _Server:
|
||||||
|
def __init__(self, secs):
|
||||||
|
self.library = _Lib(secs)
|
||||||
|
|
||||||
|
|
||||||
|
_SECTIONS = [_Sec("movie", "Movies"), _Sec("movie", "4K Movies"),
|
||||||
|
_Sec("show", "TV Shows"), _Sec("show", "YouTube")]
|
||||||
|
|
||||||
|
|
||||||
|
def test_scan_uses_only_the_mapped_libraries():
|
||||||
|
src = PlexVideoSource(_Server(_SECTIONS), movies_lib="Movies", tv_lib="TV Shows")
|
||||||
|
assert [s.title for s in src._scan_sections("movie", src._movies_lib)] == ["Movies"]
|
||||||
|
assert [s.title for s in src._scan_sections("show", src._tv_lib)] == ["TV Shows"]
|
||||||
|
# the 4K + YouTube libraries are NOT scanned…
|
||||||
|
titles = [s.title for s in src._scan_sections("movie", src._movies_lib)] + \
|
||||||
|
[s.title for s in src._scan_sections("show", src._tv_lib)]
|
||||||
|
assert "4K Movies" not in titles and "YouTube" not in titles
|
||||||
|
# …but available_libraries still LISTS them all (for the Settings dropdown).
|
||||||
|
avail = src.available_libraries()
|
||||||
|
assert {x["title"] for x in avail["movies"]} == {"Movies", "4K Movies"}
|
||||||
|
assert {x["title"] for x in avail["tv"]} == {"TV Shows", "YouTube"}
|
||||||
|
|
||||||
|
|
||||||
|
def test_unmapped_kind_scans_nothing_not_everything():
|
||||||
|
# The hardening: a missing selection must scan NOTHING, never fall back to all.
|
||||||
|
src = PlexVideoSource(_Server(_SECTIONS), movies_lib=None, tv_lib=None)
|
||||||
|
assert src._scan_sections("movie", src._movies_lib) == []
|
||||||
|
assert src._scan_sections("show", src._tv_lib) == []
|
||||||
|
# _sections (used for LISTING) still returns all — only the SCAN path is gated.
|
||||||
|
assert len(src._sections("movie")) == 2
|
||||||
|
|
||||||
|
|
||||||
|
# ── 2. scan pauses every enricher (incl. the YouTube date singleton) ───────
|
||||||
|
|
||||||
|
class _FakeYT:
|
||||||
|
def __init__(self, paused=False):
|
||||||
|
self._paused = paused
|
||||||
|
|
||||||
|
def pause(self):
|
||||||
|
self._paused = True
|
||||||
|
|
||||||
|
def resume(self):
|
||||||
|
self._paused = False
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture()
|
||||||
|
def engine(tmp_path):
|
||||||
|
db = VideoDatabase(database_path=str(tmp_path / "video_library.db"))
|
||||||
|
return VideoEnrichmentEngine(db, clients={})
|
||||||
|
|
||||||
|
|
||||||
|
def test_scan_pauses_and_resumes_youtube_date_enricher(engine, monkeypatch):
|
||||||
|
fake = _FakeYT()
|
||||||
|
monkeypatch.setattr(yt_mod, "get_youtube_date_enricher", lambda: fake)
|
||||||
|
engine.pause_for_scan()
|
||||||
|
assert fake._paused is True
|
||||||
|
assert "youtube" in engine._scan_paused
|
||||||
|
engine.resume_after_scan()
|
||||||
|
assert fake._paused is False
|
||||||
|
|
||||||
|
|
||||||
|
def test_scan_never_resumes_a_manually_paused_youtube_enricher(engine, monkeypatch):
|
||||||
|
fake = _FakeYT(paused=True) # the user paused it themselves
|
||||||
|
monkeypatch.setattr(yt_mod, "get_youtube_date_enricher", lambda: fake)
|
||||||
|
engine.pause_for_scan()
|
||||||
|
assert "youtube" not in engine._scan_paused # we didn't touch it
|
||||||
|
engine.resume_after_scan()
|
||||||
|
assert fake._paused is True # still paused after the scan
|
||||||
Loading…
Reference in a new issue