video enrichment: cascade episode backfill from the TMDB show worker
Episodes ride along with their show instead of being a separate (tens-of-thousands) queue: when the TMDB worker matches a show, it now backfills every season's episodes — still / overview / rating — via /tv/<id>/season/<n> (one call per season, gap-only so server data is never clobbered). Also backfills season overviews. The worker manager 'knows about it': the TMDB breakdown gains an Episodes coverage entry (matched = has art, rest = pending), shown as its own card; the Episodes view lists episodes still missing art. It's coverage-only, kept out of the worker's idle/pending calc so it never blocks 'Complete'. Seam tests: client season parse, worker cascade fills episodes, gap-only backfill + season overview, breakdown coverage (tmdb only), missing-art list, idle calc ignores episode coverage.
This commit is contained in:
parent
878e467f69
commit
80f1051e8a
6 changed files with 212 additions and 4 deletions
|
|
@ -115,6 +115,29 @@ class TMDBClient:
|
|||
logger.exception("TMDB details fetch failed for %s", title or tmdb_id)
|
||||
return {"id": tmdb_id, "metadata": {k: v for k, v in meta.items() if v}}
|
||||
|
||||
def season_episodes(self, tv_id, season_number):
|
||||
"""Episode-level data for one season (still/overview/rating) — the show
|
||||
worker cascades over a show's seasons to backfill episodes the media
|
||||
server lacked. Returns {'overview', 'episodes': [...]} or None."""
|
||||
if not self.api_key or tv_id is None or season_number is None:
|
||||
return None
|
||||
import requests
|
||||
r = requests.get(self.BASE + "/tv/" + str(tv_id) + "/season/" + str(season_number),
|
||||
params={"api_key": self.api_key}, timeout=15)
|
||||
r.raise_for_status()
|
||||
data = r.json() or {}
|
||||
out = []
|
||||
for e in (data.get("episodes") or []):
|
||||
en = e.get("episode_number")
|
||||
if en is None:
|
||||
continue
|
||||
ep = {"episode_number": en, "overview": e.get("overview"),
|
||||
"rating": e.get("vote_average") or None}
|
||||
if e.get("still_path"):
|
||||
ep["still_url"] = self.IMG + e["still_path"]
|
||||
out.append(ep)
|
||||
return {"overview": data.get("overview"), "episodes": out}
|
||||
|
||||
|
||||
class TVDBClient:
|
||||
BASE = "https://api4.thetvdb.com/v4"
|
||||
|
|
|
|||
|
|
@ -123,18 +123,41 @@ class VideoEnrichmentWorker:
|
|||
logger.info("Matched %s '%s' -> %s ID: %s%s", item["kind"], item["title"],
|
||||
self.display_name, result["id"],
|
||||
" (by server id)" if item.get("known_id") else "")
|
||||
# Cascade: a matched show backfills its episodes' art/overview/rating
|
||||
# from the same provider (one call per season), so episodes ride along
|
||||
# with their show instead of being a separate (huge) queue.
|
||||
if item["kind"] == "show" and hasattr(self.client, "season_episodes"):
|
||||
self._cascade_episodes(item["id"], result["id"])
|
||||
else:
|
||||
self.db.enrichment_apply(self.service, item["kind"], item["id"], matched=False)
|
||||
self.stats["not_found"] += 1
|
||||
logger.info("No %s match for %s '%s'", self.display_name, item["kind"], item["title"])
|
||||
return True
|
||||
|
||||
def _cascade_episodes(self, show_id, tv_id) -> None:
|
||||
"""Backfill a show's episodes from the provider (one call per season).
|
||||
Best-effort: a season failure never aborts the show's enrichment."""
|
||||
try:
|
||||
seasons = self.db.show_season_numbers(show_id)
|
||||
except Exception:
|
||||
logger.exception("episode backfill: season list failed for show %s", show_id)
|
||||
return
|
||||
for snum in seasons:
|
||||
try:
|
||||
data = self.client.season_episodes(tv_id, snum)
|
||||
if data and data.get("episodes"):
|
||||
self.db.backfill_episodes(show_id, snum, data["episodes"], data.get("overview"))
|
||||
except Exception:
|
||||
logger.exception("episode backfill failed: show %s season %s", show_id, snum)
|
||||
|
||||
# ── status (same shape the music enrichment API returns) ──────────────────
|
||||
def get_stats(self) -> dict:
|
||||
breakdown = self.db.enrichment_breakdown(self.service)
|
||||
# Errored items are outstanding (retried later), so they count as pending
|
||||
# work — the worker isn't "Complete" while any remain.
|
||||
pending = sum(b["pending"] + b.get("errors", 0) for b in breakdown.values())
|
||||
# work — the worker isn't "Complete" while any remain. Episode art is a
|
||||
# coverage-only cascade (no queue), so it's excluded from idle/pending.
|
||||
pending = sum(b["pending"] + b.get("errors", 0)
|
||||
for b in breakdown.values() if not b.get("coverage_only"))
|
||||
running = self.running and not self.paused and self.enabled
|
||||
idle = running and pending == 0 and self.current_item is None
|
||||
progress = {}
|
||||
|
|
|
|||
|
|
@ -277,12 +277,67 @@ class VideoDatabase:
|
|||
"errors": conn.execute(f"SELECT COUNT(*) FROM {tbl} WHERE {sc}='error'").fetchone()[0],
|
||||
"pending": conn.execute(f"SELECT COUNT(*) FROM {tbl} WHERE {sc} IS NULL").fetchone()[0],
|
||||
}
|
||||
# TMDB also cascades episode art (still) backfill from the show worker,
|
||||
# so the manager sees episode coverage. Not a queue (matched = has a
|
||||
# still; the rest are "pending" art) — kept out of the idle/pending
|
||||
# calc by the worker so it never blocks "Complete".
|
||||
if service == "tmdb":
|
||||
total = conn.execute("SELECT COUNT(*) FROM episodes").fetchone()[0]
|
||||
with_still = conn.execute(
|
||||
"SELECT COUNT(*) FROM episodes WHERE still_url IS NOT NULL AND still_url<>''").fetchone()[0]
|
||||
out["episode"] = {"matched": with_still, "not_found": 0, "errors": 0,
|
||||
"pending": total - with_still, "coverage_only": True}
|
||||
return out
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
def show_season_numbers(self, show_id: int) -> list:
|
||||
conn = self._get_connection()
|
||||
try:
|
||||
return [r["season_number"] for r in conn.execute(
|
||||
"SELECT season_number FROM seasons WHERE show_id=? ORDER BY season_number",
|
||||
(show_id,)).fetchall()]
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
def backfill_episodes(self, show_id: int, season_number: int, episodes: list,
|
||||
season_overview: str | None = None) -> int:
|
||||
"""Gap-only backfill of episode still/overview/rating (and the season's
|
||||
overview) from enrichment — never clobbers what the server provided.
|
||||
Returns the number of episode rows touched."""
|
||||
conn = self._get_connection()
|
||||
touched = 0
|
||||
try:
|
||||
for e in (episodes or []):
|
||||
en = e.get("episode_number")
|
||||
if en is None:
|
||||
continue
|
||||
sets, params = [], []
|
||||
for col in ("still_url", "overview", "rating"):
|
||||
val = e.get(col)
|
||||
if val is None:
|
||||
continue
|
||||
sets.append(f"{col}=COALESCE(NULLIF({col}, ''), ?)")
|
||||
params.append(val)
|
||||
if not sets:
|
||||
continue
|
||||
params += [show_id, season_number, en]
|
||||
cur = conn.execute(
|
||||
f"UPDATE episodes SET {', '.join(sets)} "
|
||||
"WHERE show_id=? AND season_number=? AND episode_number=?", params)
|
||||
touched += cur.rowcount
|
||||
if season_overview:
|
||||
conn.execute("UPDATE seasons SET overview=COALESCE(NULLIF(overview, ''), ?) "
|
||||
"WHERE show_id=? AND season_number=?", (season_overview, show_id, season_number))
|
||||
conn.commit()
|
||||
return touched
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
def enrichment_unmatched(self, service: str, kind: str, status: str = "not_found",
|
||||
search=None, limit: int = 50, offset: int = 0) -> dict:
|
||||
if kind == "episode" and service == "tmdb":
|
||||
return self._episodes_missing_art(search, limit, offset)
|
||||
spec = _ENRICH.get(service, {}).get(kind)
|
||||
if not spec:
|
||||
return {"items": [], "total": 0}
|
||||
|
|
@ -315,6 +370,31 @@ class VideoDatabase:
|
|||
finally:
|
||||
conn.close()
|
||||
|
||||
def _episodes_missing_art(self, search, limit, offset) -> dict:
|
||||
"""Episodes still lacking a still image (for the manager's Episodes view).
|
||||
Read-only: episode art is backfilled as a cascade, not a retry queue."""
|
||||
where = ["(e.still_url IS NULL OR e.still_url='')"]
|
||||
params: list = []
|
||||
if search:
|
||||
where.append("(e.title LIKE ? COLLATE NOCASE OR sh.title LIKE ? COLLATE NOCASE)")
|
||||
params += ["%" + search + "%", "%" + search + "%"]
|
||||
where_sql = " WHERE " + " AND ".join(where)
|
||||
conn = self._get_connection()
|
||||
try:
|
||||
total = conn.execute(
|
||||
f"SELECT COUNT(*) FROM episodes e JOIN shows sh ON sh.id=e.show_id{where_sql}",
|
||||
params).fetchone()[0]
|
||||
rows = conn.execute(
|
||||
"SELECT e.id, sh.title || ' · S' || e.season_number || 'E' || e.episode_number "
|
||||
"|| COALESCE(' · ' || e.title, '') AS title, e.air_date AS year, "
|
||||
"0 AS has_poster, NULL AS last_attempted "
|
||||
f"FROM episodes e JOIN shows sh ON sh.id=e.show_id{where_sql} "
|
||||
"ORDER BY sh.sort_title, e.season_number, e.episode_number LIMIT ? OFFSET ?",
|
||||
params + [limit, offset]).fetchall()
|
||||
return {"items": [dict(r) | {"has_poster": False} for r in rows], "total": total}
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
def enrichment_retry(self, service: str, kind: str, scope: str = "failed", item_id=None) -> int:
|
||||
"""Re-queue items by resetting status/last_attempted to NULL."""
|
||||
spec = _ENRICH.get(service, {}).get(kind)
|
||||
|
|
|
|||
|
|
@ -558,6 +558,43 @@ def test_enrichment_backfills_season_posters_only_when_missing(db):
|
|||
assert db.show_detail(sid)["seasons"][0]["has_poster"] is True
|
||||
|
||||
|
||||
def test_backfill_episodes_gap_only_and_season_overview(db):
|
||||
sid = db.upsert_show_tree("plex", {"server_id": "s1", "title": "S", "seasons": [
|
||||
{"season_number": 1, "episodes": [
|
||||
{"episode_number": 1, "overview": "server overview"}, # has overview already
|
||||
{"episode_number": 2}]}]}) # bare
|
||||
n = db.backfill_episodes(sid, 1, [
|
||||
{"episode_number": 1, "still_url": "/e1.jpg", "overview": "tmdb overview", "rating": 8.0},
|
||||
{"episode_number": 2, "still_url": "/e2.jpg", "overview": "tmdb e2", "rating": 7.0},
|
||||
], season_overview="Season one")
|
||||
assert n == 2
|
||||
eps = {e["episode_number"]: e for e in db.show_detail(sid)["seasons"][0]["episodes"]}
|
||||
assert eps[1]["has_still"] is True and eps[1]["overview"] == "server overview" # overview kept
|
||||
assert eps[2]["has_still"] is True and eps[2]["overview"] == "tmdb e2" # gap filled
|
||||
with db.connect() as c:
|
||||
assert c.execute("SELECT overview FROM seasons WHERE show_id=? AND season_number=1",
|
||||
(sid,)).fetchone()["overview"] == "Season one"
|
||||
|
||||
|
||||
def test_breakdown_reports_episode_art_coverage_for_tmdb(db):
|
||||
db.upsert_show_tree("plex", {"server_id": "s1", "title": "S", "seasons": [
|
||||
{"season_number": 1, "episodes": [{"episode_number": 1, "still_url": "/e1.jpg"},
|
||||
{"episode_number": 2}]}]})
|
||||
bd = db.enrichment_breakdown("tmdb")
|
||||
assert bd["episode"]["matched"] == 1 and bd["episode"]["pending"] == 1
|
||||
assert bd["episode"].get("coverage_only") is True
|
||||
# TVDB doesn't cascade episodes → no episode coverage entry.
|
||||
assert "episode" not in db.enrichment_breakdown("tvdb")
|
||||
|
||||
|
||||
def test_unmatched_lists_episodes_missing_art(db):
|
||||
db.upsert_show_tree("plex", {"server_id": "s1", "title": "Show", "seasons": [
|
||||
{"season_number": 1, "episodes": [{"episode_number": 1, "title": "Has", "still_url": "/e1.jpg"},
|
||||
{"episode_number": 2, "title": "Missing"}]}]})
|
||||
res = db.enrichment_unmatched("tmdb", "episode", status="unmatched")
|
||||
assert res["total"] == 1 and "Missing" in res["items"][0]["title"]
|
||||
|
||||
|
||||
def test_error_status_is_distinct_and_retryable_in_ui(db):
|
||||
a = db.upsert_movie("plex", {"server_id": "m1", "title": "A"})
|
||||
db.enrichment_apply("tmdb", "movie", a, matched=False, error=True)
|
||||
|
|
|
|||
|
|
@ -98,6 +98,51 @@ def test_tmdb_pulls_full_metadata(monkeypatch):
|
|||
assert m["genres"] == ["Sci-Fi", "Drama"] and m["status"] == "Released" and m["imdb_id"] == "tt1"
|
||||
|
||||
|
||||
def test_show_worker_cascades_episode_backfill(db):
|
||||
# A matched show backfills its episodes' art via the client's season_episodes
|
||||
# cascade (episodes ride along with their show — no separate queue).
|
||||
sid = db.upsert_show_tree("plex", {"server_id": "s1", "title": "S", "seasons": [
|
||||
{"season_number": 1, "episodes": [{"episode_number": 1}, {"episode_number": 2}]}]})
|
||||
|
||||
class CascadeClient:
|
||||
enabled = True
|
||||
def match(self, kind, title, year, known_id=None):
|
||||
return {"id": 1396, "metadata": {}}
|
||||
def season_episodes(self, tv_id, snum):
|
||||
assert tv_id == 1396
|
||||
return {"overview": "S%d" % snum, "episodes": [
|
||||
{"episode_number": 1, "still_url": "/e1.jpg", "overview": "O1", "rating": 8.0}]}
|
||||
|
||||
w = VideoEnrichmentWorker(db, "tmdb", CascadeClient())
|
||||
assert w.process_one() is True
|
||||
eps = {e["episode_number"]: e for e in db.show_detail(sid)["seasons"][0]["episodes"]}
|
||||
assert eps[1]["has_still"] is True and eps[2]["has_still"] is False # cascade filled E1
|
||||
|
||||
|
||||
def test_get_stats_excludes_episode_coverage_from_pending(db):
|
||||
sid = db.upsert_show_tree("plex", {"server_id": "s1", "title": "S", "seasons": [
|
||||
{"season_number": 1, "episodes": [{"episode_number": 1}]}]}) # episode has no still
|
||||
db.enrichment_apply("tmdb", "show", sid, matched=True, external_id=1) # show done
|
||||
stats = VideoEnrichmentWorker(db, "tmdb", FakeClient(None)).get_stats()
|
||||
assert "episode" in stats["breakdown"] # manager sees episode coverage
|
||||
assert stats["stats"]["pending"] == 0 # but it doesn't block "Complete"
|
||||
|
||||
|
||||
def test_tmdb_season_episodes_parses(monkeypatch):
|
||||
class _Resp:
|
||||
def __init__(self, b): self._b = b
|
||||
def raise_for_status(self): pass
|
||||
def json(self): return self._b
|
||||
body = {"overview": "Season 1", "episodes": [
|
||||
{"episode_number": 1, "still_path": "/a.jpg", "overview": "O", "vote_average": 8.1},
|
||||
{"episode_number": 2, "still_path": None, "overview": "P", "vote_average": 0}]}
|
||||
monkeypatch.setitem(sys.modules, "requests", types.SimpleNamespace(get=lambda u, **k: _Resp(body)))
|
||||
res = TMDBClient("KEY").season_episodes(1396, 1)
|
||||
assert res["overview"] == "Season 1" and len(res["episodes"]) == 2
|
||||
assert res["episodes"][0]["still_url"] == "https://image.tmdb.org/t/p/original/a.jpg"
|
||||
assert "still_url" not in res["episodes"][1] # no still_path → omitted
|
||||
|
||||
|
||||
def test_tmdb_show_returns_season_posters(monkeypatch):
|
||||
class _Resp:
|
||||
def __init__(self, b): self._b = b
|
||||
|
|
|
|||
|
|
@ -31,8 +31,8 @@
|
|||
tmdb: 'https://www.themoviedb.org/assets/2/v4/logos/v2/blue_square_2-d537fb228cf3ded904ef09b136fe3fec72548ebc1fea3fbbd1ad9e36364db38b.svg',
|
||||
tvdb: 'https://www.svgrepo.com/show/443500/brand-tvdb.svg',
|
||||
};
|
||||
var GLYPH = { movie: '🎬', show: '📺' };
|
||||
var KIND_LABEL = { movie: 'Movies', show: 'Shows' };
|
||||
var GLYPH = { movie: '🎬', show: '📺', episode: '🎞️' };
|
||||
var KIND_LABEL = { movie: 'Movies', show: 'Shows', episode: 'Episodes' };
|
||||
|
||||
var state = {
|
||||
open: false, selected: 'tmdb', statuses: {}, breakdown: null,
|
||||
|
|
|
|||
Loading…
Reference in a new issue