From 57f254acaf87f8931ea9e92700e7e66e732d7da8 Mon Sep 17 00:00:00 2001 From: BoulderBadgeDad Date: Sat, 20 Jun 2026 12:33:45 -0700 Subject: [PATCH] video enrichment: background TMDB details backfill (fills 'status' on pre-matched items) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The media server pre-matches shows/movies (tmdb_id set), so the enrichment matcher skips them and never fetches TMDB details — leaving details-only fields like `status` (airing vs ended) blank on almost the whole library. That's why the watchlist's airing-shows default only ever saw the handful of shows whose detail page had been opened (the one path that force-fetches details). Library here: 3,371 matched shows, only 18 with status. Fix: a one-time-per-item details backfill that runs in the enrichment worker's idle loop (after the episode-sync pass). New `details_synced` marker column on shows+movies; detail_backfill_next/mark_details_synced/pending_count; worker._detail_backfill_one() re-fetches an already-matched item's TMDB details and gap-fills (never clobbers server data), then marks it done so it's attempted once. No re-scan needed — it heals the existing library in place, and once status is populated the airing-watchlist reflects real TV. It's a background gap-fill on already-matched items (like episode coverage), so it doesn't block the worker's 'Complete' status. kettui: DB seam tests + worker tests (fills status / enrich-by-id / marks-done-when-absent). ruff + isolation guards green; 94 enrichment tests pass. --- core/video/enrichment/worker.py | 39 +++++++++++++++++++++-- database/video_database.py | 55 +++++++++++++++++++++++++++++++++ tests/test_video_enrichment.py | 47 ++++++++++++++++++++++++++++ 3 files changed, 138 insertions(+), 3 deletions(-) diff --git a/core/video/enrichment/worker.py b/core/video/enrichment/worker.py index 63c4af82..f44e513c 100644 --- a/core/video/enrichment/worker.py +++ b/core/video/enrichment/worker.py @@ -118,10 +118,13 @@ class VideoEnrichmentWorker: try: priority = self.db.get_setting("enrichment_priority") or None except Exception: - pass + logger.debug("enrichment priority lookup failed", exc_info=True) item = self.db.enrichment_next(self.service, self.retry_days, priority=priority) if not item: - return self._sync_episodes_once() + # No pending matches → use idle time for the full episode-list sync + # (which also fills details for shows it touches), then the details + # backfill for everything already episode-synced but missing `status`. + return self._sync_episodes_once() or self._detail_backfill_one() self.current_item = {"type": item["kind"], "name": item["title"]} try: # Prefer the provider id the server already gave us (enrich BY ID, no @@ -240,6 +243,33 @@ class VideoEnrichmentWorker: self.db.mark_episodes_synced(show["id"]) # move on (never loop on one show) return True + def _detail_backfill_one(self) -> bool: + """Background TMDB details backfill: re-fetch ONE already-matched show/movie + that's missing details-only fields (status, network, tagline, rating…) and + gap-fill them. The matcher skips server-pre-matched items, so without this + their `status` stays blank — and the watchlist's airing-default can't see + them. TMDB-only. Returns True if it did work (so the loop rate-limits).""" + if not hasattr(self.client, "match"): + return False + item = self.db.detail_backfill_next("show") or self.db.detail_backfill_next("movie") + if not item: + return False + self.current_item = {"type": item["kind"], "name": item["title"]} + try: + result = self.client.match(item["kind"], item["title"], item.get("year"), + known_id=item.get("tmdb_id")) + if result and result.get("id"): + # Gap-fill only (never clobbers server data); fills status et al. + self.db.enrichment_apply("tmdb", item["kind"], item["id"], matched=True, + external_id=result["id"], metadata=result.get("metadata")) + logger.info("Backfilled details for %s '%s'", item["kind"], item["title"]) + self.db.mark_details_synced(item["kind"], item["id"]) # attempted once → don't re-pick + except Exception: + # Transient call failure — leave details_synced=0 so it retries later. + logger.exception("detail backfill failed for %s '%s'", item["kind"], item["title"]) + self.stats["errors"] += 1 + return True + def _cascade_episodes(self, show_id, tv_id, season_numbers=None) -> None: """Backfill a show's FULL episode list from the provider (one call per season) — owned + missing. Best-effort: a season failure never aborts the @@ -278,7 +308,10 @@ class VideoEnrichmentWorker: try: pending += self.db.episode_sync_pending_count() except Exception: - pass + logger.debug("episode_sync_pending_count failed", exc_info=True) + # NB: the TMDB details backfill (status/network/…) is a background gap-fill on + # ALREADY-matched items — like episode coverage, it doesn't block "Complete". + # It still runs in the idle loop; it's just not counted as blocking pending. cooling = self._cooldown_until > time.monotonic() running = self.running and not self.paused and self.enabled and not cooling idle = running and pending == 0 and self.current_item is None diff --git a/database/video_database.py b/database/video_database.py index a690fac7..58577776 100644 --- a/database/video_database.py +++ b/database/video_database.py @@ -201,6 +201,12 @@ _COLUMN_MIGRATIONS = [ ("youtube_video_stats", "dearrow_title", "TEXT"), ("youtube_video_stats", "dearrow_status", "TEXT"), ("youtube_video_stats", "dearrow_attempted", "TEXT"), + # TMDB details backfill: the server pre-matches shows/movies (so the matcher + # skips them) but never supplies details-only fields like `status` (airing vs + # ended) — which the watchlist's airing-default depends on. This marker drives a + # one-time per-item detail re-fetch that fills those gaps. Starts 0 = needs it. + ("shows", "details_synced", "INTEGER NOT NULL DEFAULT 0"), + ("movies", "details_synced", "INTEGER NOT NULL DEFAULT 0"), ] @@ -865,6 +871,55 @@ class VideoDatabase: finally: conn.close() + # ── TMDB details backfill (status / network / tagline / rating …) ────────── + # The media server pre-matches items (tmdb_id set), so the matcher skips them and + # never fetches details-only fields. This one-time pass re-fetches details for a + # matched item and gap-fills, then marks it done so it isn't re-picked. + _DETAIL_TBL = {"show": "shows", "movie": "movies"} + + def detail_backfill_next(self, kind: str) -> dict | None: + """Next matched show/movie (has tmdb_id) whose details haven't been + backfilled yet. Returns {kind, id, title, year, tmdb_id} or None.""" + tbl = self._DETAIL_TBL.get(kind) + if not tbl: + return None + conn = self._get_connection() + try: + row = conn.execute( + f"SELECT id, title, year, tmdb_id FROM {tbl} " + f"WHERE tmdb_id IS NOT NULL AND details_synced=0 ORDER BY id LIMIT 1").fetchone() + if not row: + return None + d = dict(row) + d["kind"] = kind + return d + finally: + conn.close() + + def mark_details_synced(self, kind: str, item_id: int) -> None: + """Flag that an item's TMDB details were backfilled (attempted once), so the + background pass doesn't re-pick it even if a field stayed empty.""" + tbl = self._DETAIL_TBL.get(kind) + if not tbl: + return + conn = self._get_connection() + try: + conn.execute(f"UPDATE {tbl} SET details_synced=1 WHERE id=?", (item_id,)) + conn.commit() + finally: + conn.close() + + def detail_backfill_pending_count(self) -> int: + conn = self._get_connection() + try: + n = 0 + for tbl in ("shows", "movies"): + n += conn.execute( + f"SELECT COUNT(*) FROM {tbl} WHERE tmdb_id IS NOT NULL AND details_synced=0").fetchone()[0] + return n + finally: + conn.close() + def _ratings_breakdown(self) -> dict: """OMDb 'coverage' breakdown: matched = ratings present, pending = has an imdb_id but not fetched, not_found = fetched but OMDb had no rating.""" diff --git a/tests/test_video_enrichment.py b/tests/test_video_enrichment.py index 1c27a13e..bdc626b4 100644 --- a/tests/test_video_enrichment.py +++ b/tests/test_video_enrichment.py @@ -160,6 +160,53 @@ def test_worker_background_episode_sync_pulls_full_list(db): assert db.episode_sync_pending_count() == 0 +def test_detail_backfill_next_only_matched_unsynced(db): + # Matched items (have tmdb_id) that haven't had details backfilled are queued; + # un-matched ones are skipped. Shows + movies are independent queues. + a = db.upsert_show_tree("plex", {"server_id": "s1", "title": "A", "tmdb_id": 1, "seasons": []}) + db.upsert_show_tree("plex", {"server_id": "s2", "title": "B", "seasons": []}) # no tmdb_id → skip + nx = db.detail_backfill_next("show") + assert nx and nx["id"] == a and nx["kind"] == "show" and nx["tmdb_id"] == 1 + db.mark_details_synced("show", a) + assert db.detail_backfill_next("show") is None + mid = db.upsert_movie("plex", {"server_id": "m1", "title": "M", "tmdb_id": 9}) + assert db.detail_backfill_next("movie")["id"] == mid + db.mark_details_synced("movie", mid) + assert db.detail_backfill_next("movie") is None + assert db.detail_backfill_pending_count() == 0 + + +def test_worker_detail_backfill_fills_status(db): + # The real bug: a show pre-matched by the server (tmdb_id set) + already + # episode-synced, but `status` never captured (server doesn't supply it, the + # matcher skips matched rows). With the match + episode queues clear, the worker + # re-fetches details and gap-fills `status` — what the airing-watchlist needs. + sid = db.upsert_show_tree("plex", {"server_id": "s1", "title": "S", "tmdb_id": 1396, "seasons": []}) + db.enrichment_apply("tmdb", "show", sid, matched=True, external_id=1396) + db.mark_episodes_synced(sid) # episode queue clear → detail backfill runs + assert db.detail_backfill_pending_count() == 1 + client = FakeClient({"id": 1396, "metadata": {"status": "Returning Series", "network": "HBO"}}) + w = VideoEnrichmentWorker(db, "tmdb", client) + assert w.process_one() is True # no match/episode work → detail backfill + assert client.calls == [("show", "S", None, 1396)] # enrich BY id, no re-search + with db.connect() as c: + row = c.execute("SELECT status, details_synced FROM shows WHERE id=?", (sid,)).fetchone() + assert row["status"] == "Returning Series" + assert row["details_synced"] == 1 + assert db.detail_backfill_pending_count() == 0 # done → not re-picked + + +def test_detail_backfill_marks_done_even_when_status_absent(db): + # If TMDB returns no status, we still mark it attempted so it isn't re-fetched + # forever (bounded: one attempt per item). + sid = db.upsert_show_tree("plex", {"server_id": "s1", "title": "S", "tmdb_id": 5, "seasons": []}) + db.enrichment_apply("tmdb", "show", sid, matched=True, external_id=5) + db.mark_episodes_synced(sid) + w = VideoEnrichmentWorker(db, "tmdb", FakeClient({"id": 5, "metadata": {}})) + assert w.process_one() is True + assert db.detail_backfill_pending_count() == 0 + + def test_show_match_info(db): sid = db.upsert_show_tree("plex", {"server_id": "s1", "title": "S", "year": 2019, "tmdb_id": 1396, "seasons": []})