video scan: capture the provider IDs the server already has (tmdb/imdb/tvdb)

The servers already matched everything to their agents — we were dropping the
IDs. Now we store them:
- Plex: parse item.guids (imdb://, tmdb://, tvdb://); Jellyfin: parse
  item.ProviderIds (added ProviderIds to the requested Fields).
- Stored on movies (tmdb_id, imdb_id), shows (tvdb_id, tmdb_id, imdb_id), and
  episodes (tvdb_id) via the upserts.
- Dropped the over-strict UNIQUE on movies.tmdb_id / shows.tvdb_id (same title
  can legitimately live in two libraries; we dedupe on server_id). Scanner now
  wraps each upsert in try/except so one bad item can't abort a scan.
Tests: guid/ProviderIds parsing + IDs persisted. 38 video-DB/scanner tests green.

core/video/scanner.py

@@ -150,7 +150,11 @@ class VideoLibraryScanner:
                         break
                     continue
                 consec = 0
-                self.db.upsert_movie(server, item)
+                try:
+                    self.db.upsert_movie(server, item)
+                except Exception:
+                    logger.exception("video scan: skipping movie %s", sid)
+                    continue
                 seen_movies.add(sid)
                 movies += 1
                 processed += 1
@@ -178,7 +182,11 @@ class VideoLibraryScanner:
                         break
                     continue
                 consec = 0
-                self.db.upsert_show_tree(server, show)
+                try:
+                    self.db.upsert_show_tree(server, show)
+                except Exception:
+                    logger.exception("video scan: skipping show %s", sid)
+                    continue
                 seen_shows.add(sid)
                 shows += 1
                 episodes += sum(len(s.get("episodes", [])) for s in show.get("seasons", []))

core/video/sources.py

@@ -13,6 +13,8 @@ these adapters.
 
 from __future__ import annotations
 
+import re
+
 from utils.logging_config import get_logger
 
 logger = get_logger("video_sources")
@@ -22,6 +24,45 @@ logger = get_logger("video_sources")
 PLEX_SCAN_TIMEOUT = 120
 
 
+def _to_int(val):
+    if val is None:
+        return None
+    m = re.match(r"\d+", str(val))
+    return int(m.group()) if m else None
+
+
+def _parse_plex_guids(obj) -> dict:
+    """tmdb/imdb/tvdb ids from a Plex item's guids — Plex already matched them."""
+    out = {"tmdb_id": None, "imdb_id": None, "tvdb_id": None}
+    try:
+        for g in (getattr(obj, "guids", None) or []):
+            gid = getattr(g, "id", "") or ""
+            if "://" not in gid:
+                continue
+            scheme, value = gid.split("://", 1)
+            scheme = scheme.lower()
+            if scheme == "imdb":
+                out["imdb_id"] = (value.split("?")[0] or None)
+            elif scheme == "tmdb":
+                out["tmdb_id"] = _to_int(value)
+            elif scheme == "tvdb":
+                out["tvdb_id"] = _to_int(value)
+    except Exception:
+        pass
+    return out
+
+
+def _parse_jf_providers(item) -> dict:
+    """tmdb/imdb/tvdb ids from a Jellyfin item's ProviderIds."""
+    providers = item.get("ProviderIds") or {}
+    low = {(k or "").lower(): v for k, v in providers.items()}
+    return {
+        "imdb_id": low.get("imdb") or None,
+        "tmdb_id": _to_int(low.get("tmdb")),
+        "tvdb_id": _to_int(low.get("tvdb")),
+    }
+
+
 def _build_source(movies_lib=None, tv_lib=None):
     """Build a media source for the active server, restricted to the named
     Movies/TV libraries when given. Reuses the SHARED connection config — but
@@ -156,7 +197,7 @@ class PlexVideoSource:
 
     def _movie(self, m) -> dict:
         dur = getattr(m, "duration", None)
-        return {
+        d = {
             "server_id": str(m.ratingKey),
             "title": m.title,
             "year": getattr(m, "year", None),
@@ -167,6 +208,8 @@ class PlexVideoSource:
             "runtime_minutes": int(dur / 60000) if dur else None,
             "file": self._part_file(m),
         }
+        d.update(_parse_plex_guids(m))
+        return d
 
     def _episode(self, ep, snum, enum) -> dict:
         dur = getattr(ep, "duration", None)
@@ -179,6 +222,7 @@ class PlexVideoSource:
             "overview": getattr(ep, "summary", None),
             "air_date": aired.date().isoformat() if aired else None,
             "runtime_minutes": int(dur / 60000) if dur else None,
+            "tvdb_id": _parse_plex_guids(ep).get("tvdb_id"),
             "file": self._part_file(ep),
         }
 
@@ -199,7 +243,7 @@ class PlexVideoSource:
         seasons = [{"server_id": None, "season_number": n, "title": None,
                     "overview": None, "poster_url": None, "episodes": eps}
                    for n, eps in sorted(seasons_map.items())]
-        return {
+        d = {
             "server_id": str(sh.ratingKey),
             "title": sh.title,
             "year": getattr(sh, "year", None),
@@ -210,11 +254,13 @@ class PlexVideoSource:
             "content_rating": getattr(sh, "contentRating", None),
             "seasons": seasons,
         }
+        d.update(_parse_plex_guids(sh))
+        return d
 
 
 # ── Jellyfin ────────────────────────────────────────────────────────────────
-_JF_MOVIE_FIELDS = "Overview,Path,MediaSources,ProductionYear,OfficialRating,RunTimeTicks,Studios"
+_JF_MOVIE_FIELDS = "Overview,Path,MediaSources,ProductionYear,OfficialRating,RunTimeTicks,Studios,ProviderIds"
-_JF_EP_FIELDS = "Overview,Path,MediaSources,PremiereDate,RunTimeTicks,IndexNumber,ParentIndexNumber"
+_JF_EP_FIELDS = "Overview,Path,MediaSources,PremiereDate,RunTimeTicks,IndexNumber,ParentIndexNumber,ProviderIds"
 
 
 class JellyfinVideoSource:
@@ -312,7 +358,7 @@ class JellyfinVideoSource:
     def _movie(self, it) -> dict:
         studios = it.get("Studios") or []
         ticks = it.get("RunTimeTicks")
-        return {
+        d = {
             "server_id": str(it["Id"]),
             "title": it.get("Name"),
             "year": it.get("ProductionYear"),
@@ -323,12 +369,14 @@ class JellyfinVideoSource:
             "runtime_minutes": int(ticks / 600_000_000) if ticks else None,
             "file": self._file(it),
         }
+        d.update(_parse_jf_providers(it))
+        return d
 
     def iter_shows(self, incremental=False):
         path = f"/Users/{self.uid}/Items"
         for view in self._views("tvshows", self._tv_lib):
             params = {"ParentId": view["Id"], "IncludeItemTypes": "Series",
-                      "Recursive": "true", "Fields": "Overview,ProductionYear,OfficialRating"}
+                      "Recursive": "true", "Fields": "Overview,ProductionYear,OfficialRating,ProviderIds"}
             if incremental:
                 params.update({"SortBy": "DateCreated", "SortOrder": "Descending", "Limit": "50"})
                 items = (self._req(path, params) or {}).get("Items", [])
@@ -362,6 +410,7 @@ class JellyfinVideoSource:
                     "overview": ep.get("Overview"),
                     "air_date": aired[:10] if aired else None,
                     "runtime_minutes": int(ticks / 600_000_000) if ticks else None,
+                    "tvdb_id": _parse_jf_providers(ep).get("tvdb_id"),
                     "file": self._file(ep),
                 })
             for snum, eps in sorted(by_season.items()):
@@ -370,7 +419,7 @@ class JellyfinVideoSource:
                                 "poster_url": None, "episodes": eps})
         except Exception:
             logger.exception("Jellyfin: failed reading episodes for %s", it.get("Name", "?"))
-        return {
+        d = {
             "server_id": series_id,
             "title": it.get("Name"),
             "year": it.get("ProductionYear"),
@@ -381,3 +430,5 @@ class JellyfinVideoSource:
             "content_rating": it.get("OfficialRating"),
             "seasons": seasons,
         }
+        d.update(_parse_jf_providers(it))
+        return d

database/video_database.py

@@ -208,17 +208,18 @@ class VideoDatabase:
         try:
             conn.execute(
                 "INSERT INTO movies (server_source, server_id, title, sort_title, year, overview, "
-                "runtime_minutes, content_rating, studio, poster_url, has_file, updated_at) "
+                "runtime_minutes, content_rating, studio, poster_url, tmdb_id, imdb_id, has_file, updated_at) "
-                "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, CURRENT_TIMESTAMP) "
+                "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, CURRENT_TIMESTAMP) "
                 "ON CONFLICT(server_source, server_id) DO UPDATE SET "
                 "title=excluded.title, sort_title=excluded.sort_title, year=excluded.year, "
                 "overview=excluded.overview, runtime_minutes=excluded.runtime_minutes, "
                 "content_rating=excluded.content_rating, studio=excluded.studio, "
-                "poster_url=excluded.poster_url, has_file=excluded.has_file, updated_at=CURRENT_TIMESTAMP",
+                "poster_url=excluded.poster_url, tmdb_id=excluded.tmdb_id, imdb_id=excluded.imdb_id, "
+                "has_file=excluded.has_file, updated_at=CURRENT_TIMESTAMP",
                 (server_source, item["server_id"], item.get("title"), _sort_title(item.get("title")),
                  item.get("year"), item.get("overview"), item.get("runtime_minutes"),
                  item.get("content_rating"), item.get("studio"), item.get("poster_url"),
-                 1 if item.get("file") else 0),
+                 item.get("tmdb_id"), item.get("imdb_id"), 1 if item.get("file") else 0),
             )
             movie_id = conn.execute(
                 "SELECT id FROM movies WHERE server_source=? AND server_id=?",
@@ -238,16 +239,18 @@ class VideoDatabase:
         try:
             conn.execute(
                 "INSERT INTO shows (server_source, server_id, title, sort_title, year, overview, status, "
-                "network, runtime_minutes, content_rating, poster_url, updated_at) "
+                "network, runtime_minutes, content_rating, poster_url, tvdb_id, tmdb_id, imdb_id, updated_at) "
-                "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, CURRENT_TIMESTAMP) "
+                "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, CURRENT_TIMESTAMP) "
                 "ON CONFLICT(server_source, server_id) DO UPDATE SET "
                 "title=excluded.title, sort_title=excluded.sort_title, year=excluded.year, "
                 "overview=excluded.overview, status=excluded.status, network=excluded.network, "
                 "runtime_minutes=excluded.runtime_minutes, content_rating=excluded.content_rating, "
-                "poster_url=excluded.poster_url, updated_at=CURRENT_TIMESTAMP",
+                "poster_url=excluded.poster_url, tvdb_id=excluded.tvdb_id, tmdb_id=excluded.tmdb_id, "
+                "imdb_id=excluded.imdb_id, updated_at=CURRENT_TIMESTAMP",
                 (server_source, item["server_id"], item.get("title"), _sort_title(item.get("title")),
                  item.get("year"), item.get("overview"), item.get("status"), item.get("network"),
-                 item.get("runtime_minutes"), item.get("content_rating"), item.get("poster_url")),
+                 item.get("runtime_minutes"), item.get("content_rating"), item.get("poster_url"),
+                 item.get("tvdb_id"), item.get("tmdb_id"), item.get("imdb_id")),
             )
             show_id = conn.execute(
                 "SELECT id FROM shows WHERE server_source=? AND server_id=?",
@@ -280,16 +283,17 @@ class VideoDatabase:
                     conn.execute(
                         "INSERT INTO episodes (show_id, season_id, server_source, server_id, "
                         "season_number, episode_number, title, overview, air_date, "
-                        "runtime_minutes, has_file) "
+                        "runtime_minutes, tvdb_id, has_file) "
-                        "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) "
+                        "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) "
                         "ON CONFLICT(show_id, season_number, episode_number) DO UPDATE SET "
                         "season_id=excluded.season_id, server_source=excluded.server_source, "
                         "server_id=excluded.server_id, title=excluded.title, "
                         "overview=excluded.overview, air_date=excluded.air_date, "
-                        "runtime_minutes=excluded.runtime_minutes, has_file=excluded.has_file",
+                        "runtime_minutes=excluded.runtime_minutes, tvdb_id=excluded.tvdb_id, "
+                        "has_file=excluded.has_file",
                         (show_id, season_id, server_source, ep.get("server_id"), snum, enum,
                          ep.get("title"), ep.get("overview"), ep.get("air_date"),
-                         ep.get("runtime_minutes"), 1 if ep.get("file") else 0),
+                         ep.get("runtime_minutes"), ep.get("tvdb_id"), 1 if ep.get("file") else 0),
                     )
                     ep_id = conn.execute(
                         "SELECT id FROM episodes WHERE show_id=? AND season_number=? AND episode_number=?",

database/video_schema.sql

@@ -63,7 +63,7 @@ CREATE TABLE IF NOT EXISTS movies (
     id                   INTEGER PRIMARY KEY,
     server_source        TEXT,            -- 'plex' | 'jellyfin' (NULL = not on a server yet, e.g. wishlist)
     server_id            TEXT,            -- media server native id (Plex ratingKey / Jellyfin Item Id)
-    tmdb_id              INTEGER UNIQUE,
+    tmdb_id              INTEGER,         -- not unique: same film can sit in >1 library
     imdb_id              TEXT,
     title                TEXT NOT NULL,
     sort_title           TEXT,
@@ -97,7 +97,7 @@ CREATE TABLE IF NOT EXISTS shows (
     id                 INTEGER PRIMARY KEY,
     server_source      TEXT,             -- 'plex' | 'jellyfin' (NULL = not on a server yet)
     server_id          TEXT,             -- media server native id
-    tvdb_id            INTEGER UNIQUE,
+    tvdb_id            INTEGER,          -- not unique (same series can sit in >1 library)
     tmdb_id            INTEGER,
     imdb_id            TEXT,
     title              TEXT NOT NULL,

tests/test_video_database.py

@@ -224,6 +224,23 @@ def test_upsert_show_tree_builds_seasons_episodes_and_prunes(db):
     assert eps == [1]
 
 
+def test_upsert_stores_provider_ids(db):
+    mid = db.upsert_movie("plex", {"server_id": "m1", "title": "Inception",
+                                   "tmdb_id": 27205, "imdb_id": "tt1375666"})
+    with db.connect() as c:
+        row = c.execute("SELECT tmdb_id, imdb_id FROM movies WHERE id=?", (mid,)).fetchone()
+    assert (row["tmdb_id"], row["imdb_id"]) == (27205, "tt1375666")
+
+    sid = db.upsert_show_tree("plex", {"server_id": "s1", "title": "Show", "tvdb_id": 121361,
+                                       "tmdb_id": 1396, "imdb_id": "tt0903747", "seasons": [
+        {"season_number": 1, "episodes": [{"episode_number": 1, "tvdb_id": 349232}]}]})
+    with db.connect() as c:
+        srow = c.execute("SELECT tvdb_id, tmdb_id, imdb_id FROM shows WHERE id=?", (sid,)).fetchone()
+        erow = c.execute("SELECT tvdb_id FROM episodes WHERE show_id=?", (sid,)).fetchone()
+    assert (srow["tvdb_id"], srow["tmdb_id"], srow["imdb_id"]) == (121361, 1396, "tt0903747")
+    assert erow["tvdb_id"] == 349232
+
+
 def test_prune_missing_skips_when_over_half_would_be_removed(db):
     # >100 movies; a scan that "sees" only a couple must NOT wipe the rest
     # (mirrors music's deep-scan 50% safety against partial server failures).

tests/test_video_scanner.py

@@ -167,6 +167,27 @@ def test_incremental_falls_back_to_full_on_small_library(db):
     assert st["movies"] == 3
 
 
+def test_parse_plex_guids():
+    from core.video.sources import _parse_plex_guids
+
+    class _G:
+        def __init__(self, gid): self.id = gid
+
+    class _Obj:
+        def __init__(self, guids): self.guids = guids
+
+    got = _parse_plex_guids(_Obj([_G("imdb://tt1375666"), _G("tmdb://27205"), _G("tvdb://121361")]))
+    assert got == {"tmdb_id": 27205, "imdb_id": "tt1375666", "tvdb_id": 121361}
+    assert _parse_plex_guids(_Obj([])) == {"tmdb_id": None, "imdb_id": None, "tvdb_id": None}
+
+
+def test_parse_jellyfin_providers():
+    from core.video.sources import _parse_jf_providers
+    got = _parse_jf_providers({"ProviderIds": {"Imdb": "tt123", "Tmdb": "27205", "Tvdb": "121361"}})
+    assert got == {"imdb_id": "tt123", "tmdb_id": 27205, "tvdb_id": 121361}
+    assert _parse_jf_providers({}) == {"imdb_id": None, "tmdb_id": None, "tvdb_id": None}
+
+
 def test_core_video_imports_nothing_from_music():
     base = Path(__file__).resolve().parent.parent / "core" / "video"
     for py in base.glob("*.py"):