video scan: capture the provider IDs the server already has (tmdb/imdb/tvdb)
The servers already matched everything to their agents — we were dropping the IDs. Now we store them: - Plex: parse item.guids (imdb://, tmdb://, tvdb://); Jellyfin: parse item.ProviderIds (added ProviderIds to the requested Fields). - Stored on movies (tmdb_id, imdb_id), shows (tvdb_id, tmdb_id, imdb_id), and episodes (tvdb_id) via the upserts. - Dropped the over-strict UNIQUE on movies.tmdb_id / shows.tvdb_id (same title can legitimately live in two libraries; we dedupe on server_id). Scanner now wraps each upsert in try/except so one bad item can't abort a scan. Tests: guid/ProviderIds parsing + IDs persisted. 38 video-DB/scanner tests green.
This commit is contained in:
BoulderBadgeDad
parent
9eecd6d2c4
commit
13e03a624c
6 changed files with 124 additions and 23 deletions
|
|
@ -150,7 +150,11 @@ class VideoLibraryScanner:
|
|||
break
|
||||
continue
|
||||
consec = 0
|
||||
self.db.upsert_movie(server, item)
|
||||
try:
|
||||
self.db.upsert_movie(server, item)
|
||||
except Exception:
|
||||
logger.exception("video scan: skipping movie %s", sid)
|
||||
continue
|
||||
seen_movies.add(sid)
|
||||
movies += 1
|
||||
processed += 1
|
||||
|
|
@ -178,7 +182,11 @@ class VideoLibraryScanner:
|
|||
break
|
||||
continue
|
||||
consec = 0
|
||||
self.db.upsert_show_tree(server, show)
|
||||
try:
|
||||
self.db.upsert_show_tree(server, show)
|
||||
except Exception:
|
||||
logger.exception("video scan: skipping show %s", sid)
|
||||
continue
|
||||
seen_shows.add(sid)
|
||||
shows += 1
|
||||
episodes += sum(len(s.get("episodes", [])) for s in show.get("seasons", []))
|
||||
|
|
|
|||
|
|
@ -13,6 +13,8 @@ these adapters.
|
|||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
|
||||
from utils.logging_config import get_logger
|
||||
|
||||
logger = get_logger("video_sources")
|
||||
|
|
@ -22,6 +24,45 @@ logger = get_logger("video_sources")
|
|||
PLEX_SCAN_TIMEOUT = 120
|
||||
|
||||
|
||||
def _to_int(val):
|
||||
if val is None:
|
||||
return None
|
||||
m = re.match(r"\d+", str(val))
|
||||
return int(m.group()) if m else None
|
||||
|
||||
|
||||
def _parse_plex_guids(obj) -> dict:
|
||||
"""tmdb/imdb/tvdb ids from a Plex item's guids — Plex already matched them."""
|
||||
out = {"tmdb_id": None, "imdb_id": None, "tvdb_id": None}
|
||||
try:
|
||||
for g in (getattr(obj, "guids", None) or []):
|
||||
gid = getattr(g, "id", "") or ""
|
||||
if "://" not in gid:
|
||||
continue
|
||||
scheme, value = gid.split("://", 1)
|
||||
scheme = scheme.lower()
|
||||
if scheme == "imdb":
|
||||
out["imdb_id"] = (value.split("?")[0] or None)
|
||||
elif scheme == "tmdb":
|
||||
out["tmdb_id"] = _to_int(value)
|
||||
elif scheme == "tvdb":
|
||||
out["tvdb_id"] = _to_int(value)
|
||||
except Exception:
|
||||
pass
|
||||
return out
|
||||
|
||||
|
||||
def _parse_jf_providers(item) -> dict:
|
||||
"""tmdb/imdb/tvdb ids from a Jellyfin item's ProviderIds."""
|
||||
providers = item.get("ProviderIds") or {}
|
||||
low = {(k or "").lower(): v for k, v in providers.items()}
|
||||
return {
|
||||
"imdb_id": low.get("imdb") or None,
|
||||
"tmdb_id": _to_int(low.get("tmdb")),
|
||||
"tvdb_id": _to_int(low.get("tvdb")),
|
||||
}
|
||||
|
||||
|
||||
def _build_source(movies_lib=None, tv_lib=None):
|
||||
"""Build a media source for the active server, restricted to the named
|
||||
Movies/TV libraries when given. Reuses the SHARED connection config — but
|
||||
|
|
@ -156,7 +197,7 @@ class PlexVideoSource:
|
|||
|
||||
def _movie(self, m) -> dict:
|
||||
dur = getattr(m, "duration", None)
|
||||
return {
|
||||
d = {
|
||||
"server_id": str(m.ratingKey),
|
||||
"title": m.title,
|
||||
"year": getattr(m, "year", None),
|
||||
|
|
@ -167,6 +208,8 @@ class PlexVideoSource:
|
|||
"runtime_minutes": int(dur / 60000) if dur else None,
|
||||
"file": self._part_file(m),
|
||||
}
|
||||
d.update(_parse_plex_guids(m))
|
||||
return d
|
||||
|
||||
def _episode(self, ep, snum, enum) -> dict:
|
||||
dur = getattr(ep, "duration", None)
|
||||
|
|
@ -179,6 +222,7 @@ class PlexVideoSource:
|
|||
"overview": getattr(ep, "summary", None),
|
||||
"air_date": aired.date().isoformat() if aired else None,
|
||||
"runtime_minutes": int(dur / 60000) if dur else None,
|
||||
"tvdb_id": _parse_plex_guids(ep).get("tvdb_id"),
|
||||
"file": self._part_file(ep),
|
||||
}
|
||||
|
||||
|
|
@ -199,7 +243,7 @@ class PlexVideoSource:
|
|||
seasons = [{"server_id": None, "season_number": n, "title": None,
|
||||
"overview": None, "poster_url": None, "episodes": eps}
|
||||
for n, eps in sorted(seasons_map.items())]
|
||||
return {
|
||||
d = {
|
||||
"server_id": str(sh.ratingKey),
|
||||
"title": sh.title,
|
||||
"year": getattr(sh, "year", None),
|
||||
|
|
@ -210,11 +254,13 @@ class PlexVideoSource:
|
|||
"content_rating": getattr(sh, "contentRating", None),
|
||||
"seasons": seasons,
|
||||
}
|
||||
d.update(_parse_plex_guids(sh))
|
||||
return d
|
||||
|
||||
|
||||
# ── Jellyfin ────────────────────────────────────────────────────────────────
|
||||
_JF_MOVIE_FIELDS = "Overview,Path,MediaSources,ProductionYear,OfficialRating,RunTimeTicks,Studios"
|
||||
_JF_EP_FIELDS = "Overview,Path,MediaSources,PremiereDate,RunTimeTicks,IndexNumber,ParentIndexNumber"
|
||||
_JF_MOVIE_FIELDS = "Overview,Path,MediaSources,ProductionYear,OfficialRating,RunTimeTicks,Studios,ProviderIds"
|
||||
_JF_EP_FIELDS = "Overview,Path,MediaSources,PremiereDate,RunTimeTicks,IndexNumber,ParentIndexNumber,ProviderIds"
|
||||
|
||||
|
||||
class JellyfinVideoSource:
|
||||
|
|
@ -312,7 +358,7 @@ class JellyfinVideoSource:
|
|||
def _movie(self, it) -> dict:
|
||||
studios = it.get("Studios") or []
|
||||
ticks = it.get("RunTimeTicks")
|
||||
return {
|
||||
d = {
|
||||
"server_id": str(it["Id"]),
|
||||
"title": it.get("Name"),
|
||||
"year": it.get("ProductionYear"),
|
||||
|
|
@ -323,12 +369,14 @@ class JellyfinVideoSource:
|
|||
"runtime_minutes": int(ticks / 600_000_000) if ticks else None,
|
||||
"file": self._file(it),
|
||||
}
|
||||
d.update(_parse_jf_providers(it))
|
||||
return d
|
||||
|
||||
def iter_shows(self, incremental=False):
|
||||
path = f"/Users/{self.uid}/Items"
|
||||
for view in self._views("tvshows", self._tv_lib):
|
||||
params = {"ParentId": view["Id"], "IncludeItemTypes": "Series",
|
||||
"Recursive": "true", "Fields": "Overview,ProductionYear,OfficialRating"}
|
||||
"Recursive": "true", "Fields": "Overview,ProductionYear,OfficialRating,ProviderIds"}
|
||||
if incremental:
|
||||
params.update({"SortBy": "DateCreated", "SortOrder": "Descending", "Limit": "50"})
|
||||
items = (self._req(path, params) or {}).get("Items", [])
|
||||
|
|
@ -362,6 +410,7 @@ class JellyfinVideoSource:
|
|||
"overview": ep.get("Overview"),
|
||||
"air_date": aired[:10] if aired else None,
|
||||
"runtime_minutes": int(ticks / 600_000_000) if ticks else None,
|
||||
"tvdb_id": _parse_jf_providers(ep).get("tvdb_id"),
|
||||
"file": self._file(ep),
|
||||
})
|
||||
for snum, eps in sorted(by_season.items()):
|
||||
|
|
@ -370,7 +419,7 @@ class JellyfinVideoSource:
|
|||
"poster_url": None, "episodes": eps})
|
||||
except Exception:
|
||||
logger.exception("Jellyfin: failed reading episodes for %s", it.get("Name", "?"))
|
||||
return {
|
||||
d = {
|
||||
"server_id": series_id,
|
||||
"title": it.get("Name"),
|
||||
"year": it.get("ProductionYear"),
|
||||
|
|
@ -381,3 +430,5 @@ class JellyfinVideoSource:
|
|||
"content_rating": it.get("OfficialRating"),
|
||||
"seasons": seasons,
|
||||
}
|
||||
d.update(_parse_jf_providers(it))
|
||||
return d
|
||||
|
|
|
|||
|
|
@ -208,17 +208,18 @@ class VideoDatabase:
|
|||
try:
|
||||
conn.execute(
|
||||
"INSERT INTO movies (server_source, server_id, title, sort_title, year, overview, "
|
||||
"runtime_minutes, content_rating, studio, poster_url, has_file, updated_at) "
|
||||
"VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, CURRENT_TIMESTAMP) "
|
||||
"runtime_minutes, content_rating, studio, poster_url, tmdb_id, imdb_id, has_file, updated_at) "
|
||||
"VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, CURRENT_TIMESTAMP) "
|
||||
"ON CONFLICT(server_source, server_id) DO UPDATE SET "
|
||||
"title=excluded.title, sort_title=excluded.sort_title, year=excluded.year, "
|
||||
"overview=excluded.overview, runtime_minutes=excluded.runtime_minutes, "
|
||||
"content_rating=excluded.content_rating, studio=excluded.studio, "
|
||||
"poster_url=excluded.poster_url, has_file=excluded.has_file, updated_at=CURRENT_TIMESTAMP",
|
||||
"poster_url=excluded.poster_url, tmdb_id=excluded.tmdb_id, imdb_id=excluded.imdb_id, "
|
||||
"has_file=excluded.has_file, updated_at=CURRENT_TIMESTAMP",
|
||||
(server_source, item["server_id"], item.get("title"), _sort_title(item.get("title")),
|
||||
item.get("year"), item.get("overview"), item.get("runtime_minutes"),
|
||||
item.get("content_rating"), item.get("studio"), item.get("poster_url"),
|
||||
1 if item.get("file") else 0),
|
||||
item.get("tmdb_id"), item.get("imdb_id"), 1 if item.get("file") else 0),
|
||||
)
|
||||
movie_id = conn.execute(
|
||||
"SELECT id FROM movies WHERE server_source=? AND server_id=?",
|
||||
|
|
@ -238,16 +239,18 @@ class VideoDatabase:
|
|||
try:
|
||||
conn.execute(
|
||||
"INSERT INTO shows (server_source, server_id, title, sort_title, year, overview, status, "
|
||||
"network, runtime_minutes, content_rating, poster_url, updated_at) "
|
||||
"VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, CURRENT_TIMESTAMP) "
|
||||
"network, runtime_minutes, content_rating, poster_url, tvdb_id, tmdb_id, imdb_id, updated_at) "
|
||||
"VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, CURRENT_TIMESTAMP) "
|
||||
"ON CONFLICT(server_source, server_id) DO UPDATE SET "
|
||||
"title=excluded.title, sort_title=excluded.sort_title, year=excluded.year, "
|
||||
"overview=excluded.overview, status=excluded.status, network=excluded.network, "
|
||||
"runtime_minutes=excluded.runtime_minutes, content_rating=excluded.content_rating, "
|
||||
"poster_url=excluded.poster_url, updated_at=CURRENT_TIMESTAMP",
|
||||
"poster_url=excluded.poster_url, tvdb_id=excluded.tvdb_id, tmdb_id=excluded.tmdb_id, "
|
||||
"imdb_id=excluded.imdb_id, updated_at=CURRENT_TIMESTAMP",
|
||||
(server_source, item["server_id"], item.get("title"), _sort_title(item.get("title")),
|
||||
item.get("year"), item.get("overview"), item.get("status"), item.get("network"),
|
||||
item.get("runtime_minutes"), item.get("content_rating"), item.get("poster_url")),
|
||||
item.get("runtime_minutes"), item.get("content_rating"), item.get("poster_url"),
|
||||
item.get("tvdb_id"), item.get("tmdb_id"), item.get("imdb_id")),
|
||||
)
|
||||
show_id = conn.execute(
|
||||
"SELECT id FROM shows WHERE server_source=? AND server_id=?",
|
||||
|
|
@ -280,16 +283,17 @@ class VideoDatabase:
|
|||
conn.execute(
|
||||
"INSERT INTO episodes (show_id, season_id, server_source, server_id, "
|
||||
"season_number, episode_number, title, overview, air_date, "
|
||||
"runtime_minutes, has_file) "
|
||||
"VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) "
|
||||
"runtime_minutes, tvdb_id, has_file) "
|
||||
"VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) "
|
||||
"ON CONFLICT(show_id, season_number, episode_number) DO UPDATE SET "
|
||||
"season_id=excluded.season_id, server_source=excluded.server_source, "
|
||||
"server_id=excluded.server_id, title=excluded.title, "
|
||||
"overview=excluded.overview, air_date=excluded.air_date, "
|
||||
"runtime_minutes=excluded.runtime_minutes, has_file=excluded.has_file",
|
||||
"runtime_minutes=excluded.runtime_minutes, tvdb_id=excluded.tvdb_id, "
|
||||
"has_file=excluded.has_file",
|
||||
(show_id, season_id, server_source, ep.get("server_id"), snum, enum,
|
||||
ep.get("title"), ep.get("overview"), ep.get("air_date"),
|
||||
ep.get("runtime_minutes"), 1 if ep.get("file") else 0),
|
||||
ep.get("runtime_minutes"), ep.get("tvdb_id"), 1 if ep.get("file") else 0),
|
||||
)
|
||||
ep_id = conn.execute(
|
||||
"SELECT id FROM episodes WHERE show_id=? AND season_number=? AND episode_number=?",
|
||||
|
|
|
|||
|
|
@ -63,7 +63,7 @@ CREATE TABLE IF NOT EXISTS movies (
|
|||
id INTEGER PRIMARY KEY,
|
||||
server_source TEXT, -- 'plex' | 'jellyfin' (NULL = not on a server yet, e.g. wishlist)
|
||||
server_id TEXT, -- media server native id (Plex ratingKey / Jellyfin Item Id)
|
||||
tmdb_id INTEGER UNIQUE,
|
||||
tmdb_id INTEGER, -- not unique: same film can sit in >1 library
|
||||
imdb_id TEXT,
|
||||
title TEXT NOT NULL,
|
||||
sort_title TEXT,
|
||||
|
|
@ -97,7 +97,7 @@ CREATE TABLE IF NOT EXISTS shows (
|
|||
id INTEGER PRIMARY KEY,
|
||||
server_source TEXT, -- 'plex' | 'jellyfin' (NULL = not on a server yet)
|
||||
server_id TEXT, -- media server native id
|
||||
tvdb_id INTEGER UNIQUE,
|
||||
tvdb_id INTEGER, -- not unique (same series can sit in >1 library)
|
||||
tmdb_id INTEGER,
|
||||
imdb_id TEXT,
|
||||
title TEXT NOT NULL,
|
||||
|
|
|
|||
|
|
@ -224,6 +224,23 @@ def test_upsert_show_tree_builds_seasons_episodes_and_prunes(db):
|
|||
assert eps == [1]
|
||||
|
||||
|
||||
def test_upsert_stores_provider_ids(db):
|
||||
mid = db.upsert_movie("plex", {"server_id": "m1", "title": "Inception",
|
||||
"tmdb_id": 27205, "imdb_id": "tt1375666"})
|
||||
with db.connect() as c:
|
||||
row = c.execute("SELECT tmdb_id, imdb_id FROM movies WHERE id=?", (mid,)).fetchone()
|
||||
assert (row["tmdb_id"], row["imdb_id"]) == (27205, "tt1375666")
|
||||
|
||||
sid = db.upsert_show_tree("plex", {"server_id": "s1", "title": "Show", "tvdb_id": 121361,
|
||||
"tmdb_id": 1396, "imdb_id": "tt0903747", "seasons": [
|
||||
{"season_number": 1, "episodes": [{"episode_number": 1, "tvdb_id": 349232}]}]})
|
||||
with db.connect() as c:
|
||||
srow = c.execute("SELECT tvdb_id, tmdb_id, imdb_id FROM shows WHERE id=?", (sid,)).fetchone()
|
||||
erow = c.execute("SELECT tvdb_id FROM episodes WHERE show_id=?", (sid,)).fetchone()
|
||||
assert (srow["tvdb_id"], srow["tmdb_id"], srow["imdb_id"]) == (121361, 1396, "tt0903747")
|
||||
assert erow["tvdb_id"] == 349232
|
||||
|
||||
|
||||
def test_prune_missing_skips_when_over_half_would_be_removed(db):
|
||||
# >100 movies; a scan that "sees" only a couple must NOT wipe the rest
|
||||
# (mirrors music's deep-scan 50% safety against partial server failures).
|
||||
|
|
|
|||
|
|
@ -167,6 +167,27 @@ def test_incremental_falls_back_to_full_on_small_library(db):
|
|||
assert st["movies"] == 3
|
||||
|
||||
|
||||
def test_parse_plex_guids():
|
||||
from core.video.sources import _parse_plex_guids
|
||||
|
||||
class _G:
|
||||
def __init__(self, gid): self.id = gid
|
||||
|
||||
class _Obj:
|
||||
def __init__(self, guids): self.guids = guids
|
||||
|
||||
got = _parse_plex_guids(_Obj([_G("imdb://tt1375666"), _G("tmdb://27205"), _G("tvdb://121361")]))
|
||||
assert got == {"tmdb_id": 27205, "imdb_id": "tt1375666", "tvdb_id": 121361}
|
||||
assert _parse_plex_guids(_Obj([])) == {"tmdb_id": None, "imdb_id": None, "tvdb_id": None}
|
||||
|
||||
|
||||
def test_parse_jellyfin_providers():
|
||||
from core.video.sources import _parse_jf_providers
|
||||
got = _parse_jf_providers({"ProviderIds": {"Imdb": "tt123", "Tmdb": "27205", "Tvdb": "121361"}})
|
||||
assert got == {"imdb_id": "tt123", "tmdb_id": 27205, "tvdb_id": 121361}
|
||||
assert _parse_jf_providers({}) == {"imdb_id": None, "tmdb_id": None, "tvdb_id": None}
|
||||
|
||||
|
||||
def test_core_video_imports_nothing_from_music():
|
||||
base = Path(__file__).resolve().parent.parent / "core" / "video"
|
||||
for py in base.glob("*.py"):
|
||||
|
|
|
|||
Loading…
Reference in a new issue