video DB: server-sourced scan upserts (movies/shows/seasons/episodes)
Server (Plex/Jellyfin) is the source of truth, so every scanned row carries (server_source, server_id) for upsert + stale-removal — mirroring how music keys on server_source + ratingKey. - schema: server_source/server_id columns on movies/shows/episodes (+ server_id on seasons); unique (server_source,server_id) on movies/shows (multiple NULLs allowed so wishlist rows never block). - VideoDatabase.upsert_movie / upsert_show_tree: take normalized, server- agnostic dicts (a Plex/Jellyfin adapter produces them — DB never touches a media SDK), set has_file + media_files, and prune episodes/seasons the server no longer reports. - prune_missing(): removes top-level movies/shows the scan didn't see (cascades clean children). 6 new tests (insert/update/file-replace, season/episode build+prune, top-level prune); 18 video-DB tests green.
This commit is contained in:
BoulderBadgeDad
parent
401a9be0ec
commit
462fa50423
3 changed files with 219 additions and 0 deletions
|
|
@ -161,6 +161,159 @@ class VideoDatabase:
|
|||
finally:
|
||||
conn.close()
|
||||
|
||||
# ── scan upserts (server is the source of truth) ──────────────────────────
|
||||
# The scanner passes normalized, server-agnostic dicts (a Plex/Jellyfin
|
||||
# adapter produces them) so this layer never touches a media-server SDK.
|
||||
@staticmethod
|
||||
def _set_media_file(conn, owner_col: str, owner_id: int, file: dict | None) -> None:
|
||||
"""Replace the media_files row(s) for one owner. owner_col is internal
|
||||
('movie_id'|'episode_id'|'video_id'), never user input."""
|
||||
conn.execute(f"DELETE FROM media_files WHERE {owner_col} = ?", (owner_id,))
|
||||
if not file:
|
||||
return
|
||||
conn.execute(
|
||||
f"INSERT INTO media_files ({owner_col}, relative_path, size_bytes, resolution, "
|
||||
"video_codec, audio_codec, release_source, quality, runtime_seconds) "
|
||||
"VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)",
|
||||
(owner_id,
|
||||
file.get("relative_path") or file.get("path") or "",
|
||||
file.get("size_bytes"), file.get("resolution"), file.get("video_codec"),
|
||||
file.get("audio_codec"), file.get("release_source"), file.get("quality"),
|
||||
file.get("runtime_seconds")),
|
||||
)
|
||||
|
||||
def upsert_movie(self, server_source: str, item: dict) -> int:
|
||||
"""Insert/update one movie (keyed on server id) and its file. Returns row id."""
|
||||
conn = self._get_connection()
|
||||
try:
|
||||
conn.execute(
|
||||
"INSERT INTO movies (server_source, server_id, title, year, overview, "
|
||||
"runtime_minutes, content_rating, studio, poster_url, has_file, updated_at) "
|
||||
"VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, CURRENT_TIMESTAMP) "
|
||||
"ON CONFLICT(server_source, server_id) DO UPDATE SET "
|
||||
"title=excluded.title, year=excluded.year, overview=excluded.overview, "
|
||||
"runtime_minutes=excluded.runtime_minutes, content_rating=excluded.content_rating, "
|
||||
"studio=excluded.studio, poster_url=excluded.poster_url, "
|
||||
"has_file=excluded.has_file, updated_at=CURRENT_TIMESTAMP",
|
||||
(server_source, item["server_id"], item.get("title"), item.get("year"),
|
||||
item.get("overview"), item.get("runtime_minutes"), item.get("content_rating"),
|
||||
item.get("studio"), item.get("poster_url"), 1 if item.get("file") else 0),
|
||||
)
|
||||
movie_id = conn.execute(
|
||||
"SELECT id FROM movies WHERE server_source=? AND server_id=?",
|
||||
(server_source, item["server_id"]),
|
||||
).fetchone()["id"]
|
||||
self._set_media_file(conn, "movie_id", movie_id, item.get("file"))
|
||||
conn.commit()
|
||||
return movie_id
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
def upsert_show_tree(self, server_source: str, item: dict) -> int:
|
||||
"""Insert/update a show with its seasons + episodes (and files) in one
|
||||
transaction. Episodes/seasons no longer present on the server for this
|
||||
show are pruned. Returns the show row id."""
|
||||
conn = self._get_connection()
|
||||
try:
|
||||
conn.execute(
|
||||
"INSERT INTO shows (server_source, server_id, title, year, overview, status, "
|
||||
"network, runtime_minutes, content_rating, poster_url, updated_at) "
|
||||
"VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, CURRENT_TIMESTAMP) "
|
||||
"ON CONFLICT(server_source, server_id) DO UPDATE SET "
|
||||
"title=excluded.title, year=excluded.year, overview=excluded.overview, "
|
||||
"status=excluded.status, network=excluded.network, "
|
||||
"runtime_minutes=excluded.runtime_minutes, content_rating=excluded.content_rating, "
|
||||
"poster_url=excluded.poster_url, updated_at=CURRENT_TIMESTAMP",
|
||||
(server_source, item["server_id"], item.get("title"), item.get("year"),
|
||||
item.get("overview"), item.get("status"), item.get("network"),
|
||||
item.get("runtime_minutes"), item.get("content_rating"), item.get("poster_url")),
|
||||
)
|
||||
show_id = conn.execute(
|
||||
"SELECT id FROM shows WHERE server_source=? AND server_id=?",
|
||||
(server_source, item["server_id"]),
|
||||
).fetchone()["id"]
|
||||
|
||||
seen_seasons: set[int] = set()
|
||||
seen_eps: set[tuple[int, int]] = set()
|
||||
for season in item.get("seasons", []):
|
||||
snum = season["season_number"]
|
||||
seen_seasons.add(snum)
|
||||
conn.execute(
|
||||
"INSERT INTO seasons (show_id, server_id, season_number, title, overview, poster_url) "
|
||||
"VALUES (?, ?, ?, ?, ?, ?) "
|
||||
"ON CONFLICT(show_id, season_number) DO UPDATE SET "
|
||||
"server_id=excluded.server_id, title=excluded.title, "
|
||||
"overview=excluded.overview, poster_url=excluded.poster_url",
|
||||
(show_id, season.get("server_id"), snum, season.get("title"),
|
||||
season.get("overview"), season.get("poster_url")),
|
||||
)
|
||||
season_id = conn.execute(
|
||||
"SELECT id FROM seasons WHERE show_id=? AND season_number=?", (show_id, snum)
|
||||
).fetchone()["id"]
|
||||
|
||||
for ep in season.get("episodes", []):
|
||||
enum = ep["episode_number"]
|
||||
seen_eps.add((snum, enum))
|
||||
conn.execute(
|
||||
"INSERT INTO episodes (show_id, season_id, server_source, server_id, "
|
||||
"season_number, episode_number, title, overview, air_date, "
|
||||
"runtime_minutes, has_file) "
|
||||
"VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) "
|
||||
"ON CONFLICT(show_id, season_number, episode_number) DO UPDATE SET "
|
||||
"season_id=excluded.season_id, server_source=excluded.server_source, "
|
||||
"server_id=excluded.server_id, title=excluded.title, "
|
||||
"overview=excluded.overview, air_date=excluded.air_date, "
|
||||
"runtime_minutes=excluded.runtime_minutes, has_file=excluded.has_file",
|
||||
(show_id, season_id, server_source, ep.get("server_id"), snum, enum,
|
||||
ep.get("title"), ep.get("overview"), ep.get("air_date"),
|
||||
ep.get("runtime_minutes"), 1 if ep.get("file") else 0),
|
||||
)
|
||||
ep_id = conn.execute(
|
||||
"SELECT id FROM episodes WHERE show_id=? AND season_number=? AND episode_number=?",
|
||||
(show_id, snum, enum),
|
||||
).fetchone()["id"]
|
||||
self._set_media_file(conn, "episode_id", ep_id, ep.get("file"))
|
||||
|
||||
# Prune episodes/seasons that vanished from the server for this show.
|
||||
for row in conn.execute(
|
||||
"SELECT season_number, episode_number FROM episodes WHERE show_id=?", (show_id,)
|
||||
).fetchall():
|
||||
if (row["season_number"], row["episode_number"]) not in seen_eps:
|
||||
conn.execute(
|
||||
"DELETE FROM episodes WHERE show_id=? AND season_number=? AND episode_number=?",
|
||||
(show_id, row["season_number"], row["episode_number"]),
|
||||
)
|
||||
for row in conn.execute(
|
||||
"SELECT season_number FROM seasons WHERE show_id=?", (show_id,)
|
||||
).fetchall():
|
||||
if row["season_number"] not in seen_seasons:
|
||||
conn.execute("DELETE FROM seasons WHERE show_id=? AND season_number=?",
|
||||
(show_id, row["season_number"]))
|
||||
conn.commit()
|
||||
return show_id
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
def prune_missing(self, table: str, server_source: str, seen_ids) -> int:
|
||||
"""Delete top-level rows for a server that the scan no longer saw.
|
||||
``table`` is internal ('movies'|'shows'); cascades clean children."""
|
||||
if table not in ("movies", "shows"):
|
||||
raise ValueError(f"prune_missing: unexpected table {table!r}")
|
||||
seen = {str(s) for s in seen_ids}
|
||||
conn = self._get_connection()
|
||||
try:
|
||||
existing = [r["server_id"] for r in conn.execute(
|
||||
f"SELECT server_id FROM {table} WHERE server_source=?", (server_source,)
|
||||
).fetchall()]
|
||||
stale = [sid for sid in existing if str(sid) not in seen]
|
||||
for sid in stale:
|
||||
conn.execute(f"DELETE FROM {table} WHERE server_source=? AND server_id=?",
|
||||
(server_source, sid))
|
||||
conn.commit()
|
||||
return len(stale)
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
# ── health ───────────────────────────────────────────────────────────────
|
||||
def health_check(self) -> bool:
|
||||
"""True when the DB opens and passes a quick integrity check."""
|
||||
|
|
|
|||
|
|
@ -61,6 +61,8 @@ CREATE TABLE IF NOT EXISTS video_settings (
|
|||
-- ── Content: Movies ─────────────────────────────────────────────────────────
|
||||
CREATE TABLE IF NOT EXISTS movies (
|
||||
id INTEGER PRIMARY KEY,
|
||||
server_source TEXT, -- 'plex' | 'jellyfin' (NULL = not on a server yet, e.g. wishlist)
|
||||
server_id TEXT, -- media server native id (Plex ratingKey / Jellyfin Item Id)
|
||||
tmdb_id INTEGER UNIQUE,
|
||||
imdb_id TEXT,
|
||||
title TEXT NOT NULL,
|
||||
|
|
@ -86,10 +88,15 @@ CREATE TABLE IF NOT EXISTS movies (
|
|||
CREATE INDEX IF NOT EXISTS idx_movies_tmdb ON movies(tmdb_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_movies_monitored ON movies(monitored, has_file);
|
||||
CREATE INDEX IF NOT EXISTS idx_movies_release ON movies(release_date);
|
||||
-- Upsert/stale-removal key: the server's native id. Multiple NULLs are allowed
|
||||
-- (wishlist items not yet on a server), so this never blocks non-server rows.
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS ux_movies_server ON movies(server_source, server_id);
|
||||
|
||||
-- ── Content: TV (shows → seasons → episodes) ────────────────────────────────
|
||||
CREATE TABLE IF NOT EXISTS shows (
|
||||
id INTEGER PRIMARY KEY,
|
||||
server_source TEXT, -- 'plex' | 'jellyfin' (NULL = not on a server yet)
|
||||
server_id TEXT, -- media server native id
|
||||
tvdb_id INTEGER UNIQUE,
|
||||
tmdb_id INTEGER,
|
||||
imdb_id TEXT,
|
||||
|
|
@ -112,10 +119,12 @@ CREATE TABLE IF NOT EXISTS shows (
|
|||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_shows_tvdb ON shows(tvdb_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_shows_monitored ON shows(monitored);
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS ux_shows_server ON shows(server_source, server_id);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS seasons (
|
||||
id INTEGER PRIMARY KEY,
|
||||
show_id INTEGER NOT NULL REFERENCES shows(id) ON DELETE CASCADE,
|
||||
server_id TEXT, -- media server native id (for reference/refresh)
|
||||
season_number INTEGER NOT NULL,
|
||||
title TEXT,
|
||||
overview TEXT,
|
||||
|
|
@ -129,6 +138,8 @@ CREATE TABLE IF NOT EXISTS episodes (
|
|||
id INTEGER PRIMARY KEY,
|
||||
show_id INTEGER NOT NULL REFERENCES shows(id) ON DELETE CASCADE,
|
||||
season_id INTEGER NOT NULL REFERENCES seasons(id) ON DELETE CASCADE,
|
||||
server_source TEXT, -- 'plex' | 'jellyfin'
|
||||
server_id TEXT, -- media server native id
|
||||
season_number INTEGER NOT NULL,
|
||||
episode_number INTEGER NOT NULL,
|
||||
title TEXT,
|
||||
|
|
|
|||
|
|
@ -170,6 +170,61 @@ def test_settings_kv_roundtrip(db):
|
|||
assert db.get_setting("download_dir") == "/data/video2"
|
||||
|
||||
|
||||
# ── scan upserts (server source of truth) ────────────────────────────────────
|
||||
|
||||
def test_upsert_movie_inserts_updates_and_attaches_file(db):
|
||||
mid = db.upsert_movie("plex", {
|
||||
"server_id": "p1", "title": "Dune", "year": 2021,
|
||||
"file": {"relative_path": "Dune.mkv", "size_bytes": 2000, "resolution": "2160p"}})
|
||||
with db.connect() as c:
|
||||
row = c.execute("SELECT title,year,has_file FROM movies WHERE id=?", (mid,)).fetchone()
|
||||
assert (row["title"], row["year"], row["has_file"]) == ("Dune", 2021, 1)
|
||||
f = c.execute("SELECT relative_path,size_bytes,resolution FROM media_files WHERE movie_id=?",
|
||||
(mid,)).fetchone()
|
||||
assert (f["relative_path"], f["size_bytes"], f["resolution"]) == ("Dune.mkv", 2000, "2160p")
|
||||
# Re-scan same server id -> same row, updated fields, file replaced (not duplicated).
|
||||
mid2 = db.upsert_movie("plex", {
|
||||
"server_id": "p1", "title": "Dune: Part One", "year": 2021,
|
||||
"file": {"relative_path": "Dune1.mkv", "size_bytes": 3000}})
|
||||
assert mid2 == mid
|
||||
with db.connect() as c:
|
||||
assert c.execute("SELECT title FROM movies WHERE id=?", (mid,)).fetchone()["title"] == "Dune: Part One"
|
||||
files = c.execute("SELECT relative_path FROM media_files WHERE movie_id=?", (mid,)).fetchall()
|
||||
assert [r["relative_path"] for r in files] == ["Dune1.mkv"]
|
||||
assert db.dashboard_stats()["library"]["movies"] == 1
|
||||
|
||||
|
||||
def test_upsert_show_tree_builds_seasons_episodes_and_prunes(db):
|
||||
item = {"server_id": "s1", "title": "Show", "seasons": [
|
||||
{"season_number": 1, "server_id": "se1", "episodes": [
|
||||
{"episode_number": 1, "title": "E1", "air_date": "2020-01-01",
|
||||
"file": {"relative_path": "e1.mkv", "size_bytes": 10}},
|
||||
{"episode_number": 2, "title": "E2", "air_date": "2020-01-08"}]}]}
|
||||
sid = db.upsert_show_tree("plex", item)
|
||||
with db.connect() as c:
|
||||
assert c.execute("SELECT COUNT(*) FROM episodes WHERE show_id=?", (sid,)).fetchone()[0] == 2
|
||||
assert c.execute("SELECT has_file FROM episodes WHERE show_id=? AND episode_number=1",
|
||||
(sid,)).fetchone()["has_file"] == 1
|
||||
assert c.execute("SELECT has_file FROM episodes WHERE show_id=? AND episode_number=2",
|
||||
(sid,)).fetchone()["has_file"] == 0
|
||||
# Re-scan with E2 removed from the server -> it gets pruned.
|
||||
item["seasons"][0]["episodes"] = item["seasons"][0]["episodes"][:1]
|
||||
assert db.upsert_show_tree("plex", item) == sid
|
||||
with db.connect() as c:
|
||||
eps = [r["episode_number"] for r in c.execute(
|
||||
"SELECT episode_number FROM episodes WHERE show_id=?", (sid,)).fetchall()]
|
||||
assert eps == [1]
|
||||
|
||||
|
||||
def test_prune_missing_removes_unseen_top_level(db):
|
||||
db.upsert_movie("plex", {"server_id": "a", "title": "A"})
|
||||
db.upsert_movie("plex", {"server_id": "b", "title": "B"})
|
||||
assert db.prune_missing("movies", "plex", {"a"}) == 1
|
||||
with db.connect() as c:
|
||||
ids = {r["server_id"] for r in c.execute("SELECT server_id FROM movies").fetchall()}
|
||||
assert ids == {"a"}
|
||||
|
||||
|
||||
# ── isolation: the video DB imports nothing from music ───────────────────────
|
||||
|
||||
def test_video_db_module_imports_nothing_from_music():
|
||||
|
|
|
|||
Loading…
Reference in a new issue