danvics/soulsync
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 1

video scan: survive legacy UNIQUE on tmdb_id/tvdb_id (store the row, drop the dup id)

Browse source 
Existing DBs created movies.tmdb_id / shows.tvdb_id as inline UNIQUE (can't be
dropped via migration). The new model allows the same title in >1 library, so a
second movie/show with the same id raised IntegrityError and the scanner SKIPPED
it — dropping the title (observed: 'UNIQUE constraint failed: movies.tmdb_id',
movie 548522 skipped).

upsert_movie/upsert_show_tree now use a shared _resilient_upsert: on
IntegrityError, retry WITHOUT the id columns so the row is stored (just without
the colliding id) — same pattern enrichment_apply already used. Regression tests
for both movies and shows under a simulated legacy unique index.
This commit is contained in:
BoulderBadgeDad 2026-06-14 17:33:38 -07:00
parent 2306a5740c
commit 5e8143dd1d
2 changed files with 72 additions and 37 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

80
database/video_database.py
View file

@ -435,28 +435,43 @@ class VideoDatabase:
conn.execute(f"INSERT OR IGNORE INTO {link_table} ({owner_col}, genre_id) VALUES (?, ?)",
(owner_id, gid))
@staticmethod
def _resilient_upsert(conn, table: str, base: dict, id_cols: dict) -> None:
"""INSERT…ON CONFLICT(server_source, server_id) for a movie/show row.
Resilient to a LEGACY UNIQUE on tmdb_id/tvdb_id/imdb_id (old DBs created
before those were made non-unique — SQLite can't drop an inline UNIQUE):
on IntegrityError we retry WITHOUT the id columns, so the row is still
stored (same film in >1 library) instead of being dropped by the scan.
``base`` holds the always-written cols; ``id_cols`` the droppable ids."""
def run(include_ids):
cols = list(base.keys()) + (list(id_cols.keys()) if include_ids else [])
vals = list(base.values()) + (list(id_cols.values()) if include_ids else [])
updates = [c for c in cols if c not in ("server_source", "server_id")]
set_clause = ", ".join(f"{c}=excluded.{c}" for c in updates) + ", updated_at=CURRENT_TIMESTAMP"
sql = (f"INSERT INTO {table} ({', '.join(cols)}, updated_at) "
f"VALUES ({', '.join(['?'] * len(cols))}, CURRENT_TIMESTAMP) "
f"ON CONFLICT(server_source, server_id) DO UPDATE SET {set_clause}")
conn.execute(sql, vals)
try:
run(True)
except sqlite3.IntegrityError:
conn.rollback() # legacy UNIQUE on an id — keep the row, drop the id
run(False)
def upsert_movie(self, server_source: str, item: dict) -> int:
"""Insert/update one movie (keyed on server id) and its file. Returns row id."""
conn = self._get_connection()
try:
conn.execute(
"INSERT INTO movies (server_source, server_id, title, sort_title, year, overview, "
"runtime_minutes, content_rating, studio, tagline, rating, rating_critic, "
"poster_url, tmdb_id, imdb_id, has_file, updated_at) "
"VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, CURRENT_TIMESTAMP) "
"ON CONFLICT(server_source, server_id) DO UPDATE SET "
"title=excluded.title, sort_title=excluded.sort_title, year=excluded.year, "
"overview=excluded.overview, runtime_minutes=excluded.runtime_minutes, "
"content_rating=excluded.content_rating, studio=excluded.studio, "
"tagline=excluded.tagline, rating=excluded.rating, rating_critic=excluded.rating_critic, "
"poster_url=excluded.poster_url, tmdb_id=excluded.tmdb_id, imdb_id=excluded.imdb_id, "
"has_file=excluded.has_file, updated_at=CURRENT_TIMESTAMP",
(server_source, item["server_id"], item.get("title"), _sort_title(item.get("title")),
item.get("year"), item.get("overview"), item.get("runtime_minutes"),
item.get("content_rating"), item.get("studio"), item.get("tagline"),
item.get("rating"), item.get("rating_critic"), item.get("poster_url"),
item.get("tmdb_id"), item.get("imdb_id"), 1 if item.get("file") else 0),
)
self._resilient_upsert(conn, "movies", {
"server_source": server_source, "server_id": item["server_id"],
"title": item.get("title"), "sort_title": _sort_title(item.get("title")),
"year": item.get("year"), "overview": item.get("overview"),
"runtime_minutes": item.get("runtime_minutes"), "content_rating": item.get("content_rating"),
"studio": item.get("studio"), "tagline": item.get("tagline"),
"rating": item.get("rating"), "rating_critic": item.get("rating_critic"),
"poster_url": item.get("poster_url"), "has_file": 1 if item.get("file") else 0,
}, {"tmdb_id": item.get("tmdb_id"), "imdb_id": item.get("imdb_id")})
movie_id = conn.execute(
"SELECT id FROM movies WHERE server_source=? AND server_id=?",
(server_source, item["server_id"]),
@ -474,25 +489,16 @@ class VideoDatabase:
show are pruned. Returns the show row id."""
conn = self._get_connection()
try:
conn.execute(
"INSERT INTO shows (server_source, server_id, title, sort_title, year, overview, status, "
"network, runtime_minutes, content_rating, tagline, rating, first_air_date, last_air_date, "
"poster_url, tvdb_id, tmdb_id, imdb_id, updated_at) "
"VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, CURRENT_TIMESTAMP) "
"ON CONFLICT(server_source, server_id) DO UPDATE SET "
"title=excluded.title, sort_title=excluded.sort_title, year=excluded.year, "
"overview=excluded.overview, status=excluded.status, network=excluded.network, "
"runtime_minutes=excluded.runtime_minutes, content_rating=excluded.content_rating, "
"tagline=excluded.tagline, rating=excluded.rating, first_air_date=excluded.first_air_date, "
"last_air_date=excluded.last_air_date, "
"poster_url=excluded.poster_url, tvdb_id=excluded.tvdb_id, tmdb_id=excluded.tmdb_id, "
"imdb_id=excluded.imdb_id, updated_at=CURRENT_TIMESTAMP",
(server_source, item["server_id"], item.get("title"), _sort_title(item.get("title")),
item.get("year"), item.get("overview"), item.get("status"), item.get("network"),
item.get("runtime_minutes"), item.get("content_rating"), item.get("tagline"),
item.get("rating"), item.get("first_air_date"), item.get("last_air_date"),
item.get("poster_url"), item.get("tvdb_id"), item.get("tmdb_id"), item.get("imdb_id")),
)
self._resilient_upsert(conn, "shows", {
"server_source": server_source, "server_id": item["server_id"],
"title": item.get("title"), "sort_title": _sort_title(item.get("title")),
"year": item.get("year"), "overview": item.get("overview"),
"status": item.get("status"), "network": item.get("network"),
"runtime_minutes": item.get("runtime_minutes"), "content_rating": item.get("content_rating"),
"tagline": item.get("tagline"), "rating": item.get("rating"),
"first_air_date": item.get("first_air_date"), "last_air_date": item.get("last_air_date"),
"poster_url": item.get("poster_url"),
}, {"tvdb_id": item.get("tvdb_id"), "tmdb_id": item.get("tmdb_id"), "imdb_id": item.get("imdb_id")})
show_id = conn.execute(
"SELECT id FROM shows WHERE server_source=? AND server_id=?",
(server_source, item["server_id"]),

29
tests/test_video_database.py
View file

@ -480,6 +480,35 @@ def test_enrichment_apply_survives_legacy_unique(db):
assert rb["tvdb_id"] is None and rb["tvdb_match_status"] == "matched" and rb["overview"] == "OB"
def test_upsert_movie_survives_legacy_unique_tmdb(db):
# Old DBs created movies.tmdb_id UNIQUE; the new model allows the same film in
# >1 library. The scan must STORE the second movie (dropping the colliding id),
# not skip it with an IntegrityError.
with db.connect() as c:
c.execute("CREATE UNIQUE INDEX ux_legacy_movies_tmdb ON movies(tmdb_id)")
c.commit()
a = db.upsert_movie("plex", {"server_id": "m1", "title": "A", "tmdb_id": 548522})
b = db.upsert_movie("plex", {"server_id": "m2", "title": "B", "tmdb_id": 548522})
assert a != b # both rows exist (not skipped)
with db.connect() as c:
ra = c.execute("SELECT tmdb_id FROM movies WHERE id=?", (a,)).fetchone()
rb = c.execute("SELECT title, tmdb_id FROM movies WHERE id=?", (b,)).fetchone()
assert ra["tmdb_id"] == 548522
assert rb["title"] == "B" and rb["tmdb_id"] is None # kept the row, dropped the dup id
def test_upsert_show_survives_legacy_unique_tvdb(db):
with db.connect() as c:
c.execute("CREATE UNIQUE INDEX ux_legacy_shows_tvdb2 ON shows(tvdb_id)")
c.commit()
a = db.upsert_show_tree("plex", {"server_id": "s1", "title": "A", "tvdb_id": 9000, "seasons": []})
b = db.upsert_show_tree("plex", {"server_id": "s2", "title": "B", "tvdb_id": 9000, "seasons": []})
assert a != b
with db.connect() as c:
rb = c.execute("SELECT title, tvdb_id FROM shows WHERE id=?", (b,)).fetchone()
assert rb["title"] == "B" and rb["tvdb_id"] is None
def test_enrichment_breakdown_unmatched_retry(db):
a = db.upsert_movie("plex", {"server_id": "m1", "title": "A"})
b = db.upsert_movie("plex", {"server_id": "m2", "title": "B"})