From 5e8143dd1d261504287efa04b694ba4cfb4c4db8 Mon Sep 17 00:00:00 2001 From: BoulderBadgeDad Date: Sun, 14 Jun 2026 17:33:38 -0700 Subject: [PATCH] video scan: survive legacy UNIQUE on tmdb_id/tvdb_id (store the row, drop the dup id) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Existing DBs created movies.tmdb_id / shows.tvdb_id as inline UNIQUE (can't be dropped via migration). The new model allows the same title in >1 library, so a second movie/show with the same id raised IntegrityError and the scanner SKIPPED it — dropping the title (observed: 'UNIQUE constraint failed: movies.tmdb_id', movie 548522 skipped). upsert_movie/upsert_show_tree now use a shared _resilient_upsert: on IntegrityError, retry WITHOUT the id columns so the row is stored (just without the colliding id) — same pattern enrichment_apply already used. Regression tests for both movies and shows under a simulated legacy unique index. --- database/video_database.py | 80 +++++++++++++++++++----------------- tests/test_video_database.py | 29 +++++++++++++ 2 files changed, 72 insertions(+), 37 deletions(-) diff --git a/database/video_database.py b/database/video_database.py index a6ef2a2c..ce02fa7b 100644 --- a/database/video_database.py +++ b/database/video_database.py @@ -435,28 +435,43 @@ class VideoDatabase: conn.execute(f"INSERT OR IGNORE INTO {link_table} ({owner_col}, genre_id) VALUES (?, ?)", (owner_id, gid)) + @staticmethod + def _resilient_upsert(conn, table: str, base: dict, id_cols: dict) -> None: + """INSERT…ON CONFLICT(server_source, server_id) for a movie/show row. + + Resilient to a LEGACY UNIQUE on tmdb_id/tvdb_id/imdb_id (old DBs created + before those were made non-unique — SQLite can't drop an inline UNIQUE): + on IntegrityError we retry WITHOUT the id columns, so the row is still + stored (same film in >1 library) instead of being dropped by the scan. + ``base`` holds the always-written cols; ``id_cols`` the droppable ids.""" + def run(include_ids): + cols = list(base.keys()) + (list(id_cols.keys()) if include_ids else []) + vals = list(base.values()) + (list(id_cols.values()) if include_ids else []) + updates = [c for c in cols if c not in ("server_source", "server_id")] + set_clause = ", ".join(f"{c}=excluded.{c}" for c in updates) + ", updated_at=CURRENT_TIMESTAMP" + sql = (f"INSERT INTO {table} ({', '.join(cols)}, updated_at) " + f"VALUES ({', '.join(['?'] * len(cols))}, CURRENT_TIMESTAMP) " + f"ON CONFLICT(server_source, server_id) DO UPDATE SET {set_clause}") + conn.execute(sql, vals) + try: + run(True) + except sqlite3.IntegrityError: + conn.rollback() # legacy UNIQUE on an id — keep the row, drop the id + run(False) + def upsert_movie(self, server_source: str, item: dict) -> int: """Insert/update one movie (keyed on server id) and its file. Returns row id.""" conn = self._get_connection() try: - conn.execute( - "INSERT INTO movies (server_source, server_id, title, sort_title, year, overview, " - "runtime_minutes, content_rating, studio, tagline, rating, rating_critic, " - "poster_url, tmdb_id, imdb_id, has_file, updated_at) " - "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, CURRENT_TIMESTAMP) " - "ON CONFLICT(server_source, server_id) DO UPDATE SET " - "title=excluded.title, sort_title=excluded.sort_title, year=excluded.year, " - "overview=excluded.overview, runtime_minutes=excluded.runtime_minutes, " - "content_rating=excluded.content_rating, studio=excluded.studio, " - "tagline=excluded.tagline, rating=excluded.rating, rating_critic=excluded.rating_critic, " - "poster_url=excluded.poster_url, tmdb_id=excluded.tmdb_id, imdb_id=excluded.imdb_id, " - "has_file=excluded.has_file, updated_at=CURRENT_TIMESTAMP", - (server_source, item["server_id"], item.get("title"), _sort_title(item.get("title")), - item.get("year"), item.get("overview"), item.get("runtime_minutes"), - item.get("content_rating"), item.get("studio"), item.get("tagline"), - item.get("rating"), item.get("rating_critic"), item.get("poster_url"), - item.get("tmdb_id"), item.get("imdb_id"), 1 if item.get("file") else 0), - ) + self._resilient_upsert(conn, "movies", { + "server_source": server_source, "server_id": item["server_id"], + "title": item.get("title"), "sort_title": _sort_title(item.get("title")), + "year": item.get("year"), "overview": item.get("overview"), + "runtime_minutes": item.get("runtime_minutes"), "content_rating": item.get("content_rating"), + "studio": item.get("studio"), "tagline": item.get("tagline"), + "rating": item.get("rating"), "rating_critic": item.get("rating_critic"), + "poster_url": item.get("poster_url"), "has_file": 1 if item.get("file") else 0, + }, {"tmdb_id": item.get("tmdb_id"), "imdb_id": item.get("imdb_id")}) movie_id = conn.execute( "SELECT id FROM movies WHERE server_source=? AND server_id=?", (server_source, item["server_id"]), @@ -474,25 +489,16 @@ class VideoDatabase: show are pruned. Returns the show row id.""" conn = self._get_connection() try: - conn.execute( - "INSERT INTO shows (server_source, server_id, title, sort_title, year, overview, status, " - "network, runtime_minutes, content_rating, tagline, rating, first_air_date, last_air_date, " - "poster_url, tvdb_id, tmdb_id, imdb_id, updated_at) " - "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, CURRENT_TIMESTAMP) " - "ON CONFLICT(server_source, server_id) DO UPDATE SET " - "title=excluded.title, sort_title=excluded.sort_title, year=excluded.year, " - "overview=excluded.overview, status=excluded.status, network=excluded.network, " - "runtime_minutes=excluded.runtime_minutes, content_rating=excluded.content_rating, " - "tagline=excluded.tagline, rating=excluded.rating, first_air_date=excluded.first_air_date, " - "last_air_date=excluded.last_air_date, " - "poster_url=excluded.poster_url, tvdb_id=excluded.tvdb_id, tmdb_id=excluded.tmdb_id, " - "imdb_id=excluded.imdb_id, updated_at=CURRENT_TIMESTAMP", - (server_source, item["server_id"], item.get("title"), _sort_title(item.get("title")), - item.get("year"), item.get("overview"), item.get("status"), item.get("network"), - item.get("runtime_minutes"), item.get("content_rating"), item.get("tagline"), - item.get("rating"), item.get("first_air_date"), item.get("last_air_date"), - item.get("poster_url"), item.get("tvdb_id"), item.get("tmdb_id"), item.get("imdb_id")), - ) + self._resilient_upsert(conn, "shows", { + "server_source": server_source, "server_id": item["server_id"], + "title": item.get("title"), "sort_title": _sort_title(item.get("title")), + "year": item.get("year"), "overview": item.get("overview"), + "status": item.get("status"), "network": item.get("network"), + "runtime_minutes": item.get("runtime_minutes"), "content_rating": item.get("content_rating"), + "tagline": item.get("tagline"), "rating": item.get("rating"), + "first_air_date": item.get("first_air_date"), "last_air_date": item.get("last_air_date"), + "poster_url": item.get("poster_url"), + }, {"tvdb_id": item.get("tvdb_id"), "tmdb_id": item.get("tmdb_id"), "imdb_id": item.get("imdb_id")}) show_id = conn.execute( "SELECT id FROM shows WHERE server_source=? AND server_id=?", (server_source, item["server_id"]), diff --git a/tests/test_video_database.py b/tests/test_video_database.py index c3ac555b..267a194c 100644 --- a/tests/test_video_database.py +++ b/tests/test_video_database.py @@ -480,6 +480,35 @@ def test_enrichment_apply_survives_legacy_unique(db): assert rb["tvdb_id"] is None and rb["tvdb_match_status"] == "matched" and rb["overview"] == "OB" +def test_upsert_movie_survives_legacy_unique_tmdb(db): + # Old DBs created movies.tmdb_id UNIQUE; the new model allows the same film in + # >1 library. The scan must STORE the second movie (dropping the colliding id), + # not skip it with an IntegrityError. + with db.connect() as c: + c.execute("CREATE UNIQUE INDEX ux_legacy_movies_tmdb ON movies(tmdb_id)") + c.commit() + a = db.upsert_movie("plex", {"server_id": "m1", "title": "A", "tmdb_id": 548522}) + b = db.upsert_movie("plex", {"server_id": "m2", "title": "B", "tmdb_id": 548522}) + assert a != b # both rows exist (not skipped) + with db.connect() as c: + ra = c.execute("SELECT tmdb_id FROM movies WHERE id=?", (a,)).fetchone() + rb = c.execute("SELECT title, tmdb_id FROM movies WHERE id=?", (b,)).fetchone() + assert ra["tmdb_id"] == 548522 + assert rb["title"] == "B" and rb["tmdb_id"] is None # kept the row, dropped the dup id + + +def test_upsert_show_survives_legacy_unique_tvdb(db): + with db.connect() as c: + c.execute("CREATE UNIQUE INDEX ux_legacy_shows_tvdb2 ON shows(tvdb_id)") + c.commit() + a = db.upsert_show_tree("plex", {"server_id": "s1", "title": "A", "tvdb_id": 9000, "seasons": []}) + b = db.upsert_show_tree("plex", {"server_id": "s2", "title": "B", "tvdb_id": 9000, "seasons": []}) + assert a != b + with db.connect() as c: + rb = c.execute("SELECT title, tvdb_id FROM shows WHERE id=?", (b,)).fetchone() + assert rb["title"] == "B" and rb["tvdb_id"] is None + + def test_enrichment_breakdown_unmatched_retry(db): a = db.upsert_movie("plex", {"server_id": "m1", "title": "A"}) b = db.upsert_movie("plex", {"server_id": "m2", "title": "B"})