From e952ea5f66943efea7f4711b7eebd331257e0bc2 Mon Sep 17 00:00:00 2001 From: BoulderBadgeDad Date: Wed, 17 Jun 2026 00:34:28 -0700 Subject: [PATCH] YouTube channels (2/4): schema bridge + DB methods Bridges YouTube onto the existing watchlist/wishlist tables (the chosen approach) via a generic source/source_id (+ parent_source_id on wishlist) and a stable surrogate for the NOT NULL tmdb_id, so existing dedup/group-by machinery is untouched. SCHEMA_VERSION 13, additive column migrations. - Channel follow = video_watchlist kind='channel' (source='youtube'). - Wished video = video_wishlist kind='video' grouped by parent channel. - youtube_surrogate_id(), add/remove/list/hydrate channels, add_videos_to_wishlist, query_youtube_wishlist (channel=group, videos=newest-first feed), youtube_wishlist_counts, scoped removal. - Kept wishlist_counts/watchlist_counts byte-identical (exact-equality tests); YouTube counts live on their own method. 80 DB tests green. --- database/video_database.py | 217 ++++++++++++++++++++++++++++++++++- database/video_schema.sql | 41 ++++--- tests/test_video_database.py | 99 ++++++++++++++++ 3 files changed, 342 insertions(+), 15 deletions(-) diff --git a/database/video_database.py b/database/video_database.py index 8576b295..bc30bdf9 100644 --- a/database/video_database.py +++ b/database/video_database.py @@ -16,6 +16,7 @@ verbatim on first init. from __future__ import annotations +import hashlib import os import re import sqlite3 @@ -29,7 +30,7 @@ logger = get_logger("video_database") # Bump when video_schema.sql changes in a way worth recording. Stored in # PRAGMA user_version as a backstop indicator (nothing gates on it yet). -SCHEMA_VERSION = 12 +SCHEMA_VERSION = 13 _DEFAULT_DB_PATH = "database/video_library.db" _SCHEMA_FILE = Path(__file__).resolve().parent / "video_schema.sql" @@ -102,9 +103,24 @@ _COLUMN_MIGRATIONS = [ ("video_wishlist", "still_url", "TEXT"), # episode still thumbnail (captured at add time) ("video_wishlist", "season_poster_url", "TEXT"), # the episode's season poster ("video_wishlist", "episode_overview", "TEXT"), # episode synopsis + # generic source bridge (YouTube channels/videos ride the existing tables) + ("video_watchlist", "source", "TEXT NOT NULL DEFAULT 'tmdb'"), + ("video_watchlist", "source_id", "TEXT"), + ("video_wishlist", "source", "TEXT NOT NULL DEFAULT 'tmdb'"), + ("video_wishlist", "source_id", "TEXT"), + ("video_wishlist", "parent_source_id", "TEXT"), # owning channel youtube id (video rows) ] +def youtube_surrogate_id(source_id: str) -> int: + """A stable positive 60-bit int derived from a YouTube id, used as the + NOT NULL ``tmdb_id`` surrogate for non-tmdb rows so the existing + UNIQUE(kind, tmdb_id) dedup + group-by machinery keeps working unchanged. + Collision probability across realistic channel counts is negligible.""" + h = hashlib.sha1((source_id or "").encode("utf-8")).hexdigest() + return int(h[:15], 16) # 60 bits — comfortably inside SQLite's signed 64-bit INTEGER + + class VideoDatabase: """Connection + schema manager for the isolated video library DB.""" @@ -1763,6 +1779,205 @@ class VideoDatabase: finally: conn.close() + # ── YouTube channels (bridged onto the watchlist/wishlist tables) ───────── + # A followed CHANNEL is a video_watchlist row (kind='channel', source='youtube', + # source_id=channel id). Its wished VIDEOS are video_wishlist rows + # (kind='video', source_id=video id, parent_source_id=channel id). tmdb_id on + # both carries the channel's surrogate so existing dedup/grouping just works. + def add_channel_to_watchlist(self, channel: dict) -> bool: + """Follow a YouTube channel. ``channel`` = {youtube_id, title, avatar_url?}. + Idempotent upsert on the channel surrogate. Returns True on success.""" + cid = (channel or {}).get("youtube_id") + title = (channel or {}).get("title") + if not cid or not title: + return False + conn = self._get_connection() + try: + conn.execute( + """INSERT INTO video_watchlist (kind, tmdb_id, title, poster_url, source, source_id, state) + VALUES ('channel', ?, ?, ?, 'youtube', ?, 'follow') + ON CONFLICT(kind, tmdb_id) DO UPDATE SET + state='follow', title=excluded.title, + poster_url=COALESCE(excluded.poster_url, video_watchlist.poster_url), + source='youtube', source_id=excluded.source_id""", + (youtube_surrogate_id(cid), title, channel.get("avatar_url"), cid)) + conn.commit() + return True + except Exception: + logger.exception("add_channel_to_watchlist failed (%s)", cid) + return False + finally: + conn.close() + + def remove_channel_from_watchlist(self, youtube_id: str) -> bool: + """Un-follow a channel (hard delete — channels have no airing-default to + guard against, so no tombstone). Its already-wished videos are left alone.""" + if not youtube_id: + return False + conn = self._get_connection() + try: + conn.execute("DELETE FROM video_watchlist WHERE kind='channel' AND source_id=?", (youtube_id,)) + conn.commit() + return True + finally: + conn.close() + + def list_watchlist_channels(self) -> list[dict]: + """Followed channels (newest first), each with a wished-video count for the card.""" + conn = self._get_connection() + try: + rows = conn.execute( + "SELECT w.title, w.poster_url, w.source_id, w.date_added, " + "(SELECT COUNT(*) FROM video_wishlist v WHERE v.kind='video' " + " AND v.parent_source_id = w.source_id) AS video_count " + "FROM video_watchlist w WHERE w.kind='channel' AND w.state='follow' " + "ORDER BY w.date_added DESC, w.id DESC").fetchall() + return [{"kind": "channel", "youtube_id": r["source_id"], "title": r["title"], + "poster_url": r["poster_url"], "video_count": r["video_count"], + "date_added": r["date_added"]} for r in rows] + finally: + conn.close() + + def channel_watch_state(self, youtube_ids) -> dict: + """{youtube_id: True} for followed channels — hydrates the Follow button.""" + out: dict = {} + ids = [str(x) for x in (youtube_ids or []) if x] + if not ids: + return out + conn = self._get_connection() + try: + for i in range(0, len(ids), 400): + chunk = ids[i:i + 400] + ph = ",".join("?" * len(chunk)) + for r in conn.execute( + f"SELECT source_id FROM video_watchlist WHERE kind='channel' " + f"AND state='follow' AND source_id IN ({ph})", chunk): + out[r["source_id"]] = True + return out + finally: + conn.close() + + def add_videos_to_wishlist(self, channel: dict, videos: list, *, server_source=None) -> int: + """Wish for a channel's videos. ``channel`` = {youtube_id, title, avatar_url?}; + ``videos`` = [{youtube_id, title, published_at?, thumbnail_url?, description?}, …]. + Idempotent per video id. Returns the count written.""" + cid = (channel or {}).get("youtube_id") + ctitle = (channel or {}).get("title") + if not cid or not ctitle or not videos: + return 0 + avatar = (channel or {}).get("avatar_url") + surrogate = youtube_surrogate_id(cid) + conn = self._get_connection() + n = 0 + try: + for v in videos: + vid = v.get("youtube_id") + if not vid: + continue + conn.execute( + """INSERT INTO video_wishlist + (kind, tmdb_id, title, poster_url, episode_title, still_url, + episode_overview, air_date, source, source_id, parent_source_id, server_source) + VALUES ('video', ?, ?, ?, ?, ?, ?, ?, 'youtube', ?, ?, ?) + ON CONFLICT(source_id) WHERE kind='video' DO UPDATE SET + title=excluded.title, + poster_url=COALESCE(excluded.poster_url, video_wishlist.poster_url), + episode_title=COALESCE(excluded.episode_title, video_wishlist.episode_title), + still_url=COALESCE(excluded.still_url, video_wishlist.still_url), + episode_overview=COALESCE(excluded.episode_overview, video_wishlist.episode_overview), + air_date=COALESCE(excluded.air_date, video_wishlist.air_date)""", + (surrogate, ctitle, avatar, v.get("title"), v.get("thumbnail_url"), + v.get("description"), v.get("published_at"), vid, cid, server_source)) + n += 1 + conn.commit() + return n + except Exception: + logger.exception("add_videos_to_wishlist failed (%s)", cid) + conn.rollback() + return 0 + finally: + conn.close() + + def remove_youtube_from_wishlist(self, scope: str, source_id: str) -> int: + """Remove wished videos: scope 'channel' (all of a channel, source_id=channel + id) or 'video' (one, source_id=video id). Returns rows removed.""" + if not source_id: + return 0 + if scope == "channel": + sql = "DELETE FROM video_wishlist WHERE kind='video' AND parent_source_id=?" + elif scope == "video": + sql = "DELETE FROM video_wishlist WHERE kind='video' AND source_id=?" + else: + return 0 + conn = self._get_connection() + try: + cur = conn.execute(sql, (source_id,)) + conn.commit() + return cur.rowcount + finally: + conn.close() + + def youtube_wishlist_counts(self) -> dict: + """{'channel': n distinct channels, 'video': n videos} in the wishlist.""" + conn = self._get_connection() + try: + video = conn.execute("SELECT COUNT(*) c FROM video_wishlist WHERE kind='video'").fetchone()["c"] + channel = conn.execute( + "SELECT COUNT(DISTINCT parent_source_id) c FROM video_wishlist WHERE kind='video'").fetchone()["c"] + return {"channel": channel, "video": video} + finally: + conn.close() + + def query_youtube_wishlist(self, *, search=None, sort="added", page=1, limit=60) -> dict: + """Wished YouTube videos grouped by channel (channel = orb, videos = flat + newest-first feed). ``sort`` ∈ added | oldest | title | wanted. Mirrors the + {items, pagination} shape of query_wishlist.""" + try: + page = max(1, int(page or 1)) + limit = max(1, min(200, int(limit or 60))) + except (TypeError, ValueError): + page, limit = 1, 60 + s = (search or "").strip() + conn = self._get_connection() + try: + where, args = ["kind='video'"], [] + if s: + where.append("(title LIKE ? COLLATE NOCASE OR episode_title LIKE ? COLLATE NOCASE)") + args += ["%" + s + "%", "%" + s + "%"] + wsql = " WHERE " + " AND ".join(where) + total = conn.execute( + "SELECT COUNT(DISTINCT parent_source_id) c FROM video_wishlist" + wsql, args).fetchone()["c"] + order = {"title": "title COLLATE NOCASE", "wanted": "video_count DESC, last_added DESC", + "oldest": "last_added ASC", "added": "last_added DESC"}.get(sort, "last_added DESC") + chan_rows = conn.execute( + "SELECT parent_source_id, MAX(title) AS title, MAX(poster_url) AS poster_url, " + "COUNT(*) AS video_count, " + "SUM(CASE WHEN status='downloaded' THEN 1 ELSE 0 END) AS done, " + "MAX(date_added) AS last_added " + "FROM video_wishlist" + wsql + + " GROUP BY parent_source_id ORDER BY " + order + " LIMIT ? OFFSET ?", + args + [limit, (page - 1) * limit]).fetchall() + items = [] + for cr in chan_rows: + vids = conn.execute( + "SELECT source_id, episode_title, still_url, episode_overview, air_date, status " + "FROM video_wishlist WHERE kind='video' AND parent_source_id=? " + "ORDER BY (air_date IS NULL), air_date DESC, id DESC", (cr["parent_source_id"],)).fetchall() + items.append({ + "kind": "channel", "youtube_id": cr["parent_source_id"], "title": cr["title"], + "poster_url": cr["poster_url"], "video_count": cr["video_count"], + "done": cr["done"] or 0, + "videos": [{"youtube_id": v["source_id"], "title": v["episode_title"], + "still_url": v["still_url"], "overview": v["episode_overview"], + "published_at": v["air_date"], "status": v["status"]} for v in vids]}) + finally: + conn.close() + total_pages = max(1, (total + limit - 1) // limit) + page = min(page, total_pages) + return {"items": items, "pagination": { + "page": page, "total_pages": total_pages, "total_count": total, + "has_prev": page > 1, "has_next": page < total_pages}} + def movie_detail(self, movie_id: int) -> dict | None: """Full movie detail: the movie + owned/file info. Drives the (isolated) video movie-detail page.""" diff --git a/database/video_schema.sql b/database/video_schema.sql index 402d1433..a02530a2 100644 --- a/database/video_schema.sql +++ b/database/video_schema.sql @@ -338,11 +338,15 @@ CREATE INDEX IF NOT EXISTS idx_activity_created ON activity(created_at); -- later phase — this table just records membership + enough to render + link. CREATE TABLE IF NOT EXISTS video_watchlist ( id INTEGER PRIMARY KEY AUTOINCREMENT, - kind TEXT NOT NULL, -- 'show' | 'person' - tmdb_id INTEGER NOT NULL, - title TEXT NOT NULL, -- show title / person name - poster_url TEXT, -- poster (show) / photo (person) + kind TEXT NOT NULL, -- 'show' | 'person' | 'channel' (youtube) + tmdb_id INTEGER NOT NULL, -- tmdb id; for non-tmdb sources a stable surrogate of source_id + title TEXT NOT NULL, -- show title / person name / channel title + poster_url TEXT, -- poster (show) / photo (person) / avatar (channel) library_id INTEGER, -- shows.id when owned (else NULL) + -- generic source bridge: 'tmdb' (default) or 'youtube'; source_id = native id + -- (channel youtube id) for non-tmdb rows. One table, both worlds. + source TEXT NOT NULL DEFAULT 'tmdb', + source_id TEXT, -- 'follow' = explicit user follow. 'mute' = a TOMBSTONE: the user -- un-followed something that is on the watchlist by default (an actively -- airing library show), so the default must not re-add it. Library shows @@ -359,30 +363,39 @@ CREATE INDEX IF NOT EXISTS idx_video_watchlist_kind ON video_watchlist(kind); -- so the wishlist only ever holds things you can actually acquire right now. CREATE TABLE IF NOT EXISTS video_wishlist ( id INTEGER PRIMARY KEY AUTOINCREMENT, - kind TEXT NOT NULL, -- 'movie' | 'episode' - tmdb_id INTEGER NOT NULL, -- movie's tmdb id | the SHOW's tmdb id (episode rows) - title TEXT NOT NULL, -- movie title | show title - poster_url TEXT, -- movie/show poster + kind TEXT NOT NULL, -- 'movie' | 'episode' | 'video' (youtube) + tmdb_id INTEGER NOT NULL, -- movie's tmdb id | the SHOW's tmdb id (episode) | channel surrogate (video) + title TEXT NOT NULL, -- movie title | show title | channel title (video rows) + poster_url TEXT, -- movie/show poster | channel avatar (video rows) year INTEGER, -- movie year (movie rows) season_number INTEGER, -- episode rows episode_number INTEGER, -- episode rows - episode_title TEXT, -- episode rows - still_url TEXT, -- episode still thumbnail (episode rows) - episode_overview TEXT, -- episode synopsis (episode rows) + episode_title TEXT, -- episode rows | video title (video rows) + still_url TEXT, -- episode still | video thumbnail (video rows) + episode_overview TEXT, -- episode synopsis | video description (video rows) season_poster_url TEXT, -- the episode's SEASON poster (episode rows) - air_date TEXT, -- episode rows (already aired by the time it's here) + air_date TEXT, -- episode air date | video published_at (video rows) status TEXT NOT NULL DEFAULT 'wanted', -- wanted|searching|downloading|downloaded|failed library_id INTEGER, -- owned movies.id/shows.id when re-downloading server_source TEXT, -- server context that added it (informational) + -- generic source bridge (mirrors video_watchlist). For 'video' rows: + -- source='youtube', source_id=video youtube id, parent_source_id=channel youtube id. + source TEXT NOT NULL DEFAULT 'tmdb', + source_id TEXT, + parent_source_id TEXT, -- owning channel's youtube id (video rows) date_added TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP ); --- one row per movie, one per (show, season, episode) — partial uniques so the two --- shapes don't collide and re-adding is an idempotent upsert. +-- one row per movie, one per (show, season, episode), one per youtube video — +-- partial uniques so the shapes don't collide and re-adding is an idempotent upsert. CREATE UNIQUE INDEX IF NOT EXISTS idx_video_wishlist_movie ON video_wishlist(tmdb_id) WHERE kind = 'movie'; CREATE UNIQUE INDEX IF NOT EXISTS idx_video_wishlist_episode ON video_wishlist(tmdb_id, season_number, episode_number) WHERE kind = 'episode'; CREATE INDEX IF NOT EXISTS idx_video_wishlist_show ON video_wishlist(tmdb_id) WHERE kind = 'episode'; +CREATE UNIQUE INDEX IF NOT EXISTS idx_video_wishlist_video + ON video_wishlist(source_id) WHERE kind = 'video'; +CREATE INDEX IF NOT EXISTS idx_video_wishlist_channel + ON video_wishlist(parent_source_id) WHERE kind = 'video'; -- ── Derived views: Watchlist / Wishlist / Calendar ────────────────────────── -- WATCHLIST = things you follow for NEW content: monitored shows + channels. diff --git a/tests/test_video_database.py b/tests/test_video_database.py index ffd9feaf..a983d393 100644 --- a/tests/test_video_database.py +++ b/tests/test_video_database.py @@ -989,3 +989,102 @@ def test_wishlist_episode_overview_roundtrips_and_backfills(db): assert (1396, 1) in {(t["tmdb_id"], t["season_number"]) for t in db.wishlist_art_backfill_targets()} assert db.set_wishlist_episode_overview(1396, 1, 2, "Filled in.") is True assert db.set_wishlist_episode_overview(1396, 1, 1, "nope") is False # won't clobber + + +# ── YouTube channels (bridged onto watchlist/wishlist) ─────────────────────── + +from database.video_database import youtube_surrogate_id + + +def test_youtube_surrogate_id_is_stable_and_distinct(): + a = youtube_surrogate_id("UCPlayStation") + assert a == youtube_surrogate_id("UCPlayStation") # deterministic + assert a > 0 and a < (1 << 63) # fits SQLite INTEGER + assert a != youtube_surrogate_id("UCGoodMythical") # distinct ids → distinct + + +def test_follow_channel_lists_and_hydrates(db): + ch = {"youtube_id": "UCPlay", "title": "PlayStation", "avatar_url": "http://a/p.jpg"} + assert db.add_channel_to_watchlist(ch) is True + chans = db.list_watchlist_channels() + assert len(chans) == 1 + assert chans[0]["youtube_id"] == "UCPlay" and chans[0]["title"] == "PlayStation" + assert chans[0]["poster_url"] == "http://a/p.jpg" and chans[0]["video_count"] == 0 + # hydration + assert db.channel_watch_state(["UCPlay", "UCnope"]) == {"UCPlay": True} + # idempotent re-follow refreshes title, no duplicate + db.add_channel_to_watchlist({"youtube_id": "UCPlay", "title": "PlayStation US"}) + chans = db.list_watchlist_channels() + assert len(chans) == 1 and chans[0]["title"] == "PlayStation US" + + +def test_unfollow_channel_removes_row(db): + db.add_channel_to_watchlist({"youtube_id": "UCPlay", "title": "PlayStation"}) + assert db.remove_channel_from_watchlist("UCPlay") is True + assert db.list_watchlist_channels() == [] + assert db.channel_watch_state(["UCPlay"]) == {} + + +def test_add_videos_groups_under_channel_newest_first(db): + ch = {"youtube_id": "UCPlay", "title": "PlayStation", "avatar_url": "http://a/p.jpg"} + vids = [ + {"youtube_id": "v1", "title": "Old Trailer", "published_at": "2023-01-01", + "thumbnail_url": "http://t/1.jpg", "description": "older"}, + {"youtube_id": "v2", "title": "New State of Play", "published_at": "2024-06-01", + "thumbnail_url": "http://t/2.jpg", "description": "newer"}, + {"youtube_id": "v3", "title": "Undated", "thumbnail_url": "http://t/3.jpg"}, + ] + assert db.add_videos_to_wishlist(ch, vids) == 3 + res = db.query_youtube_wishlist() + assert res["pagination"]["total_count"] == 1 + grp = res["items"][0] + assert grp["youtube_id"] == "UCPlay" and grp["title"] == "PlayStation" + assert grp["poster_url"] == "http://a/p.jpg" and grp["video_count"] == 3 + # newest-first; dated before undated + assert [v["youtube_id"] for v in grp["videos"]] == ["v2", "v1", "v3"] + v2 = grp["videos"][0] + assert v2["title"] == "New State of Play" and v2["still_url"] == "http://t/2.jpg" + assert v2["overview"] == "newer" and v2["published_at"] == "2024-06-01" + assert v2["status"] == "wanted" + + +def test_add_videos_is_idempotent_per_video(db): + ch = {"youtube_id": "UCPlay", "title": "PlayStation"} + db.add_videos_to_wishlist(ch, [{"youtube_id": "v1", "title": "A"}]) + db.add_videos_to_wishlist(ch, [{"youtube_id": "v1", "title": "A (updated)"}, + {"youtube_id": "v2", "title": "B"}]) + grp = db.query_youtube_wishlist()["items"][0] + assert grp["video_count"] == 2 + titles = {v["youtube_id"]: v["title"] for v in grp["videos"]} + assert titles == {"v1": "A (updated)", "v2": "B"} + + +def test_youtube_counts_and_removal_scopes(db): + a = {"youtube_id": "UCa", "title": "Chan A"} + b = {"youtube_id": "UCb", "title": "Chan B"} + db.add_videos_to_wishlist(a, [{"youtube_id": "a1", "title": "A1"}, + {"youtube_id": "a2", "title": "A2"}]) + db.add_videos_to_wishlist(b, [{"youtube_id": "b1", "title": "B1"}]) + assert db.youtube_wishlist_counts() == {"channel": 2, "video": 3} + # remove one video + assert db.remove_youtube_from_wishlist("video", "a1") == 1 + assert db.youtube_wishlist_counts() == {"channel": 2, "video": 2} + # remove a whole channel + assert db.remove_youtube_from_wishlist("channel", "UCa") == 1 # only a2 left + assert db.youtube_wishlist_counts() == {"channel": 1, "video": 1} + + +def test_youtube_rows_do_not_disturb_tmdb_counts(db): + """The bridge must not leak into the existing movie/episode + show/person counts.""" + db.add_movie_to_wishlist(101, "A Movie", year=2020) + db.add_episodes_to_wishlist(202, "A Show", [{"season_number": 1, "episode_number": 1}]) + db.add_to_watchlist("show", 303, "Watched Show") + db.add_channel_to_watchlist({"youtube_id": "UCx", "title": "Chan"}) + db.add_videos_to_wishlist({"youtube_id": "UCx", "title": "Chan"}, + [{"youtube_id": "x1", "title": "X1"}]) + # existing shapes unchanged + assert db.wishlist_counts() == {"movie": 1, "show": 1, "episode": 1, "total": 2} + assert db.watchlist_counts() == {"show": 1, "person": 0, "total": 1} + # youtube counts live on their own surface + assert db.youtube_wishlist_counts() == {"channel": 1, "video": 1} + assert db.list_watchlist_channels()[0]["video_count"] == 1