danvics/soulsync
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 1

YouTube channels (2/4): schema bridge + DB methods

Browse source 
Bridges YouTube onto the existing watchlist/wishlist tables (the chosen
approach) via a generic source/source_id (+ parent_source_id on wishlist) and a
stable surrogate for the NOT NULL tmdb_id, so existing dedup/group-by machinery
is untouched. SCHEMA_VERSION 13, additive column migrations.

- Channel follow  = video_watchlist kind='channel' (source='youtube').
- Wished video    = video_wishlist  kind='video' grouped by parent channel.
- youtube_surrogate_id(), add/remove/list/hydrate channels, add_videos_to_wishlist,
  query_youtube_wishlist (channel=group, videos=newest-first feed),
  youtube_wishlist_counts, scoped removal.
- Kept wishlist_counts/watchlist_counts byte-identical (exact-equality tests);
  YouTube counts live on their own method. 80 DB tests green.
This commit is contained in:
BoulderBadgeDad 2026-06-17 00:34:28 -07:00
parent 68383a16b3
commit e952ea5f66
3 changed files with 342 additions and 15 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

217
database/video_database.py
View file

@ -16,6 +16,7 @@ verbatim on first init.
from __future__ import annotations
import hashlib
import os
import re
import sqlite3
@ -29,7 +30,7 @@ logger = get_logger("video_database")
# Bump when video_schema.sql changes in a way worth recording. Stored in
# PRAGMA user_version as a backstop indicator (nothing gates on it yet).
SCHEMA_VERSION = 12
SCHEMA_VERSION = 13
_DEFAULT_DB_PATH = "database/video_library.db"
_SCHEMA_FILE = Path(__file__).resolve().parent / "video_schema.sql"
@ -102,9 +103,24 @@ _COLUMN_MIGRATIONS = [
("video_wishlist", "still_url", "TEXT"), # episode still thumbnail (captured at add time)
("video_wishlist", "season_poster_url", "TEXT"), # the episode's season poster
("video_wishlist", "episode_overview", "TEXT"), # episode synopsis
# generic source bridge (YouTube channels/videos ride the existing tables)
("video_watchlist", "source", "TEXT NOT NULL DEFAULT 'tmdb'"),
("video_watchlist", "source_id", "TEXT"),
("video_wishlist", "source", "TEXT NOT NULL DEFAULT 'tmdb'"),
("video_wishlist", "source_id", "TEXT"),
("video_wishlist", "parent_source_id", "TEXT"), # owning channel youtube id (video rows)
]
def youtube_surrogate_id(source_id: str) -> int:
"""A stable positive 60-bit int derived from a YouTube id, used as the
NOT NULL ``tmdb_id`` surrogate for non-tmdb rows so the existing
UNIQUE(kind, tmdb_id) dedup + group-by machinery keeps working unchanged.
Collision probability across realistic channel counts is negligible."""
h = hashlib.sha1((source_id or "").encode("utf-8")).hexdigest()
return int(h[:15], 16) # 60 bits — comfortably inside SQLite's signed 64-bit INTEGER
class VideoDatabase:
"""Connection + schema manager for the isolated video library DB."""
@ -1763,6 +1779,205 @@ class VideoDatabase:
finally:
conn.close()
# ── YouTube channels (bridged onto the watchlist/wishlist tables) ─────────
# A followed CHANNEL is a video_watchlist row (kind='channel', source='youtube',
# source_id=channel id). Its wished VIDEOS are video_wishlist rows
# (kind='video', source_id=video id, parent_source_id=channel id). tmdb_id on
# both carries the channel's surrogate so existing dedup/grouping just works.
def add_channel_to_watchlist(self, channel: dict) -> bool:
"""Follow a YouTube channel. ``channel`` = {youtube_id, title, avatar_url?}.
Idempotent upsert on the channel surrogate. Returns True on success."""
cid = (channel or {}).get("youtube_id")
title = (channel or {}).get("title")
if not cid or not title:
return False
conn = self._get_connection()
try:
conn.execute(
"""INSERT INTO video_watchlist (kind, tmdb_id, title, poster_url, source, source_id, state)
VALUES ('channel', ?, ?, ?, 'youtube', ?, 'follow')
ON CONFLICT(kind, tmdb_id) DO UPDATE SET
state='follow', title=excluded.title,
poster_url=COALESCE(excluded.poster_url, video_watchlist.poster_url),
source='youtube', source_id=excluded.source_id""",
(youtube_surrogate_id(cid), title, channel.get("avatar_url"), cid))
conn.commit()
return True
except Exception:
logger.exception("add_channel_to_watchlist failed (%s)", cid)
return False
finally:
conn.close()
def remove_channel_from_watchlist(self, youtube_id: str) -> bool:
"""Un-follow a channel (hard delete — channels have no airing-default to
guard against, so no tombstone). Its already-wished videos are left alone."""
if not youtube_id:
return False
conn = self._get_connection()
try:
conn.execute("DELETE FROM video_watchlist WHERE kind='channel' AND source_id=?", (youtube_id,))
conn.commit()
return True
finally:
conn.close()
def list_watchlist_channels(self) -> list[dict]:
"""Followed channels (newest first), each with a wished-video count for the card."""
conn = self._get_connection()
try:
rows = conn.execute(
"SELECT w.title, w.poster_url, w.source_id, w.date_added, "
"(SELECT COUNT(*) FROM video_wishlist v WHERE v.kind='video' "
" AND v.parent_source_id = w.source_id) AS video_count "
"FROM video_watchlist w WHERE w.kind='channel' AND w.state='follow' "
"ORDER BY w.date_added DESC, w.id DESC").fetchall()
return [{"kind": "channel", "youtube_id": r["source_id"], "title": r["title"],
"poster_url": r["poster_url"], "video_count": r["video_count"],
"date_added": r["date_added"]} for r in rows]
finally:
conn.close()
def channel_watch_state(self, youtube_ids) -> dict:
"""{youtube_id: True} for followed channels — hydrates the Follow button."""
out: dict = {}
ids = [str(x) for x in (youtube_ids or []) if x]
if not ids:
return out
conn = self._get_connection()
try:
for i in range(0, len(ids), 400):
chunk = ids[i:i + 400]
ph = ",".join("?" * len(chunk))
for r in conn.execute(
f"SELECT source_id FROM video_watchlist WHERE kind='channel' "
f"AND state='follow' AND source_id IN ({ph})", chunk):
out[r["source_id"]] = True
return out
finally:
conn.close()
def add_videos_to_wishlist(self, channel: dict, videos: list, *, server_source=None) -> int:
"""Wish for a channel's videos. ``channel`` = {youtube_id, title, avatar_url?};
``videos`` = [{youtube_id, title, published_at?, thumbnail_url?, description?}, …].
Idempotent per video id. Returns the count written."""
cid = (channel or {}).get("youtube_id")
ctitle = (channel or {}).get("title")
if not cid or not ctitle or not videos:
return 0
avatar = (channel or {}).get("avatar_url")
surrogate = youtube_surrogate_id(cid)
conn = self._get_connection()
n = 0
try:
for v in videos:
vid = v.get("youtube_id")
if not vid:
continue
conn.execute(
"""INSERT INTO video_wishlist
(kind, tmdb_id, title, poster_url, episode_title, still_url,
episode_overview, air_date, source, source_id, parent_source_id, server_source)
VALUES ('video', ?, ?, ?, ?, ?, ?, ?, 'youtube', ?, ?, ?)
ON CONFLICT(source_id) WHERE kind='video' DO UPDATE SET
title=excluded.title,
poster_url=COALESCE(excluded.poster_url, video_wishlist.poster_url),
episode_title=COALESCE(excluded.episode_title, video_wishlist.episode_title),
still_url=COALESCE(excluded.still_url, video_wishlist.still_url),
episode_overview=COALESCE(excluded.episode_overview, video_wishlist.episode_overview),
air_date=COALESCE(excluded.air_date, video_wishlist.air_date)""",
(surrogate, ctitle, avatar, v.get("title"), v.get("thumbnail_url"),
v.get("description"), v.get("published_at"), vid, cid, server_source))
n += 1
conn.commit()
return n
except Exception:
logger.exception("add_videos_to_wishlist failed (%s)", cid)
conn.rollback()
return 0
finally:
conn.close()
def remove_youtube_from_wishlist(self, scope: str, source_id: str) -> int:
"""Remove wished videos: scope 'channel' (all of a channel, source_id=channel
id) or 'video' (one, source_id=video id). Returns rows removed."""
if not source_id:
return 0
if scope == "channel":
sql = "DELETE FROM video_wishlist WHERE kind='video' AND parent_source_id=?"
elif scope == "video":
sql = "DELETE FROM video_wishlist WHERE kind='video' AND source_id=?"
else:
return 0
conn = self._get_connection()
try:
cur = conn.execute(sql, (source_id,))
conn.commit()
return cur.rowcount
finally:
conn.close()
def youtube_wishlist_counts(self) -> dict:
"""{'channel': n distinct channels, 'video': n videos} in the wishlist."""
conn = self._get_connection()
try:
video = conn.execute("SELECT COUNT(*) c FROM video_wishlist WHERE kind='video'").fetchone()["c"]
channel = conn.execute(
"SELECT COUNT(DISTINCT parent_source_id) c FROM video_wishlist WHERE kind='video'").fetchone()["c"]
return {"channel": channel, "video": video}
finally:
conn.close()
def query_youtube_wishlist(self, *, search=None, sort="added", page=1, limit=60) -> dict:
"""Wished YouTube videos grouped by channel (channel = orb, videos = flat
newest-first feed). ``sort`` ∈ added | oldest | title | wanted. Mirrors the
{items, pagination} shape of query_wishlist."""
try:
page = max(1, int(page or 1))
limit = max(1, min(200, int(limit or 60)))
except (TypeError, ValueError):
page, limit = 1, 60
s = (search or "").strip()
conn = self._get_connection()
try:
where, args = ["kind='video'"], []
if s:
where.append("(title LIKE ? COLLATE NOCASE OR episode_title LIKE ? COLLATE NOCASE)")
args += ["%" + s + "%", "%" + s + "%"]
wsql = " WHERE " + " AND ".join(where)
total = conn.execute(
"SELECT COUNT(DISTINCT parent_source_id) c FROM video_wishlist" + wsql, args).fetchone()["c"]
order = {"title": "title COLLATE NOCASE", "wanted": "video_count DESC, last_added DESC",
"oldest": "last_added ASC", "added": "last_added DESC"}.get(sort, "last_added DESC")
chan_rows = conn.execute(
"SELECT parent_source_id, MAX(title) AS title, MAX(poster_url) AS poster_url, "
"COUNT(*) AS video_count, "
"SUM(CASE WHEN status='downloaded' THEN 1 ELSE 0 END) AS done, "
"MAX(date_added) AS last_added "
"FROM video_wishlist" + wsql +
" GROUP BY parent_source_id ORDER BY " + order + " LIMIT ? OFFSET ?",
args + [limit, (page - 1) * limit]).fetchall()
items = []
for cr in chan_rows:
vids = conn.execute(
"SELECT source_id, episode_title, still_url, episode_overview, air_date, status "
"FROM video_wishlist WHERE kind='video' AND parent_source_id=? "
"ORDER BY (air_date IS NULL), air_date DESC, id DESC", (cr["parent_source_id"],)).fetchall()
items.append({
"kind": "channel", "youtube_id": cr["parent_source_id"], "title": cr["title"],
"poster_url": cr["poster_url"], "video_count": cr["video_count"],
"done": cr["done"] or 0,
"videos": [{"youtube_id": v["source_id"], "title": v["episode_title"],
"still_url": v["still_url"], "overview": v["episode_overview"],
"published_at": v["air_date"], "status": v["status"]} for v in vids]})
finally:
conn.close()
total_pages = max(1, (total + limit - 1) // limit)
page = min(page, total_pages)
return {"items": items, "pagination": {
"page": page, "total_pages": total_pages, "total_count": total,
"has_prev": page > 1, "has_next": page < total_pages}}
def movie_detail(self, movie_id: int) -> dict | None:
"""Full movie detail: the movie + owned/file info. Drives the (isolated)
video movie-detail page."""

41
database/video_schema.sql
View file

@ -338,11 +338,15 @@ CREATE INDEX IF NOT EXISTS idx_activity_created ON activity(created_at);
-- later phase — this table just records membership + enough to render + link.
CREATE TABLE IF NOT EXISTS video_watchlist (
id INTEGER PRIMARY KEY AUTOINCREMENT,
kind TEXT NOT NULL, -- 'show' | 'person'
tmdb_id INTEGER NOT NULL,
title TEXT NOT NULL, -- show title / person name
poster_url TEXT, -- poster (show) / photo (person)
kind TEXT NOT NULL, -- 'show' | 'person' | 'channel' (youtube)
tmdb_id INTEGER NOT NULL, -- tmdb id; for non-tmdb sources a stable surrogate of source_id
title TEXT NOT NULL, -- show title / person name / channel title
poster_url TEXT, -- poster (show) / photo (person) / avatar (channel)
library_id INTEGER, -- shows.id when owned (else NULL)
-- generic source bridge: 'tmdb' (default) or 'youtube'; source_id = native id
-- (channel youtube id) for non-tmdb rows. One table, both worlds.
source TEXT NOT NULL DEFAULT 'tmdb',
source_id TEXT,
-- 'follow' = explicit user follow. 'mute' = a TOMBSTONE: the user
-- un-followed something that is on the watchlist by default (an actively
-- airing library show), so the default must not re-add it. Library shows
@ -359,30 +363,39 @@ CREATE INDEX IF NOT EXISTS idx_video_watchlist_kind ON video_watchlist(kind);
-- so the wishlist only ever holds things you can actually acquire right now.
CREATE TABLE IF NOT EXISTS video_wishlist (
id INTEGER PRIMARY KEY AUTOINCREMENT,
kind TEXT NOT NULL, -- 'movie' | 'episode'
tmdb_id INTEGER NOT NULL, -- movie's tmdb id | the SHOW's tmdb id (episode rows)
title TEXT NOT NULL, -- movie title | show title
poster_url TEXT, -- movie/show poster
kind TEXT NOT NULL, -- 'movie' | 'episode' | 'video' (youtube)
tmdb_id INTEGER NOT NULL, -- movie's tmdb id | the SHOW's tmdb id (episode) | channel surrogate (video)
title TEXT NOT NULL, -- movie title | show title | channel title (video rows)
poster_url TEXT, -- movie/show poster | channel avatar (video rows)
year INTEGER, -- movie year (movie rows)
season_number INTEGER, -- episode rows
episode_number INTEGER, -- episode rows
episode_title TEXT, -- episode rows
still_url TEXT, -- episode still thumbnail (episode rows)
episode_overview TEXT, -- episode synopsis (episode rows)
episode_title TEXT, -- episode rows | video title (video rows)
still_url TEXT, -- episode still | video thumbnail (video rows)
episode_overview TEXT, -- episode synopsis | video description (video rows)
season_poster_url TEXT, -- the episode's SEASON poster (episode rows)
air_date TEXT, -- episode rows (already aired by the time it's here)
air_date TEXT, -- episode air date | video published_at (video rows)
status TEXT NOT NULL DEFAULT 'wanted', -- wanted|searching|downloading|downloaded|failed
library_id INTEGER, -- owned movies.id/shows.id when re-downloading
server_source TEXT, -- server context that added it (informational)
-- generic source bridge (mirrors video_watchlist). For 'video' rows:
-- source='youtube', source_id=video youtube id, parent_source_id=channel youtube id.
source TEXT NOT NULL DEFAULT 'tmdb',
source_id TEXT,
parent_source_id TEXT, -- owning channel's youtube id (video rows)
date_added TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP
);
-- one row per movie, one per (show, season, episode) — partial uniques so the two
-- shapes don't collide and re-adding is an idempotent upsert.
-- one row per movie, one per (show, season, episode), one per youtube video —
-- partial uniques so the shapes don't collide and re-adding is an idempotent upsert.
CREATE UNIQUE INDEX IF NOT EXISTS idx_video_wishlist_movie
ON video_wishlist(tmdb_id) WHERE kind = 'movie';
CREATE UNIQUE INDEX IF NOT EXISTS idx_video_wishlist_episode
ON video_wishlist(tmdb_id, season_number, episode_number) WHERE kind = 'episode';
CREATE INDEX IF NOT EXISTS idx_video_wishlist_show ON video_wishlist(tmdb_id) WHERE kind = 'episode';
CREATE UNIQUE INDEX IF NOT EXISTS idx_video_wishlist_video
ON video_wishlist(source_id) WHERE kind = 'video';
CREATE INDEX IF NOT EXISTS idx_video_wishlist_channel
ON video_wishlist(parent_source_id) WHERE kind = 'video';
-- ── Derived views: Watchlist / Wishlist / Calendar ──────────────────────────
-- WATCHLIST = things you follow for NEW content: monitored shows + channels.

99
tests/test_video_database.py
View file

@ -989,3 +989,102 @@ def test_wishlist_episode_overview_roundtrips_and_backfills(db):
assert (1396, 1) in {(t["tmdb_id"], t["season_number"]) for t in db.wishlist_art_backfill_targets()}
assert db.set_wishlist_episode_overview(1396, 1, 2, "Filled in.") is True
assert db.set_wishlist_episode_overview(1396, 1, 1, "nope") is False # won't clobber
# ── YouTube channels (bridged onto watchlist/wishlist) ───────────────────────
from database.video_database import youtube_surrogate_id
def test_youtube_surrogate_id_is_stable_and_distinct():
a = youtube_surrogate_id("UCPlayStation")
assert a == youtube_surrogate_id("UCPlayStation") # deterministic
assert a > 0 and a < (1 << 63) # fits SQLite INTEGER
assert a != youtube_surrogate_id("UCGoodMythical") # distinct ids → distinct
def test_follow_channel_lists_and_hydrates(db):
ch = {"youtube_id": "UCPlay", "title": "PlayStation", "avatar_url": "http://a/p.jpg"}
assert db.add_channel_to_watchlist(ch) is True
chans = db.list_watchlist_channels()
assert len(chans) == 1
assert chans[0]["youtube_id"] == "UCPlay" and chans[0]["title"] == "PlayStation"
assert chans[0]["poster_url"] == "http://a/p.jpg" and chans[0]["video_count"] == 0
# hydration
assert db.channel_watch_state(["UCPlay", "UCnope"]) == {"UCPlay": True}
# idempotent re-follow refreshes title, no duplicate
db.add_channel_to_watchlist({"youtube_id": "UCPlay", "title": "PlayStation US"})
chans = db.list_watchlist_channels()
assert len(chans) == 1 and chans[0]["title"] == "PlayStation US"
def test_unfollow_channel_removes_row(db):
db.add_channel_to_watchlist({"youtube_id": "UCPlay", "title": "PlayStation"})
assert db.remove_channel_from_watchlist("UCPlay") is True
assert db.list_watchlist_channels() == []
assert db.channel_watch_state(["UCPlay"]) == {}
def test_add_videos_groups_under_channel_newest_first(db):
ch = {"youtube_id": "UCPlay", "title": "PlayStation", "avatar_url": "http://a/p.jpg"}
vids = [
{"youtube_id": "v1", "title": "Old Trailer", "published_at": "2023-01-01",
"thumbnail_url": "http://t/1.jpg", "description": "older"},
{"youtube_id": "v2", "title": "New State of Play", "published_at": "2024-06-01",
"thumbnail_url": "http://t/2.jpg", "description": "newer"},
{"youtube_id": "v3", "title": "Undated", "thumbnail_url": "http://t/3.jpg"},
]
assert db.add_videos_to_wishlist(ch, vids) == 3
res = db.query_youtube_wishlist()
assert res["pagination"]["total_count"] == 1
grp = res["items"][0]
assert grp["youtube_id"] == "UCPlay" and grp["title"] == "PlayStation"
assert grp["poster_url"] == "http://a/p.jpg" and grp["video_count"] == 3
# newest-first; dated before undated
assert [v["youtube_id"] for v in grp["videos"]] == ["v2", "v1", "v3"]
v2 = grp["videos"][0]
assert v2["title"] == "New State of Play" and v2["still_url"] == "http://t/2.jpg"
assert v2["overview"] == "newer" and v2["published_at"] == "2024-06-01"
assert v2["status"] == "wanted"
def test_add_videos_is_idempotent_per_video(db):
ch = {"youtube_id": "UCPlay", "title": "PlayStation"}
db.add_videos_to_wishlist(ch, [{"youtube_id": "v1", "title": "A"}])
db.add_videos_to_wishlist(ch, [{"youtube_id": "v1", "title": "A (updated)"},
{"youtube_id": "v2", "title": "B"}])
grp = db.query_youtube_wishlist()["items"][0]
assert grp["video_count"] == 2
titles = {v["youtube_id"]: v["title"] for v in grp["videos"]}
assert titles == {"v1": "A (updated)", "v2": "B"}
def test_youtube_counts_and_removal_scopes(db):
a = {"youtube_id": "UCa", "title": "Chan A"}
b = {"youtube_id": "UCb", "title": "Chan B"}
db.add_videos_to_wishlist(a, [{"youtube_id": "a1", "title": "A1"},
{"youtube_id": "a2", "title": "A2"}])
db.add_videos_to_wishlist(b, [{"youtube_id": "b1", "title": "B1"}])
assert db.youtube_wishlist_counts() == {"channel": 2, "video": 3}
# remove one video
assert db.remove_youtube_from_wishlist("video", "a1") == 1
assert db.youtube_wishlist_counts() == {"channel": 2, "video": 2}
# remove a whole channel
assert db.remove_youtube_from_wishlist("channel", "UCa") == 1 # only a2 left
assert db.youtube_wishlist_counts() == {"channel": 1, "video": 1}
def test_youtube_rows_do_not_disturb_tmdb_counts(db):
"""The bridge must not leak into the existing movie/episode + show/person counts."""
db.add_movie_to_wishlist(101, "A Movie", year=2020)
db.add_episodes_to_wishlist(202, "A Show", [{"season_number": 1, "episode_number": 1}])
db.add_to_watchlist("show", 303, "Watched Show")
db.add_channel_to_watchlist({"youtube_id": "UCx", "title": "Chan"})
db.add_videos_to_wishlist({"youtube_id": "UCx", "title": "Chan"},
[{"youtube_id": "x1", "title": "X1"}])
# existing shapes unchanged
assert db.wishlist_counts() == {"movie": 1, "show": 1, "episode": 1, "total": 2}
assert db.watchlist_counts() == {"show": 1, "person": 0, "total": 1}
# youtube counts live on their own surface
assert db.youtube_wishlist_counts() == {"channel": 1, "video": 1}
assert db.list_watchlist_channels()[0]["video_count"] == 1