Remember each channel: cache the video catalog + metadata (instant re-open)

Channel pages re-fetched everything every open (re-stream + ~3s yt-dlp metadata).
Now we remember what we learn about a channel and serve it cache-first:

- schema: youtube_channel_videos (list) + youtube_channel_meta (avatar/subs/tags/
  banner); dates stay in youtube_video_dates, merged on read. DB: cache_/get_
  channel_videos + cache_/get_channel_meta (upserts COALESCE so refreshes never
  drop fields).
- enricher: _enrich now REMEMBERS a channel — caches the full InnerTube catalog
  (list, via new innertube_channel_catalog) + metadata + dates, not just dates.
  Since it already sweeps followed channels, watchlisted channels get pre-warmed
  in the background → opening them is instant.
- channel endpoint: cache-first. A remembered channel renders from cache with NO
  network (returns from_cache:true); a miss resolves live (yt-dlp) and remembers
  it. The /videos batch endpoint caches every page it streams.
- frontend: cache hit renders the full catalog instantly, then re-streams to
  refresh QUIETLY (new uploads/date fixes); only the first (miss) load shows the
  'loading full history' banner.

Live-validated (MrBeast catalog: 90/3 pages, all titled+dated). 190 tests green.
This commit is contained in:
BoulderBadgeDad 2026-06-17 18:08:04 -07:00
parent d91693ba53
commit 267f11c848
10 changed files with 289 additions and 44 deletions

View file

@ -158,37 +158,53 @@ def register_routes(bp):
limit = 60
try:
db = get_video_db()
channel = yt.resolve_channel("https://www.youtube.com/channel/" + channel_id,
limit=max(1, min(90, limit)))
if not channel or not channel.get("youtube_id"):
return jsonify({"success": False, "error": "Channel not found"}), 404
cid = channel["youtube_id"]
cid = str(channel_id).strip()
following = bool(db.channel_watch_state([cid]))
# Opening a channel page → enrich its upload dates in the background
# (followed or not — you're looking at it, so you want the years).
# Opening a channel page → (re)remember it in the background (followed or
# not — you're looking at it). The enricher caches list + meta + dates.
try:
from core.video.youtube_enrichment import get_youtube_date_enricher
get_youtube_date_enricher().enqueue(cid, channel.get("title"))
get_youtube_date_enricher().enqueue(cid)
except Exception:
pass
# backfill the real avatar onto wished rows (flat listing often omits it)
if channel.get("avatar_url"):
try:
db.set_wishlist_channel_poster(cid, channel["avatar_url"])
except Exception:
pass
# CACHE-FIRST: a remembered channel renders instantly (no yt-dlp). The
# page's background re-stream + the enricher keep it fresh.
meta = db.get_channel_meta(cid)
cached_vids = db.get_channel_videos(cid)
from_cache = bool(meta and cached_vids)
if from_cache:
channel = {"youtube_id": cid, "title": meta.get("title"), "handle": meta.get("handle"),
"description": meta.get("description"), "avatar_url": meta.get("avatar_url"),
"banner_url": meta.get("banner_url"), "subscriber_count": meta.get("subscriber_count"),
"view_count": meta.get("view_count"), "tags": meta.get("tags") or [],
"videos": cached_vids}
else:
# MISS → fetch live (yt-dlp header + recent uploads) and remember it.
channel = yt.resolve_channel("https://www.youtube.com/channel/" + cid,
limit=max(1, min(90, limit)))
if not channel or not channel.get("youtube_id"):
return jsonify({"success": False, "error": "Channel not found"}), 404
cid = channel["youtube_id"]
db.cache_channel_meta(cid, channel)
db.cache_channel_videos(cid, channel.get("videos") or [])
if channel.get("avatar_url"):
try:
db.set_wishlist_channel_poster(cid, channel["avatar_url"])
except Exception:
pass
vids = channel.get("videos") or []
ids = [v.get("youtube_id") for v in vids]
wished = db.youtube_video_wish_state(ids)
# Upload dates → real year-seasons. Merge the persistent cache with a
# fresh RSS fetch (recent ~15); cache anything new so it fills in over time.
# Dates → year-seasons. Cached list already carries them; only pull a
# fresh RSS (recent ~15) on a live MISS so the cache-hit stays instant.
dates = db.get_video_dates(ids)
try:
rss = yt.channel_recent_dates(cid)
except Exception:
rss = {}
if rss:
dates.update(rss)
if not from_cache:
try:
dates.update(yt.channel_recent_dates(cid) or {})
except Exception:
pass
for v in vids:
v["wished"] = v.get("youtube_id") in wished
if not v.get("published_at") and dates.get(v.get("youtube_id")):
@ -199,7 +215,7 @@ def register_routes(bp):
except Exception:
pass
return jsonify({"success": True, "kind": "channel", "source": "youtube",
"channel": channel, "following": following})
"channel": channel, "following": following, "from_cache": from_cache})
except Exception:
logger.exception("youtube channel detail failed for %r", channel_id)
return jsonify({"success": False, "error": "Could not load channel"}), 500
@ -235,6 +251,7 @@ def register_routes(bp):
v["published_at"] = cached[vid]
v["wished"] = vid in wished
try:
db.cache_channel_videos(channel_id, videos) # remember the list
db.cache_video_dates([{"youtube_id": v["youtube_id"], "published_at": v.get("published_at")}
for v in videos if v.get("youtube_id") and v.get("published_at")])
except Exception:

View file

@ -575,6 +575,31 @@ def innertube_channel_videos_page(channel_id, continuation=None, now=None, post=
return {"videos": videos, "continuation": innertube_continuation(j)}
def innertube_channel_catalog(channel_id, pages=_INNERTUBE_PAGES, now=None, post=None):
"""A channel's video catalog (list + approximate dates) via InnerTube, up to
``pages`` pages: [{youtube_id, title, thumbnail_url, published_at}]. Like
innertube_channel_dates but keeps title/thumbnail so callers can REMEMBER the
whole list, not just date it. Bounded + throttled; [] on any failure."""
cid = str(channel_id or "").strip()
if not cid.startswith("UC"):
return []
if now is None:
now = datetime.now(timezone.utc).date()
out, seen, token = [], set(), None
for _ in range(max(1, pages)):
page = innertube_channel_videos_page(cid, continuation=token, now=now, post=post)
for v in page.get("videos") or []:
if v.get("youtube_id") and v["youtube_id"] not in seen:
seen.add(v["youtube_id"])
out.append(v)
token = page.get("continuation")
if not token:
break
if post is None:
time.sleep(_INNERTUBE_DELAY) # rate-limit politeness
return out
def search_channels(query, limit=6, ydl_factory=None):
"""Search YouTube for CHANNELS (the results page filtered to channels) → a few
{youtube_id, title, handle, avatar_url, subscriber_count} for the search page.

View file

@ -134,7 +134,8 @@ class YoutubeDateEnricher:
self._q.task_done()
def _enrich(self, channel_id):
"""Fetch + cache a channel's upload dates. Proxy in bulk; per-video fallback."""
"""REMEMBER a channel: cache its video catalog (list) + metadata + upload
dates so re-opening it is instant. InnerTube bulk; per-video yt-dlp fallback."""
from core.video import youtube as yt
db = self._db_factory()
cid = str(channel_id or "").strip()
@ -143,15 +144,27 @@ class YoutubeDateEnricher:
self._current = self._titles.get(cid) or cid
logger.debug("Enriching dates for %s (%s)", self._current, cid)
# PRIMARY: YouTube's own InnerTube browse API (no key/Java/proxy) — bulk
# dates for the whole videos tab in a handful of requests. The path we
# prefer; approximate dates, great for year-seasons.
# Remember channel metadata (avatar/subs/tags) for an instant header re-open.
try:
meta = yt.resolve_channel("https://www.youtube.com/channel/" + cid, limit=1)
if meta and meta.get("youtube_id"):
db.cache_channel_meta(cid, meta)
except Exception:
logger.debug("channel meta fetch failed for %s", cid, exc_info=True)
# PRIMARY: InnerTube — the whole videos tab (list + approximate dates) in a
# handful of requests. Remember the LIST too (not just dates) so the page
# can render the full catalog cache-first on the next open.
dates = {}
try:
dates = yt.innertube_channel_dates(cid) or {}
catalog = yt.innertube_channel_catalog(cid) or []
except Exception:
logger.debug("innertube date fetch failed for %s", cid, exc_info=True)
logger.debug("innertube returned %d dates for %s", len(dates), cid)
catalog = []
logger.debug("innertube catalog fetch failed for %s", cid, exc_info=True)
if catalog:
db.cache_channel_videos(cid, catalog)
dates = {v["youtube_id"]: v["published_at"] for v in catalog if v.get("published_at")}
logger.debug("innertube remembered %d videos (%d dated) for %s", len(catalog), len(dates), cid)
# SECONDARY (opt-in): a configured Piped/Invidious proxy — only if InnerTube
# came up empty (the public instances are unreliable/API-disabled).
if not dates:
@ -206,11 +219,11 @@ class YoutubeDateEnricher:
self._channels_done += 1
self._dates_total += len(dates) + filled
# Tag the source so legacy (pre-InnerTube) rows are recognisable and upgrade.
method = "innertube" if dates else "fallback"
method = "innertube" if (catalog or dates) else "fallback"
db.mark_channel_dates_enriched(cid, len(dates) + filled, method=method)
# Terse per-channel summary (like the worker's "Synced full episode list…").
logger.info("Dated %d videos for %s (%d bulk + %d per-video)",
len(dates) + filled, self._current, len(dates), filled)
logger.info("Remembered %d videos for %s (%d dated, %d per-video)",
len(catalog) or (len(dates) + filled), self._current, len(dates), filled)
_enricher = None

View file

@ -2088,6 +2088,97 @@ class VideoDatabase:
finally:
conn.close()
# ── Remembered channel catalog + metadata (cache-first channel pages) ──────
def cache_channel_videos(self, channel_id, videos) -> int:
"""Remember a channel's videos (id/title/thumbnail). Upsert — refreshes
title/thumbnail, never deletes (older pages stay remembered)."""
cid = str(channel_id or "").strip()
rows = [(cid, v.get("youtube_id"), v.get("title"), v.get("thumbnail_url"))
for v in (videos or []) if isinstance(v, dict) and v.get("youtube_id")]
if not cid or not rows:
return 0
conn = self._get_connection()
try:
conn.executemany(
"INSERT INTO youtube_channel_videos (channel_id, youtube_id, title, thumbnail_url) "
"VALUES (?,?,?,?) ON CONFLICT(channel_id, youtube_id) DO UPDATE SET "
"title=COALESCE(excluded.title, title), "
"thumbnail_url=COALESCE(excluded.thumbnail_url, thumbnail_url), "
"cached_at=CURRENT_TIMESTAMP", rows)
conn.commit()
return len(rows)
finally:
conn.close()
def get_channel_videos(self, channel_id) -> list:
"""The remembered video list with dates merged from youtube_video_dates,
newest first (undated last). [] if nothing is cached for the channel."""
cid = str(channel_id or "").strip()
if not cid:
return []
conn = self._get_connection()
try:
rows = conn.execute(
"SELECT v.youtube_id, v.title, v.thumbnail_url, d.published_at "
"FROM youtube_channel_videos v "
"LEFT JOIN youtube_video_dates d ON d.youtube_id = v.youtube_id "
"WHERE v.channel_id=? "
"ORDER BY (d.published_at IS NULL), d.published_at DESC, v.rowid",
(cid,)).fetchall()
return [{"youtube_id": r["youtube_id"], "title": r["title"],
"thumbnail_url": r["thumbnail_url"], "published_at": r["published_at"]}
for r in rows]
finally:
conn.close()
def cache_channel_meta(self, channel_id, meta) -> None:
"""Remember a channel's header metadata (avatar/subs/tags/…) for instant re-open."""
cid = str(channel_id or "").strip()
if not cid or not isinstance(meta, dict):
return
import json
tags = meta.get("tags")
conn = self._get_connection()
try:
conn.execute(
"INSERT INTO youtube_channel_meta (channel_id, title, handle, description, "
"avatar_url, banner_url, subscriber_count, view_count, tags, cached_at) "
"VALUES (?,?,?,?,?,?,?,?,?,CURRENT_TIMESTAMP) ON CONFLICT(channel_id) DO UPDATE SET "
"title=COALESCE(excluded.title, title), handle=COALESCE(excluded.handle, handle), "
"description=COALESCE(excluded.description, description), "
"avatar_url=COALESCE(excluded.avatar_url, avatar_url), "
"banner_url=COALESCE(excluded.banner_url, banner_url), "
"subscriber_count=COALESCE(excluded.subscriber_count, subscriber_count), "
"view_count=COALESCE(excluded.view_count, view_count), "
"tags=COALESCE(excluded.tags, tags), cached_at=CURRENT_TIMESTAMP",
(cid, meta.get("title"), meta.get("handle"), meta.get("description"),
meta.get("avatar_url"), meta.get("banner_url"),
meta.get("subscriber_count"), meta.get("view_count"),
json.dumps(tags) if tags else None))
conn.commit()
finally:
conn.close()
def get_channel_meta(self, channel_id):
"""The remembered channel metadata dict (tags decoded), or None."""
cid = str(channel_id or "").strip()
if not cid:
return None
import json
conn = self._get_connection()
try:
r = conn.execute("SELECT * FROM youtube_channel_meta WHERE channel_id=?", (cid,)).fetchone()
if not r:
return None
d = dict(r)
try:
d["tags"] = json.loads(d["tags"]) if d.get("tags") else []
except (ValueError, TypeError):
d["tags"] = []
return d
finally:
conn.close()
def youtube_wishlist_counts(self) -> dict:
"""{'channel': n distinct channels, 'video': n videos} in the wishlist."""
conn = self._get_connection()

View file

@ -280,6 +280,34 @@ CREATE TABLE IF NOT EXISTS youtube_channel_enrichment (
method TEXT -- 'innertube' | 'fallback'; NULL = legacy → re-enrich once
);
-- Remembered per-channel catalog so re-opening a channel (especially a watchlisted
-- one) is instant: served cache-first, then a background re-stream refreshes it.
-- Upload dates stay in youtube_video_dates (merged on read); this holds the list.
CREATE TABLE IF NOT EXISTS youtube_channel_videos (
channel_id TEXT NOT NULL,
youtube_id TEXT NOT NULL,
title TEXT,
thumbnail_url TEXT,
cached_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP,
PRIMARY KEY (channel_id, youtube_id)
);
CREATE INDEX IF NOT EXISTS idx_ycv_channel ON youtube_channel_videos(channel_id);
-- Remembered channel metadata (avatar/subs/tags/banner) so the header renders
-- instantly on re-open without a yt-dlp re-fetch.
CREATE TABLE IF NOT EXISTS youtube_channel_meta (
channel_id TEXT PRIMARY KEY,
title TEXT,
handle TEXT,
description TEXT,
avatar_url TEXT,
banner_url TEXT,
subscriber_count INTEGER,
view_count INTEGER,
tags TEXT, -- JSON array
cached_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP
);
-- ── Owned media files (the Library = content that has a file) ────────────────
-- Exactly one owner FK is set (no polymorphic id). 1 row per physical file;
-- usually 1:1 with its content, but the table allows history/extras.

View file

@ -629,6 +629,27 @@ def test_youtube_channel_detail_hydrates_follow_and_wished(tmp_path, monkeypatch
assert videoapi._video_db.get_video_dates(["v1"]) == {"v1": "2024-06-01"}
def test_youtube_channel_detail_is_cache_first(tmp_path, monkeypatch):
client, videoapi = _make_client(tmp_path)
import core.video.youtube as ytmod
db = videoapi._video_db
# Pre-remember the channel (as the enricher / a prior open would have).
db.cache_channel_meta("UCmem", {"title": "Remembered", "avatar_url": "av", "subscriber_count": 9})
db.cache_channel_videos("UCmem", [{"youtube_id": "m1", "title": "M1", "thumbnail_url": "t1"}])
db.cache_video_dates([{"youtube_id": "m1", "published_at": "2021-07-07"}])
# A cache HIT must NOT touch the network (yt-dlp / RSS).
monkeypatch.setattr(ytmod, "resolve_channel",
lambda *a, **k: (_ for _ in ()).throw(AssertionError("resolved on a cache hit")))
monkeypatch.setattr(ytmod, "channel_recent_dates",
lambda *a, **k: (_ for _ in ()).throw(AssertionError("RSS on a cache hit")))
d = client.get("/api/video/youtube/channel/UCmem").get_json()
assert d["success"] is True and d["from_cache"] is True
assert d["channel"]["title"] == "Remembered" and d["channel"]["subscriber_count"] == 9
vids = d["channel"]["videos"]
assert len(vids) == 1 and vids[0]["youtube_id"] == "m1" and vids[0]["published_at"] == "2021-07-07"
def test_youtube_channel_detail_404_on_unresolvable(tmp_path, monkeypatch):
client, _ = _make_client(tmp_path)
import core.video.youtube as ytmod

View file

@ -1192,3 +1192,27 @@ def test_legacy_enrichment_rows_upgrade(db):
# After the InnerTube re-run, the normal window applies again (no churn).
db.mark_channel_dates_enriched("UCleg", date_count=240, method="innertube")
assert db.channel_dates_enriched_recently("UCleg") is True
def test_remembered_channel_videos_and_meta(db):
# Cache a list (out of date order) + a date for one of them.
db.cache_video_dates([{"youtube_id": "b", "published_at": "2020-05-01"}])
db.cache_channel_videos("UCc", [
{"youtube_id": "a", "title": "A", "thumbnail_url": "ta"}, # undated → sorts last
{"youtube_id": "b", "title": "B", "thumbnail_url": "tb"}])
got = db.get_channel_videos("UCc")
assert [v["youtube_id"] for v in got] == ["b", "a"] # dated first, undated last
assert got[0]["published_at"] == "2020-05-01" and got[1]["published_at"] is None
assert got[0]["title"] == "B" and got[0]["thumbnail_url"] == "tb"
# Upsert refreshes fields without dropping the row; COALESCE keeps a non-null title.
db.cache_channel_videos("UCc", [{"youtube_id": "a", "title": "A2", "thumbnail_url": None}])
a = next(v for v in db.get_channel_videos("UCc") if v["youtube_id"] == "a")
assert a["title"] == "A2" and a["thumbnail_url"] == "ta" # title updated, thumb kept
assert db.get_channel_videos("UNKNOWN") == []
# Metadata round-trips with tags decoded.
assert db.get_channel_meta("UCc") is None
db.cache_channel_meta("UCc", {"title": "Chan", "handle": "@c", "avatar_url": "av",
"subscriber_count": 1234, "tags": ["x", "y"]})
m = db.get_channel_meta("UCc")
assert m["title"] == "Chan" and m["subscriber_count"] == 1234 and m["tags"] == ["x", "y"]

View file

@ -500,3 +500,20 @@ def test_innertube_channel_videos_page_guards():
# a continuation token works even without a UC id (it encodes the channel itself)
assert yt.innertube_channel_videos_page("", continuation="TOK",
post=lambda p: _page([_lk_item("v9", "1 day ago")]))["videos"]
def test_innertube_channel_catalog_pages_until_no_token():
now = date(2026, 6, 17)
pages = [_page([_lk_item("v1", "1 year ago"), _lk_item("v2", "2 years ago")], token="TOK"),
_page([_lk_item("v3", "5 days ago")])] # no token → stop
seq = {"n": 0}
def post(payload):
i = seq["n"]; seq["n"] += 1
return pages[i]
cat = yt.innertube_channel_catalog("UCxxxx", pages=5, now=now, post=post)
assert [v["youtube_id"] for v in cat] == ["v1", "v2", "v3"] # full list, in order
assert cat[0]["title"] == "Title v1" and cat[2]["published_at"] == "2026-06-12"
assert seq["n"] == 2 # stopped when the token ran out
assert yt.innertube_channel_catalog("not-uc", post=lambda p: None) == []

View file

@ -13,30 +13,36 @@ def db(tmp_path):
return VideoDatabase(database_path=str(tmp_path / "video_library.db"))
def test_enrich_caches_bulk_dates_and_marks_done(db, monkeypatch):
def test_enrich_remembers_catalog_meta_and_dates(db, monkeypatch):
import core.video.youtube as yt
# InnerTube (primary) covers everything → per-video fallback must NOT run
monkeypatch.setattr(yt, "innertube_channel_dates",
lambda cid, *a, **k: {"v1": "2024-06-01", "v2": "2023-02-02"})
# InnerTube catalog (primary) covers everything → per-video fallback must NOT run
monkeypatch.setattr(yt, "innertube_channel_catalog", lambda cid, *a, **k: [
{"youtube_id": "v1", "title": "A", "thumbnail_url": "t1", "published_at": "2024-06-01"},
{"youtube_id": "v2", "title": "B", "thumbnail_url": "t2", "published_at": "2023-02-02"}])
monkeypatch.setattr(yt, "resolve_channel", lambda url, **k: {"youtube_id": "UCx", "title": "X", "avatar_url": "a.jpg"})
monkeypatch.setattr(yt, "video_detail", lambda vid: (_ for _ in ()).throw(AssertionError("no fallback")))
db.add_videos_to_wishlist({"youtube_id": "UCx", "title": "X"}, [{"youtube_id": "v1", "title": "A"}])
e = YoutubeDateEnricher(db_factory=lambda: db)
e._enrich("UCx")
assert db.get_video_dates(["v1", "v2"]) == {"v1": "2024-06-01", "v2": "2023-02-02"}
# the LIST + METADATA got remembered, not just the dates
remembered = db.get_channel_videos("UCx")
assert {v["youtube_id"] for v in remembered} == {"v1", "v2"}
assert db.get_channel_meta("UCx")["avatar_url"] == "a.jpg"
assert db.channel_dates_enriched_recently("UCx") is True
# already enriched → second pass is a no-op (no fetch)
monkeypatch.setattr(yt, "innertube_channel_dates", lambda *a, **k: (_ for _ in ()).throw(AssertionError("re-swept")))
monkeypatch.setattr(yt, "innertube_channel_catalog", lambda *a, **k: (_ for _ in ()).throw(AssertionError("re-swept")))
e._enrich("UCx")
def test_enrich_falls_back_to_per_video_when_bulk_empty(db, monkeypatch):
import core.video.youtube as yt
monkeypatch.setattr(yt, "innertube_channel_dates", lambda cid, *a, **k: {}) # InnerTube empty
monkeypatch.setattr(yt, "proxy_channel_dates", lambda cid, *a, **k: {}) # no proxy either
monkeypatch.setattr(yt, "innertube_channel_catalog", lambda cid, *a, **k: []) # InnerTube empty
monkeypatch.setattr(yt, "proxy_channel_dates", lambda cid, *a, **k: {}) # no proxy either
# flat resolve adds a recent upload r1 to the date-fallback set (besides wished w1/w2)
monkeypatch.setattr(yt, "resolve_channel",
lambda url, **k: {"videos": [{"youtube_id": "r1", "title": "Recent"}]})
lambda url, **k: {"youtube_id": "UCx", "videos": [{"youtube_id": "r1", "title": "Recent"}]})
monkeypatch.setattr(yt, "video_detail",
lambda vid: {"youtube_id": vid, "published_at": "2022-03-03"} if vid in ("w1", "r1") else None)
import core.video.youtube_enrichment as mod

View file

@ -1355,7 +1355,7 @@
// — this both DATES the recent videos and EXPANDS past the initial ~90 cap.
var ytLoadAllToken = 0;
function ytCancelLoad() { ytLoadAllToken++; }
function ytLoadAllVideos(id) {
function ytLoadAllVideos(id, silent) {
var token = ++ytLoadAllToken;
var byId = {};
((data && data._channel && data._channel.videos) || []).forEach(function (v) { byId[v.youtube_id] = v; });
@ -1394,7 +1394,9 @@
}
cont = resp.continuation;
if (cont && data._channel.videos.length < MAX) {
showEpSyncing(true, 'Loading the channels full video history… ' +
// Quiet when refreshing a remembered channel; only the first
// (cache-miss) load shows the "loading full history" banner.
if (!silent) showEpSyncing(true, 'Loading the channels full video history… ' +
data._channel.videos.length + ' videos so far.');
setTimeout(step, 120);
} else {
@ -1431,7 +1433,8 @@
if (sub) sub.scrollTop = 0;
ytLoadPlaylists(id);
// Stream the rest of the catalog (and fill upload dates) in batches.
ytLoadAllVideos(id);
// A remembered channel renders full from cache → refresh quietly.
ytLoadAllVideos(id, !!resp.from_cache);
})
.catch(function () { showLoading(false); setText('[data-vd-title]', 'Could not load channel'); });
}