Remember each channel: cache the video catalog + metadata (instant re-open)
Channel pages re-fetched everything every open (re-stream + ~3s yt-dlp metadata). Now we remember what we learn about a channel and serve it cache-first: - schema: youtube_channel_videos (list) + youtube_channel_meta (avatar/subs/tags/ banner); dates stay in youtube_video_dates, merged on read. DB: cache_/get_ channel_videos + cache_/get_channel_meta (upserts COALESCE so refreshes never drop fields). - enricher: _enrich now REMEMBERS a channel — caches the full InnerTube catalog (list, via new innertube_channel_catalog) + metadata + dates, not just dates. Since it already sweeps followed channels, watchlisted channels get pre-warmed in the background → opening them is instant. - channel endpoint: cache-first. A remembered channel renders from cache with NO network (returns from_cache:true); a miss resolves live (yt-dlp) and remembers it. The /videos batch endpoint caches every page it streams. - frontend: cache hit renders the full catalog instantly, then re-streams to refresh QUIETLY (new uploads/date fixes); only the first (miss) load shows the 'loading full history' banner. Live-validated (MrBeast catalog: 90/3 pages, all titled+dated). 190 tests green.
This commit is contained in:
parent
d91693ba53
commit
267f11c848
10 changed files with 289 additions and 44 deletions
|
|
@ -158,37 +158,53 @@ def register_routes(bp):
|
|||
limit = 60
|
||||
try:
|
||||
db = get_video_db()
|
||||
channel = yt.resolve_channel("https://www.youtube.com/channel/" + channel_id,
|
||||
limit=max(1, min(90, limit)))
|
||||
if not channel or not channel.get("youtube_id"):
|
||||
return jsonify({"success": False, "error": "Channel not found"}), 404
|
||||
cid = channel["youtube_id"]
|
||||
cid = str(channel_id).strip()
|
||||
following = bool(db.channel_watch_state([cid]))
|
||||
# Opening a channel page → enrich its upload dates in the background
|
||||
# (followed or not — you're looking at it, so you want the years).
|
||||
# Opening a channel page → (re)remember it in the background (followed or
|
||||
# not — you're looking at it). The enricher caches list + meta + dates.
|
||||
try:
|
||||
from core.video.youtube_enrichment import get_youtube_date_enricher
|
||||
get_youtube_date_enricher().enqueue(cid, channel.get("title"))
|
||||
get_youtube_date_enricher().enqueue(cid)
|
||||
except Exception:
|
||||
pass
|
||||
# backfill the real avatar onto wished rows (flat listing often omits it)
|
||||
if channel.get("avatar_url"):
|
||||
try:
|
||||
db.set_wishlist_channel_poster(cid, channel["avatar_url"])
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# CACHE-FIRST: a remembered channel renders instantly (no yt-dlp). The
|
||||
# page's background re-stream + the enricher keep it fresh.
|
||||
meta = db.get_channel_meta(cid)
|
||||
cached_vids = db.get_channel_videos(cid)
|
||||
from_cache = bool(meta and cached_vids)
|
||||
if from_cache:
|
||||
channel = {"youtube_id": cid, "title": meta.get("title"), "handle": meta.get("handle"),
|
||||
"description": meta.get("description"), "avatar_url": meta.get("avatar_url"),
|
||||
"banner_url": meta.get("banner_url"), "subscriber_count": meta.get("subscriber_count"),
|
||||
"view_count": meta.get("view_count"), "tags": meta.get("tags") or [],
|
||||
"videos": cached_vids}
|
||||
else:
|
||||
# MISS → fetch live (yt-dlp header + recent uploads) and remember it.
|
||||
channel = yt.resolve_channel("https://www.youtube.com/channel/" + cid,
|
||||
limit=max(1, min(90, limit)))
|
||||
if not channel or not channel.get("youtube_id"):
|
||||
return jsonify({"success": False, "error": "Channel not found"}), 404
|
||||
cid = channel["youtube_id"]
|
||||
db.cache_channel_meta(cid, channel)
|
||||
db.cache_channel_videos(cid, channel.get("videos") or [])
|
||||
if channel.get("avatar_url"):
|
||||
try:
|
||||
db.set_wishlist_channel_poster(cid, channel["avatar_url"])
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
vids = channel.get("videos") or []
|
||||
ids = [v.get("youtube_id") for v in vids]
|
||||
wished = db.youtube_video_wish_state(ids)
|
||||
# Upload dates → real year-seasons. Merge the persistent cache with a
|
||||
# fresh RSS fetch (recent ~15); cache anything new so it fills in over time.
|
||||
# Dates → year-seasons. Cached list already carries them; only pull a
|
||||
# fresh RSS (recent ~15) on a live MISS so the cache-hit stays instant.
|
||||
dates = db.get_video_dates(ids)
|
||||
try:
|
||||
rss = yt.channel_recent_dates(cid)
|
||||
except Exception:
|
||||
rss = {}
|
||||
if rss:
|
||||
dates.update(rss)
|
||||
if not from_cache:
|
||||
try:
|
||||
dates.update(yt.channel_recent_dates(cid) or {})
|
||||
except Exception:
|
||||
pass
|
||||
for v in vids:
|
||||
v["wished"] = v.get("youtube_id") in wished
|
||||
if not v.get("published_at") and dates.get(v.get("youtube_id")):
|
||||
|
|
@ -199,7 +215,7 @@ def register_routes(bp):
|
|||
except Exception:
|
||||
pass
|
||||
return jsonify({"success": True, "kind": "channel", "source": "youtube",
|
||||
"channel": channel, "following": following})
|
||||
"channel": channel, "following": following, "from_cache": from_cache})
|
||||
except Exception:
|
||||
logger.exception("youtube channel detail failed for %r", channel_id)
|
||||
return jsonify({"success": False, "error": "Could not load channel"}), 500
|
||||
|
|
@ -235,6 +251,7 @@ def register_routes(bp):
|
|||
v["published_at"] = cached[vid]
|
||||
v["wished"] = vid in wished
|
||||
try:
|
||||
db.cache_channel_videos(channel_id, videos) # remember the list
|
||||
db.cache_video_dates([{"youtube_id": v["youtube_id"], "published_at": v.get("published_at")}
|
||||
for v in videos if v.get("youtube_id") and v.get("published_at")])
|
||||
except Exception:
|
||||
|
|
|
|||
|
|
@ -575,6 +575,31 @@ def innertube_channel_videos_page(channel_id, continuation=None, now=None, post=
|
|||
return {"videos": videos, "continuation": innertube_continuation(j)}
|
||||
|
||||
|
||||
def innertube_channel_catalog(channel_id, pages=_INNERTUBE_PAGES, now=None, post=None):
|
||||
"""A channel's video catalog (list + approximate dates) via InnerTube, up to
|
||||
``pages`` pages: [{youtube_id, title, thumbnail_url, published_at}]. Like
|
||||
innertube_channel_dates but keeps title/thumbnail so callers can REMEMBER the
|
||||
whole list, not just date it. Bounded + throttled; [] on any failure."""
|
||||
cid = str(channel_id or "").strip()
|
||||
if not cid.startswith("UC"):
|
||||
return []
|
||||
if now is None:
|
||||
now = datetime.now(timezone.utc).date()
|
||||
out, seen, token = [], set(), None
|
||||
for _ in range(max(1, pages)):
|
||||
page = innertube_channel_videos_page(cid, continuation=token, now=now, post=post)
|
||||
for v in page.get("videos") or []:
|
||||
if v.get("youtube_id") and v["youtube_id"] not in seen:
|
||||
seen.add(v["youtube_id"])
|
||||
out.append(v)
|
||||
token = page.get("continuation")
|
||||
if not token:
|
||||
break
|
||||
if post is None:
|
||||
time.sleep(_INNERTUBE_DELAY) # rate-limit politeness
|
||||
return out
|
||||
|
||||
|
||||
def search_channels(query, limit=6, ydl_factory=None):
|
||||
"""Search YouTube for CHANNELS (the results page filtered to channels) → a few
|
||||
{youtube_id, title, handle, avatar_url, subscriber_count} for the search page.
|
||||
|
|
|
|||
|
|
@ -134,7 +134,8 @@ class YoutubeDateEnricher:
|
|||
self._q.task_done()
|
||||
|
||||
def _enrich(self, channel_id):
|
||||
"""Fetch + cache a channel's upload dates. Proxy in bulk; per-video fallback."""
|
||||
"""REMEMBER a channel: cache its video catalog (list) + metadata + upload
|
||||
dates so re-opening it is instant. InnerTube bulk; per-video yt-dlp fallback."""
|
||||
from core.video import youtube as yt
|
||||
db = self._db_factory()
|
||||
cid = str(channel_id or "").strip()
|
||||
|
|
@ -143,15 +144,27 @@ class YoutubeDateEnricher:
|
|||
self._current = self._titles.get(cid) or cid
|
||||
logger.debug("Enriching dates for %s (%s)", self._current, cid)
|
||||
|
||||
# PRIMARY: YouTube's own InnerTube browse API (no key/Java/proxy) — bulk
|
||||
# dates for the whole videos tab in a handful of requests. The path we
|
||||
# prefer; approximate dates, great for year-seasons.
|
||||
# Remember channel metadata (avatar/subs/tags) for an instant header re-open.
|
||||
try:
|
||||
meta = yt.resolve_channel("https://www.youtube.com/channel/" + cid, limit=1)
|
||||
if meta and meta.get("youtube_id"):
|
||||
db.cache_channel_meta(cid, meta)
|
||||
except Exception:
|
||||
logger.debug("channel meta fetch failed for %s", cid, exc_info=True)
|
||||
|
||||
# PRIMARY: InnerTube — the whole videos tab (list + approximate dates) in a
|
||||
# handful of requests. Remember the LIST too (not just dates) so the page
|
||||
# can render the full catalog cache-first on the next open.
|
||||
dates = {}
|
||||
try:
|
||||
dates = yt.innertube_channel_dates(cid) or {}
|
||||
catalog = yt.innertube_channel_catalog(cid) or []
|
||||
except Exception:
|
||||
logger.debug("innertube date fetch failed for %s", cid, exc_info=True)
|
||||
logger.debug("innertube returned %d dates for %s", len(dates), cid)
|
||||
catalog = []
|
||||
logger.debug("innertube catalog fetch failed for %s", cid, exc_info=True)
|
||||
if catalog:
|
||||
db.cache_channel_videos(cid, catalog)
|
||||
dates = {v["youtube_id"]: v["published_at"] for v in catalog if v.get("published_at")}
|
||||
logger.debug("innertube remembered %d videos (%d dated) for %s", len(catalog), len(dates), cid)
|
||||
# SECONDARY (opt-in): a configured Piped/Invidious proxy — only if InnerTube
|
||||
# came up empty (the public instances are unreliable/API-disabled).
|
||||
if not dates:
|
||||
|
|
@ -206,11 +219,11 @@ class YoutubeDateEnricher:
|
|||
self._channels_done += 1
|
||||
self._dates_total += len(dates) + filled
|
||||
# Tag the source so legacy (pre-InnerTube) rows are recognisable and upgrade.
|
||||
method = "innertube" if dates else "fallback"
|
||||
method = "innertube" if (catalog or dates) else "fallback"
|
||||
db.mark_channel_dates_enriched(cid, len(dates) + filled, method=method)
|
||||
# Terse per-channel summary (like the worker's "Synced full episode list…").
|
||||
logger.info("Dated %d videos for %s (%d bulk + %d per-video)",
|
||||
len(dates) + filled, self._current, len(dates), filled)
|
||||
logger.info("Remembered %d videos for %s (%d dated, %d per-video)",
|
||||
len(catalog) or (len(dates) + filled), self._current, len(dates), filled)
|
||||
|
||||
|
||||
_enricher = None
|
||||
|
|
|
|||
|
|
@ -2088,6 +2088,97 @@ class VideoDatabase:
|
|||
finally:
|
||||
conn.close()
|
||||
|
||||
# ── Remembered channel catalog + metadata (cache-first channel pages) ──────
|
||||
def cache_channel_videos(self, channel_id, videos) -> int:
|
||||
"""Remember a channel's videos (id/title/thumbnail). Upsert — refreshes
|
||||
title/thumbnail, never deletes (older pages stay remembered)."""
|
||||
cid = str(channel_id or "").strip()
|
||||
rows = [(cid, v.get("youtube_id"), v.get("title"), v.get("thumbnail_url"))
|
||||
for v in (videos or []) if isinstance(v, dict) and v.get("youtube_id")]
|
||||
if not cid or not rows:
|
||||
return 0
|
||||
conn = self._get_connection()
|
||||
try:
|
||||
conn.executemany(
|
||||
"INSERT INTO youtube_channel_videos (channel_id, youtube_id, title, thumbnail_url) "
|
||||
"VALUES (?,?,?,?) ON CONFLICT(channel_id, youtube_id) DO UPDATE SET "
|
||||
"title=COALESCE(excluded.title, title), "
|
||||
"thumbnail_url=COALESCE(excluded.thumbnail_url, thumbnail_url), "
|
||||
"cached_at=CURRENT_TIMESTAMP", rows)
|
||||
conn.commit()
|
||||
return len(rows)
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
def get_channel_videos(self, channel_id) -> list:
|
||||
"""The remembered video list with dates merged from youtube_video_dates,
|
||||
newest first (undated last). [] if nothing is cached for the channel."""
|
||||
cid = str(channel_id or "").strip()
|
||||
if not cid:
|
||||
return []
|
||||
conn = self._get_connection()
|
||||
try:
|
||||
rows = conn.execute(
|
||||
"SELECT v.youtube_id, v.title, v.thumbnail_url, d.published_at "
|
||||
"FROM youtube_channel_videos v "
|
||||
"LEFT JOIN youtube_video_dates d ON d.youtube_id = v.youtube_id "
|
||||
"WHERE v.channel_id=? "
|
||||
"ORDER BY (d.published_at IS NULL), d.published_at DESC, v.rowid",
|
||||
(cid,)).fetchall()
|
||||
return [{"youtube_id": r["youtube_id"], "title": r["title"],
|
||||
"thumbnail_url": r["thumbnail_url"], "published_at": r["published_at"]}
|
||||
for r in rows]
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
def cache_channel_meta(self, channel_id, meta) -> None:
|
||||
"""Remember a channel's header metadata (avatar/subs/tags/…) for instant re-open."""
|
||||
cid = str(channel_id or "").strip()
|
||||
if not cid or not isinstance(meta, dict):
|
||||
return
|
||||
import json
|
||||
tags = meta.get("tags")
|
||||
conn = self._get_connection()
|
||||
try:
|
||||
conn.execute(
|
||||
"INSERT INTO youtube_channel_meta (channel_id, title, handle, description, "
|
||||
"avatar_url, banner_url, subscriber_count, view_count, tags, cached_at) "
|
||||
"VALUES (?,?,?,?,?,?,?,?,?,CURRENT_TIMESTAMP) ON CONFLICT(channel_id) DO UPDATE SET "
|
||||
"title=COALESCE(excluded.title, title), handle=COALESCE(excluded.handle, handle), "
|
||||
"description=COALESCE(excluded.description, description), "
|
||||
"avatar_url=COALESCE(excluded.avatar_url, avatar_url), "
|
||||
"banner_url=COALESCE(excluded.banner_url, banner_url), "
|
||||
"subscriber_count=COALESCE(excluded.subscriber_count, subscriber_count), "
|
||||
"view_count=COALESCE(excluded.view_count, view_count), "
|
||||
"tags=COALESCE(excluded.tags, tags), cached_at=CURRENT_TIMESTAMP",
|
||||
(cid, meta.get("title"), meta.get("handle"), meta.get("description"),
|
||||
meta.get("avatar_url"), meta.get("banner_url"),
|
||||
meta.get("subscriber_count"), meta.get("view_count"),
|
||||
json.dumps(tags) if tags else None))
|
||||
conn.commit()
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
def get_channel_meta(self, channel_id):
|
||||
"""The remembered channel metadata dict (tags decoded), or None."""
|
||||
cid = str(channel_id or "").strip()
|
||||
if not cid:
|
||||
return None
|
||||
import json
|
||||
conn = self._get_connection()
|
||||
try:
|
||||
r = conn.execute("SELECT * FROM youtube_channel_meta WHERE channel_id=?", (cid,)).fetchone()
|
||||
if not r:
|
||||
return None
|
||||
d = dict(r)
|
||||
try:
|
||||
d["tags"] = json.loads(d["tags"]) if d.get("tags") else []
|
||||
except (ValueError, TypeError):
|
||||
d["tags"] = []
|
||||
return d
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
def youtube_wishlist_counts(self) -> dict:
|
||||
"""{'channel': n distinct channels, 'video': n videos} in the wishlist."""
|
||||
conn = self._get_connection()
|
||||
|
|
|
|||
|
|
@ -280,6 +280,34 @@ CREATE TABLE IF NOT EXISTS youtube_channel_enrichment (
|
|||
method TEXT -- 'innertube' | 'fallback'; NULL = legacy → re-enrich once
|
||||
);
|
||||
|
||||
-- Remembered per-channel catalog so re-opening a channel (especially a watchlisted
|
||||
-- one) is instant: served cache-first, then a background re-stream refreshes it.
|
||||
-- Upload dates stay in youtube_video_dates (merged on read); this holds the list.
|
||||
CREATE TABLE IF NOT EXISTS youtube_channel_videos (
|
||||
channel_id TEXT NOT NULL,
|
||||
youtube_id TEXT NOT NULL,
|
||||
title TEXT,
|
||||
thumbnail_url TEXT,
|
||||
cached_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
PRIMARY KEY (channel_id, youtube_id)
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_ycv_channel ON youtube_channel_videos(channel_id);
|
||||
|
||||
-- Remembered channel metadata (avatar/subs/tags/banner) so the header renders
|
||||
-- instantly on re-open without a yt-dlp re-fetch.
|
||||
CREATE TABLE IF NOT EXISTS youtube_channel_meta (
|
||||
channel_id TEXT PRIMARY KEY,
|
||||
title TEXT,
|
||||
handle TEXT,
|
||||
description TEXT,
|
||||
avatar_url TEXT,
|
||||
banner_url TEXT,
|
||||
subscriber_count INTEGER,
|
||||
view_count INTEGER,
|
||||
tags TEXT, -- JSON array
|
||||
cached_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP
|
||||
);
|
||||
|
||||
-- ── Owned media files (the Library = content that has a file) ────────────────
|
||||
-- Exactly one owner FK is set (no polymorphic id). 1 row per physical file;
|
||||
-- usually 1:1 with its content, but the table allows history/extras.
|
||||
|
|
|
|||
|
|
@ -629,6 +629,27 @@ def test_youtube_channel_detail_hydrates_follow_and_wished(tmp_path, monkeypatch
|
|||
assert videoapi._video_db.get_video_dates(["v1"]) == {"v1": "2024-06-01"}
|
||||
|
||||
|
||||
def test_youtube_channel_detail_is_cache_first(tmp_path, monkeypatch):
|
||||
client, videoapi = _make_client(tmp_path)
|
||||
import core.video.youtube as ytmod
|
||||
db = videoapi._video_db
|
||||
# Pre-remember the channel (as the enricher / a prior open would have).
|
||||
db.cache_channel_meta("UCmem", {"title": "Remembered", "avatar_url": "av", "subscriber_count": 9})
|
||||
db.cache_channel_videos("UCmem", [{"youtube_id": "m1", "title": "M1", "thumbnail_url": "t1"}])
|
||||
db.cache_video_dates([{"youtube_id": "m1", "published_at": "2021-07-07"}])
|
||||
|
||||
# A cache HIT must NOT touch the network (yt-dlp / RSS).
|
||||
monkeypatch.setattr(ytmod, "resolve_channel",
|
||||
lambda *a, **k: (_ for _ in ()).throw(AssertionError("resolved on a cache hit")))
|
||||
monkeypatch.setattr(ytmod, "channel_recent_dates",
|
||||
lambda *a, **k: (_ for _ in ()).throw(AssertionError("RSS on a cache hit")))
|
||||
d = client.get("/api/video/youtube/channel/UCmem").get_json()
|
||||
assert d["success"] is True and d["from_cache"] is True
|
||||
assert d["channel"]["title"] == "Remembered" and d["channel"]["subscriber_count"] == 9
|
||||
vids = d["channel"]["videos"]
|
||||
assert len(vids) == 1 and vids[0]["youtube_id"] == "m1" and vids[0]["published_at"] == "2021-07-07"
|
||||
|
||||
|
||||
def test_youtube_channel_detail_404_on_unresolvable(tmp_path, monkeypatch):
|
||||
client, _ = _make_client(tmp_path)
|
||||
import core.video.youtube as ytmod
|
||||
|
|
|
|||
|
|
@ -1192,3 +1192,27 @@ def test_legacy_enrichment_rows_upgrade(db):
|
|||
# After the InnerTube re-run, the normal window applies again (no churn).
|
||||
db.mark_channel_dates_enriched("UCleg", date_count=240, method="innertube")
|
||||
assert db.channel_dates_enriched_recently("UCleg") is True
|
||||
|
||||
|
||||
def test_remembered_channel_videos_and_meta(db):
|
||||
# Cache a list (out of date order) + a date for one of them.
|
||||
db.cache_video_dates([{"youtube_id": "b", "published_at": "2020-05-01"}])
|
||||
db.cache_channel_videos("UCc", [
|
||||
{"youtube_id": "a", "title": "A", "thumbnail_url": "ta"}, # undated → sorts last
|
||||
{"youtube_id": "b", "title": "B", "thumbnail_url": "tb"}])
|
||||
got = db.get_channel_videos("UCc")
|
||||
assert [v["youtube_id"] for v in got] == ["b", "a"] # dated first, undated last
|
||||
assert got[0]["published_at"] == "2020-05-01" and got[1]["published_at"] is None
|
||||
assert got[0]["title"] == "B" and got[0]["thumbnail_url"] == "tb"
|
||||
# Upsert refreshes fields without dropping the row; COALESCE keeps a non-null title.
|
||||
db.cache_channel_videos("UCc", [{"youtube_id": "a", "title": "A2", "thumbnail_url": None}])
|
||||
a = next(v for v in db.get_channel_videos("UCc") if v["youtube_id"] == "a")
|
||||
assert a["title"] == "A2" and a["thumbnail_url"] == "ta" # title updated, thumb kept
|
||||
assert db.get_channel_videos("UNKNOWN") == []
|
||||
|
||||
# Metadata round-trips with tags decoded.
|
||||
assert db.get_channel_meta("UCc") is None
|
||||
db.cache_channel_meta("UCc", {"title": "Chan", "handle": "@c", "avatar_url": "av",
|
||||
"subscriber_count": 1234, "tags": ["x", "y"]})
|
||||
m = db.get_channel_meta("UCc")
|
||||
assert m["title"] == "Chan" and m["subscriber_count"] == 1234 and m["tags"] == ["x", "y"]
|
||||
|
|
|
|||
|
|
@ -500,3 +500,20 @@ def test_innertube_channel_videos_page_guards():
|
|||
# a continuation token works even without a UC id (it encodes the channel itself)
|
||||
assert yt.innertube_channel_videos_page("", continuation="TOK",
|
||||
post=lambda p: _page([_lk_item("v9", "1 day ago")]))["videos"]
|
||||
|
||||
|
||||
def test_innertube_channel_catalog_pages_until_no_token():
|
||||
now = date(2026, 6, 17)
|
||||
pages = [_page([_lk_item("v1", "1 year ago"), _lk_item("v2", "2 years ago")], token="TOK"),
|
||||
_page([_lk_item("v3", "5 days ago")])] # no token → stop
|
||||
seq = {"n": 0}
|
||||
|
||||
def post(payload):
|
||||
i = seq["n"]; seq["n"] += 1
|
||||
return pages[i]
|
||||
|
||||
cat = yt.innertube_channel_catalog("UCxxxx", pages=5, now=now, post=post)
|
||||
assert [v["youtube_id"] for v in cat] == ["v1", "v2", "v3"] # full list, in order
|
||||
assert cat[0]["title"] == "Title v1" and cat[2]["published_at"] == "2026-06-12"
|
||||
assert seq["n"] == 2 # stopped when the token ran out
|
||||
assert yt.innertube_channel_catalog("not-uc", post=lambda p: None) == []
|
||||
|
|
|
|||
|
|
@ -13,30 +13,36 @@ def db(tmp_path):
|
|||
return VideoDatabase(database_path=str(tmp_path / "video_library.db"))
|
||||
|
||||
|
||||
def test_enrich_caches_bulk_dates_and_marks_done(db, monkeypatch):
|
||||
def test_enrich_remembers_catalog_meta_and_dates(db, monkeypatch):
|
||||
import core.video.youtube as yt
|
||||
# InnerTube (primary) covers everything → per-video fallback must NOT run
|
||||
monkeypatch.setattr(yt, "innertube_channel_dates",
|
||||
lambda cid, *a, **k: {"v1": "2024-06-01", "v2": "2023-02-02"})
|
||||
# InnerTube catalog (primary) covers everything → per-video fallback must NOT run
|
||||
monkeypatch.setattr(yt, "innertube_channel_catalog", lambda cid, *a, **k: [
|
||||
{"youtube_id": "v1", "title": "A", "thumbnail_url": "t1", "published_at": "2024-06-01"},
|
||||
{"youtube_id": "v2", "title": "B", "thumbnail_url": "t2", "published_at": "2023-02-02"}])
|
||||
monkeypatch.setattr(yt, "resolve_channel", lambda url, **k: {"youtube_id": "UCx", "title": "X", "avatar_url": "a.jpg"})
|
||||
monkeypatch.setattr(yt, "video_detail", lambda vid: (_ for _ in ()).throw(AssertionError("no fallback")))
|
||||
db.add_videos_to_wishlist({"youtube_id": "UCx", "title": "X"}, [{"youtube_id": "v1", "title": "A"}])
|
||||
|
||||
e = YoutubeDateEnricher(db_factory=lambda: db)
|
||||
e._enrich("UCx")
|
||||
assert db.get_video_dates(["v1", "v2"]) == {"v1": "2024-06-01", "v2": "2023-02-02"}
|
||||
# the LIST + METADATA got remembered, not just the dates
|
||||
remembered = db.get_channel_videos("UCx")
|
||||
assert {v["youtube_id"] for v in remembered} == {"v1", "v2"}
|
||||
assert db.get_channel_meta("UCx")["avatar_url"] == "a.jpg"
|
||||
assert db.channel_dates_enriched_recently("UCx") is True
|
||||
# already enriched → second pass is a no-op (no fetch)
|
||||
monkeypatch.setattr(yt, "innertube_channel_dates", lambda *a, **k: (_ for _ in ()).throw(AssertionError("re-swept")))
|
||||
monkeypatch.setattr(yt, "innertube_channel_catalog", lambda *a, **k: (_ for _ in ()).throw(AssertionError("re-swept")))
|
||||
e._enrich("UCx")
|
||||
|
||||
|
||||
def test_enrich_falls_back_to_per_video_when_bulk_empty(db, monkeypatch):
|
||||
import core.video.youtube as yt
|
||||
monkeypatch.setattr(yt, "innertube_channel_dates", lambda cid, *a, **k: {}) # InnerTube empty
|
||||
monkeypatch.setattr(yt, "proxy_channel_dates", lambda cid, *a, **k: {}) # no proxy either
|
||||
monkeypatch.setattr(yt, "innertube_channel_catalog", lambda cid, *a, **k: []) # InnerTube empty
|
||||
monkeypatch.setattr(yt, "proxy_channel_dates", lambda cid, *a, **k: {}) # no proxy either
|
||||
# flat resolve adds a recent upload r1 to the date-fallback set (besides wished w1/w2)
|
||||
monkeypatch.setattr(yt, "resolve_channel",
|
||||
lambda url, **k: {"videos": [{"youtube_id": "r1", "title": "Recent"}]})
|
||||
lambda url, **k: {"youtube_id": "UCx", "videos": [{"youtube_id": "r1", "title": "Recent"}]})
|
||||
monkeypatch.setattr(yt, "video_detail",
|
||||
lambda vid: {"youtube_id": vid, "published_at": "2022-03-03"} if vid in ("w1", "r1") else None)
|
||||
import core.video.youtube_enrichment as mod
|
||||
|
|
|
|||
|
|
@ -1355,7 +1355,7 @@
|
|||
// — this both DATES the recent videos and EXPANDS past the initial ~90 cap.
|
||||
var ytLoadAllToken = 0;
|
||||
function ytCancelLoad() { ytLoadAllToken++; }
|
||||
function ytLoadAllVideos(id) {
|
||||
function ytLoadAllVideos(id, silent) {
|
||||
var token = ++ytLoadAllToken;
|
||||
var byId = {};
|
||||
((data && data._channel && data._channel.videos) || []).forEach(function (v) { byId[v.youtube_id] = v; });
|
||||
|
|
@ -1394,7 +1394,9 @@
|
|||
}
|
||||
cont = resp.continuation;
|
||||
if (cont && data._channel.videos.length < MAX) {
|
||||
showEpSyncing(true, 'Loading the channel’s full video history… ' +
|
||||
// Quiet when refreshing a remembered channel; only the first
|
||||
// (cache-miss) load shows the "loading full history" banner.
|
||||
if (!silent) showEpSyncing(true, 'Loading the channel’s full video history… ' +
|
||||
data._channel.videos.length + ' videos so far.');
|
||||
setTimeout(step, 120);
|
||||
} else {
|
||||
|
|
@ -1431,7 +1433,8 @@
|
|||
if (sub) sub.scrollTop = 0;
|
||||
ytLoadPlaylists(id);
|
||||
// Stream the rest of the catalog (and fill upload dates) in batches.
|
||||
ytLoadAllVideos(id);
|
||||
// A remembered channel renders full from cache → refresh quietly.
|
||||
ytLoadAllVideos(id, !!resp.from_cache);
|
||||
})
|
||||
.catch(function () { showLoading(false); setText('[data-vd-title]', 'Could not load channel'); });
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in a new issue