diff --git a/api/video/youtube.py b/api/video/youtube.py index 6be64b47..252e526a 100644 --- a/api/video/youtube.py +++ b/api/video/youtube.py @@ -204,6 +204,46 @@ def register_routes(bp): logger.exception("youtube channel detail failed for %r", channel_id) return jsonify({"success": False, "error": "Could not load channel"}), 500 + @bp.route("/youtube/channel//videos", methods=["GET"]) + def video_youtube_channel_videos(channel_id): + """A batch of a channel's videos via InnerTube — the channel page streams the + WHOLE catalog by calling this with the continuation token from each response + (each page fetched once; no yt-dlp re-scan). Returns ~a hundred videos with + approximate upload dates (refined from the date cache) + next ``continuation`` + (null = no more).""" + from . import get_video_db + from core.video import youtube as yt + import time + cont = (request.args.get("continuation") or "").strip() or None + try: + db = get_video_db() + videos, token = [], cont + # ~3 InnerTube pages (~90 videos) per request, gently throttled. + for i in range(3): + page = yt.innertube_channel_videos_page(channel_id, continuation=token) + videos.extend(page.get("videos") or []) + token = page.get("continuation") + if not token or len(videos) >= 90: + break + time.sleep(0.2) + ids = [v.get("youtube_id") for v in videos if v.get("youtube_id")] + cached = db.get_video_dates(ids) + wished = db.youtube_video_wish_state(ids) + for v in videos: + vid = v.get("youtube_id") + if cached.get(vid): # a cached (possibly exact) date wins + v["published_at"] = cached[vid] + v["wished"] = vid in wished + try: + db.cache_video_dates([{"youtube_id": v["youtube_id"], "published_at": v.get("published_at")} + for v in videos if v.get("youtube_id") and v.get("published_at")]) + except Exception: + pass + return jsonify({"success": True, "videos": videos, "continuation": token}) + except Exception: + logger.exception("youtube channel videos batch failed for %r", channel_id) + return jsonify({"success": False, "videos": [], "continuation": None}), 200 + @bp.route("/youtube/video/", methods=["GET"]) def video_youtube_video_detail(video_id): """Full metadata for one video (description, views, likes, duration, tags) — diff --git a/core/video/youtube.py b/core/video/youtube.py index 8b7e66f0..0562a0e3 100644 --- a/core/video/youtube.py +++ b/core/video/youtube.py @@ -512,6 +512,69 @@ def innertube_channel_dates(channel_id, pages=_INNERTUBE_PAGES, now=None, post=N return out +def _lockup_title(lk): + """The video title from a lockupViewModel (metadata.title.content).""" + for md in _json_find_all(lk, "lockupMetadataViewModel", []): + if isinstance(md, dict) and isinstance(md.get("title"), dict): + c = md["title"].get("content") + if isinstance(c, str) and c: + return c + return None + + +def _lockup_thumb(lk): + """The first thumbnail url from a lockupViewModel (contentImage…sources[0].url).""" + for srcs in _json_find_all(lk.get("contentImage") or {}, "sources", []): + if isinstance(srcs, list) and srcs and isinstance(srcs[0], dict) and srcs[0].get("url"): + return srcs[0]["url"] + return None + + +def innertube_parse_video_items(obj): + """Full per-video metadata for VIDEO lockups in an InnerTube browse/continuation + response: [{youtube_id, title, thumbnail_url, relative}]. Same source as + innertube_parse_videos but keeps title + thumbnail so the channel page can list + the whole catalog (not just date them).""" + out, seen = [], set() + for lk in _json_find_all(obj, "lockupViewModel", []): + if not isinstance(lk, dict) or lk.get("contentType") != "LOCKUP_CONTENT_TYPE_VIDEO": + continue + vid = lk.get("contentId") + if not vid or vid in seen: + continue + seen.add(vid) + rel = next((t for t in _json_content_strings(lk.get("metadata"), []) if _REL_RE.search(t)), None) + out.append({"youtube_id": vid, "title": _lockup_title(lk), + "thumbnail_url": _lockup_thumb(lk), "relative": rel}) + return out + + +def innertube_channel_videos_page(channel_id, continuation=None, now=None, post=None): + """ONE InnerTube page of a channel's videos (with metadata) + the next token: + {"videos": [{youtube_id, title, thumbnail_url, published_at}], "continuation": token|None}. + Stateless — hand the returned token back to fetch the next page, so the caller + can stream the FULL catalog in batches (O(n), each page fetched once). {} fields + / None token on any failure.""" + cid = str(channel_id or "").strip() + if not continuation and not cid.startswith("UC"): + return {"videos": [], "continuation": None} + if now is None: + now = datetime.now(timezone.utc).date() + payload = ({"context": _INNERTUBE_CTX, "continuation": continuation} if continuation + else {"context": _INNERTUBE_CTX, "browseId": cid, "params": _VIDEOS_PARAMS}) + try: + j = _innertube_post(payload, post) + except Exception: + return {"videos": [], "continuation": None} + if not isinstance(j, dict): + return {"videos": [], "continuation": None} + videos = [] + for it in innertube_parse_video_items(j): + it["published_at"] = relative_to_date(it.pop("relative", None), now) + videos.append(it) + return {"videos": videos, "continuation": innertube_continuation(j)} + + def search_channels(query, limit=6, ydl_factory=None): """Search YouTube for CHANNELS (the results page filtered to channels) → a few {youtube_id, title, handle, avatar_url, subscriber_count} for the search page. diff --git a/tests/test_video_api.py b/tests/test_video_api.py index b1922cd7..f6972371 100644 --- a/tests/test_video_api.py +++ b/tests/test_video_api.py @@ -636,6 +636,40 @@ def test_youtube_channel_detail_404_on_unresolvable(tmp_path, monkeypatch): assert client.get("/api/video/youtube/channel/UCnope").status_code == 404 +def test_youtube_channel_videos_batch_streams_and_merges(tmp_path, monkeypatch): + client, videoapi = _make_client(tmp_path) + import core.video.youtube as ytmod + # First call (no continuation) returns two InnerTube videos + a token; a second + # call with that token would return more — but the endpoint stops at the token. + calls = [] + + def fake_page(cid, continuation=None, now=None, post=None): + calls.append(continuation) + if continuation is None: + # A full page (>=90) so the endpoint stops AT the token (one batch) rather + # than draining it; a1/a2 are the rows we assert on, the rest are filler. + vids = [{"youtube_id": "a1", "title": "A1", "thumbnail_url": "ta", "published_at": "2020-01-01"}, + {"youtube_id": "a2", "title": "A2", "thumbnail_url": "tb", "published_at": "2019-01-01"}] + vids += [{"youtube_id": "f%d" % i, "title": "F%d" % i, "thumbnail_url": "t", + "published_at": "2018-01-01"} for i in range(95)] + return {"videos": vids, "continuation": "NEXT"} + return {"videos": [], "continuation": None} + + monkeypatch.setattr(ytmod, "innertube_channel_videos_page", fake_page) + # Seed a cached (exact) date for a1 + wish a2 so the merge is observable. + videoapi._video_db.cache_video_dates([{"youtube_id": "a1", "published_at": "2020-03-15"}]) + videoapi._video_db.add_videos_to_wishlist({"youtube_id": "UCx", "title": "X"}, + [{"youtube_id": "a2", "title": "A2"}]) + + r = client.get("/api/video/youtube/channel/UCx/videos").get_json() + assert r["success"] is True + assert r["continuation"] == "NEXT" # token passed back for the next batch + vids = {v["youtube_id"]: v for v in r["videos"]} + assert vids["a1"]["published_at"] == "2020-03-15" # cached date wins over InnerTube's + assert vids["a1"]["wished"] is False and vids["a2"]["wished"] is True + assert calls[0] is None # first batch started from page 1 + + def test_youtube_wishlist_add_single_video(tmp_path): client, _ = _make_client(tmp_path) r = client.post("/api/video/youtube/wishlist/add", json={ diff --git a/tests/test_video_youtube.py b/tests/test_video_youtube.py index a5e7d2ce..c529cba9 100644 --- a/tests/test_video_youtube.py +++ b/tests/test_video_youtube.py @@ -456,3 +456,47 @@ def test_innertube_channel_dates_guards(): assert yt.innertube_channel_dates("not-a-uc-id", post=lambda p: {}) == {} # only UC… channels assert yt.innertube_channel_dates("UCx", post=lambda p: None) == {} # bad/empty response → {} assert yt.innertube_channel_dates("UCx", post=lambda p: (_ for _ in ()).throw(RuntimeError())) == {} + + +def _lk_item_thumb(vid, rel, thumb): + """_lk_item plus a contentImage so thumbnail extraction can be exercised.""" + item = _lk_item(vid, rel) + lk = item["richItemRenderer"]["content"]["lockupViewModel"] + lk["contentImage"] = {"thumbnailViewModel": {"image": {"sources": [{"url": thumb}]}}} + return item + + +def test_innertube_parse_video_items_keeps_title_and_thumbnail(): + page = _page([ + _lk_item_thumb("v1", "2 years ago", "https://i.ytimg.com/vi/v1/hq.jpg"), + _lk_item("v2", "3 months ago"), # no thumbnail → None + _lk_item("p1", "1 day ago", ctype="LOCKUP_CONTENT_TYPE_PLAYLIST"), # not a video → skip + ]) + items = yt.innertube_parse_video_items(page) + assert [it["youtube_id"] for it in items] == ["v1", "v2"] # playlist filtered out + assert items[0] == {"youtube_id": "v1", "title": "Title v1", + "thumbnail_url": "https://i.ytimg.com/vi/v1/hq.jpg", "relative": "2 years ago"} + assert items[1]["title"] == "Title v2" and items[1]["thumbnail_url"] is None + + +def test_innertube_channel_videos_page_converts_and_pages(): + now = date(2026, 6, 17) + page1 = _page([_lk_item_thumb("v1", "1 year ago", "t1"), _lk_item("v2", "2 years ago")], token="TOK") + got = yt.innertube_channel_videos_page("UCxxxx", now=now, post=lambda p: page1) + assert got["continuation"] == "TOK" + assert got["videos"][0] == {"youtube_id": "v1", "title": "Title v1", + "thumbnail_url": "t1", "published_at": "2025-06-17"} + assert got["videos"][1]["published_at"] == "2024-06-17" + # a continuation request carries the token (not browseId) and ends when none returned + end = yt.innertube_channel_videos_page("UCxxxx", continuation="TOK", now=now, + post=lambda p: (_page([_lk_item("v3", "5 days ago")]) + if p.get("continuation") == "TOK" else None)) + assert [v["youtube_id"] for v in end["videos"]] == ["v3"] and end["continuation"] is None + + +def test_innertube_channel_videos_page_guards(): + assert yt.innertube_channel_videos_page("not-uc", post=lambda p: {}) == {"videos": [], "continuation": None} + assert yt.innertube_channel_videos_page("UCx", post=lambda p: None) == {"videos": [], "continuation": None} + # a continuation token works even without a UC id (it encodes the channel itself) + assert yt.innertube_channel_videos_page("", continuation="TOK", + post=lambda p: _page([_lk_item("v9", "1 day ago")]))["videos"] diff --git a/webui/static/video/video-detail.js b/webui/static/video/video-detail.js index 0b673daf..81c038c8 100644 --- a/webui/static/video/video-detail.js +++ b/webui/static/video/video-detail.js @@ -1348,44 +1348,62 @@ episode_total: (ch.videos || []).length, episode_owned: 0 }; } - // Freshly-followed channels get their upload dates filled in over the next - // minute+ by the background enricher. Re-fetch a few times so new year-seasons - // pop in WITHOUT a manual reload (only re-renders if more years appeared). - var ytRepollTimers = []; - function ytClearRepoll() { ytRepollTimers.forEach(function (t) { clearTimeout(t); }); ytRepollTimers = []; } - function _undatedCount(seasons) { - var n = 0; - (seasons || []).forEach(function (s) { if (s.season_number === 0) n = s.episodes.length; }); - return n; - } - function ytRefetch(id) { - var before = ((data && data.seasons) || []).length; - var undatedBefore = _undatedCount(data && data.seasons); - fetch('/api/video/youtube/channel/' + encodeURIComponent(id) + '?limit=90', { headers: { 'Accept': 'application/json' } }) - .then(function (r) { return r.ok ? r.json() : null; }) - .then(function (resp) { - if (!resp || !resp.success || currentId !== id || currentSource !== 'youtube') return; - var nd = ytToShow(resp); - // Re-render when dating IMPROVED — new year-seasons appeared OR the - // "Earlier videos" (undated) bucket shrank. (Season count alone misses - // the common case where many recent videos all collapse into one year.) - if (nd.seasons.length <= before && _undatedCount(nd.seasons) >= undatedBefore) return; - data = nd; - if (!seasonByNum(selectedSeason)) selectedSeason = nd.seasons.length ? nd.seasons[0].season_number : null; - renderBillboard(nd); renderSeasonNav(); renderEpisodes(); - if (!nd.seasons.some(function (s) { return s.season_number === 0; })) { showEpSyncing(false); ytClearRepoll(); } - }) - .catch(function () { /* ignore */ }); - } - function ytScheduleRepoll(id) { - ytClearRepoll(); - // Tell the user dates are being fetched (the enricher takes ~30-45s, more - // if the channel is queued behind others), and poll until they land. - showEpSyncing(true, 'Fetching upload dates from YouTube… your year-seasons will fill in shortly.'); - [20000, 45000, 80000, 120000, 160000].forEach(function (ms) { - ytRepollTimers.push(setTimeout(function () { ytRefetch(id); }, ms)); - }); - ytRepollTimers.push(setTimeout(function () { if (currentId === id) showEpSyncing(false); }, 175000)); + // Stream the channel's FULL video catalog in batches via InnerTube (each page + // fetched once, light on rate limits) and fold it into the year-seasons live: + // each batch fills missing upload dates on the videos already shown AND appends + // older ones, re-rendering after every batch. Replaces the old date-only re-poll + // — this both DATES the recent videos and EXPANDS past the initial ~90 cap. + var ytLoadAllToken = 0; + function ytCancelLoad() { ytLoadAllToken++; } + function ytLoadAllVideos(id) { + var token = ++ytLoadAllToken; + var byId = {}; + ((data && data._channel && data._channel.videos) || []).forEach(function (v) { byId[v.youtube_id] = v; }); + var cont = null, MAX = 2000; // safety ceiling for pathological channels + function step() { + if (token !== ytLoadAllToken || currentId !== id || currentSource !== 'youtube') return; + var url = '/api/video/youtube/channel/' + encodeURIComponent(id) + '/videos' + + (cont ? '?continuation=' + encodeURIComponent(cont) : ''); + fetch(url, { headers: { 'Accept': 'application/json' } }) + .then(function (r) { return r.ok ? r.json() : null; }) + .then(function (resp) { + if (token !== ytLoadAllToken || currentId !== id || currentSource !== 'youtube') return; + if (!resp || !resp.success) { showEpSyncing(false); return; } + var prevSel = selectedSeason; + var selObj = seasonByNum(prevSel); + var prevEp = selObj ? selObj.episodes.length : -1; + var changed = false; + (resp.videos || []).forEach(function (v) { + if (!v.youtube_id) return; + var ex = byId[v.youtube_id]; + if (ex) { // already shown → backfill a missing date + if (!ex.published_at && v.published_at) { ex.published_at = v.published_at; changed = true; } + } else { // older video → add it + byId[v.youtube_id] = v; data._channel.videos.push(v); changed = true; + } + }); + if (changed) { + var nd = ytToShow({ channel: data._channel, following: data.following }); + data = nd; + if (!seasonByNum(prevSel)) selectedSeason = nd.seasons.length ? nd.seasons[0].season_number : null; + renderBillboard(nd); renderSeasonNav(); + // Only re-render the episode grid if the season you're viewing + // actually changed (most batches add OLDER years, not yours). + var nowObj = seasonByNum(selectedSeason); + if (selectedSeason !== prevSel || !nowObj || nowObj.episodes.length !== prevEp) renderEpisodes(); + } + cont = resp.continuation; + if (cont && data._channel.videos.length < MAX) { + showEpSyncing(true, 'Loading the channel’s full video history… ' + + data._channel.videos.length + ' videos so far.'); + setTimeout(step, 120); + } else { + showEpSyncing(false); + } + }) + .catch(function () { showEpSyncing(false); }); + } + step(); } function loadChannel(id) { @@ -1393,7 +1411,7 @@ if (currentId !== id) artAttemptedFor = null; currentId = id; if (!root()) return; - ytClearRepoll(); + ytCancelLoad(); showLoading(true); resetExtras(); showEpSyncing(false); ['[data-vd-episodes]', '[data-vd-season-nav]'].forEach(function (s) { var n = q(s); if (n) n.innerHTML = ''; }); var r0 = root(); if (r0) r0.style.removeProperty('--vd-accent-rgb'); @@ -1412,9 +1430,8 @@ var sub = document.querySelector('.video-subpage[data-video-subpage="video-show-detail"]'); if (sub) sub.scrollTop = 0; ytLoadPlaylists(id); - // If videos are still undated, the enricher is (or will be) filling - // them in — poll a few times so the years appear live. - if (data.seasons.some(function (s) { return s.season_number === 0; })) ytScheduleRepoll(id); + // Stream the rest of the catalog (and fill upload dates) in batches. + ytLoadAllVideos(id); }) .catch(function () { showLoading(false); setText('[data-vd-title]', 'Could not load channel'); }); }