Channel page: stream the FULL video catalog in batches (no more 90 cap)

The channel page was capped at the recent 90 (yt-dlp flat, hard min(90,limit)),
so prolific channels (Ninja Kidz etc.) showed only ~90 of hundreds. Now it loads
everything via YouTube's InnerTube API, paginated by continuation token — each
page fetched once (O(n), light on rate limits), unlike yt-dlp offset batches
which re-scan from the start every time.

- core: innertube_parse_video_items (keeps title+thumbnail, not just dates) +
  innertube_channel_videos_page(channel_id, continuation) → {videos, continuation}.
- api: GET /youtube/channel/<id>/videos?continuation=<tok> returns ~a batch of
  videos (approx dates, refined from the date cache) + the next token.
- frontend: ytLoadAllVideos streams batches after the fast initial 90 render,
  folding each into the year-seasons live — backfills dates on the videos already
  shown AND appends older ones, re-rendering per batch (episode grid only when the
  viewed season changed). Replaces the old date-only re-poll (this dates AND
  expands). Safety ceiling 2000; a 'Loading full history… N so far' indicator.

Validated live (MrBeast: 30/page, clean continuation, all dated). 101 tests green.

api/video/youtube.py

@@ -204,6 +204,46 @@ def register_routes(bp):
             logger.exception("youtube channel detail failed for %r", channel_id)
             return jsonify({"success": False, "error": "Could not load channel"}), 500
 
+    @bp.route("/youtube/channel/<channel_id>/videos", methods=["GET"])
+    def video_youtube_channel_videos(channel_id):
+        """A batch of a channel's videos via InnerTube — the channel page streams the
+        WHOLE catalog by calling this with the continuation token from each response
+        (each page fetched once; no yt-dlp re-scan). Returns ~a hundred videos with
+        approximate upload dates (refined from the date cache) + next ``continuation``
+        (null = no more)."""
+        from . import get_video_db
+        from core.video import youtube as yt
+        import time
+        cont = (request.args.get("continuation") or "").strip() or None
+        try:
+            db = get_video_db()
+            videos, token = [], cont
+            # ~3 InnerTube pages (~90 videos) per request, gently throttled.
+            for i in range(3):
+                page = yt.innertube_channel_videos_page(channel_id, continuation=token)
+                videos.extend(page.get("videos") or [])
+                token = page.get("continuation")
+                if not token or len(videos) >= 90:
+                    break
+                time.sleep(0.2)
+            ids = [v.get("youtube_id") for v in videos if v.get("youtube_id")]
+            cached = db.get_video_dates(ids)
+            wished = db.youtube_video_wish_state(ids)
+            for v in videos:
+                vid = v.get("youtube_id")
+                if cached.get(vid):                 # a cached (possibly exact) date wins
+                    v["published_at"] = cached[vid]
+                v["wished"] = vid in wished
+            try:
+                db.cache_video_dates([{"youtube_id": v["youtube_id"], "published_at": v.get("published_at")}
+                                      for v in videos if v.get("youtube_id") and v.get("published_at")])
+            except Exception:
+                pass
+            return jsonify({"success": True, "videos": videos, "continuation": token})
+        except Exception:
+            logger.exception("youtube channel videos batch failed for %r", channel_id)
+            return jsonify({"success": False, "videos": [], "continuation": None}), 200
+
     @bp.route("/youtube/video/<video_id>", methods=["GET"])
     def video_youtube_video_detail(video_id):
         """Full metadata for one video (description, views, likes, duration, tags) —

core/video/youtube.py

@@ -512,6 +512,69 @@ def innertube_channel_dates(channel_id, pages=_INNERTUBE_PAGES, now=None, post=N
     return out
 
 
+def _lockup_title(lk):
+    """The video title from a lockupViewModel (metadata.title.content)."""
+    for md in _json_find_all(lk, "lockupMetadataViewModel", []):
+        if isinstance(md, dict) and isinstance(md.get("title"), dict):
+            c = md["title"].get("content")
+            if isinstance(c, str) and c:
+                return c
+    return None
+
+
+def _lockup_thumb(lk):
+    """The first thumbnail url from a lockupViewModel (contentImage…sources[0].url)."""
+    for srcs in _json_find_all(lk.get("contentImage") or {}, "sources", []):
+        if isinstance(srcs, list) and srcs and isinstance(srcs[0], dict) and srcs[0].get("url"):
+            return srcs[0]["url"]
+    return None
+
+
+def innertube_parse_video_items(obj):
+    """Full per-video metadata for VIDEO lockups in an InnerTube browse/continuation
+    response: [{youtube_id, title, thumbnail_url, relative}]. Same source as
+    innertube_parse_videos but keeps title + thumbnail so the channel page can list
+    the whole catalog (not just date them)."""
+    out, seen = [], set()
+    for lk in _json_find_all(obj, "lockupViewModel", []):
+        if not isinstance(lk, dict) or lk.get("contentType") != "LOCKUP_CONTENT_TYPE_VIDEO":
+            continue
+        vid = lk.get("contentId")
+        if not vid or vid in seen:
+            continue
+        seen.add(vid)
+        rel = next((t for t in _json_content_strings(lk.get("metadata"), []) if _REL_RE.search(t)), None)
+        out.append({"youtube_id": vid, "title": _lockup_title(lk),
+                    "thumbnail_url": _lockup_thumb(lk), "relative": rel})
+    return out
+
+
+def innertube_channel_videos_page(channel_id, continuation=None, now=None, post=None):
+    """ONE InnerTube page of a channel's videos (with metadata) + the next token:
+    {"videos": [{youtube_id, title, thumbnail_url, published_at}], "continuation": token|None}.
+    Stateless — hand the returned token back to fetch the next page, so the caller
+    can stream the FULL catalog in batches (O(n), each page fetched once). {} fields
+    / None token on any failure."""
+    cid = str(channel_id or "").strip()
+    if not continuation and not cid.startswith("UC"):
+        return {"videos": [], "continuation": None}
+    if now is None:
+        now = datetime.now(timezone.utc).date()
+    payload = ({"context": _INNERTUBE_CTX, "continuation": continuation} if continuation
+               else {"context": _INNERTUBE_CTX, "browseId": cid, "params": _VIDEOS_PARAMS})
+    try:
+        j = _innertube_post(payload, post)
+    except Exception:
+        return {"videos": [], "continuation": None}
+    if not isinstance(j, dict):
+        return {"videos": [], "continuation": None}
+    videos = []
+    for it in innertube_parse_video_items(j):
+        it["published_at"] = relative_to_date(it.pop("relative", None), now)
+        videos.append(it)
+    return {"videos": videos, "continuation": innertube_continuation(j)}
+
+
 def search_channels(query, limit=6, ydl_factory=None):
     """Search YouTube for CHANNELS (the results page filtered to channels) → a few
     {youtube_id, title, handle, avatar_url, subscriber_count} for the search page.

tests/test_video_api.py

@@ -636,6 +636,40 @@ def test_youtube_channel_detail_404_on_unresolvable(tmp_path, monkeypatch):
     assert client.get("/api/video/youtube/channel/UCnope").status_code == 404
 
 
+def test_youtube_channel_videos_batch_streams_and_merges(tmp_path, monkeypatch):
+    client, videoapi = _make_client(tmp_path)
+    import core.video.youtube as ytmod
+    # First call (no continuation) returns two InnerTube videos + a token; a second
+    # call with that token would return more — but the endpoint stops at the token.
+    calls = []
+
+    def fake_page(cid, continuation=None, now=None, post=None):
+        calls.append(continuation)
+        if continuation is None:
+            # A full page (>=90) so the endpoint stops AT the token (one batch) rather
+            # than draining it; a1/a2 are the rows we assert on, the rest are filler.
+            vids = [{"youtube_id": "a1", "title": "A1", "thumbnail_url": "ta", "published_at": "2020-01-01"},
+                    {"youtube_id": "a2", "title": "A2", "thumbnail_url": "tb", "published_at": "2019-01-01"}]
+            vids += [{"youtube_id": "f%d" % i, "title": "F%d" % i, "thumbnail_url": "t",
+                      "published_at": "2018-01-01"} for i in range(95)]
+            return {"videos": vids, "continuation": "NEXT"}
+        return {"videos": [], "continuation": None}
+
+    monkeypatch.setattr(ytmod, "innertube_channel_videos_page", fake_page)
+    # Seed a cached (exact) date for a1 + wish a2 so the merge is observable.
+    videoapi._video_db.cache_video_dates([{"youtube_id": "a1", "published_at": "2020-03-15"}])
+    videoapi._video_db.add_videos_to_wishlist({"youtube_id": "UCx", "title": "X"},
+                                              [{"youtube_id": "a2", "title": "A2"}])
+
+    r = client.get("/api/video/youtube/channel/UCx/videos").get_json()
+    assert r["success"] is True
+    assert r["continuation"] == "NEXT"                      # token passed back for the next batch
+    vids = {v["youtube_id"]: v for v in r["videos"]}
+    assert vids["a1"]["published_at"] == "2020-03-15"       # cached date wins over InnerTube's
+    assert vids["a1"]["wished"] is False and vids["a2"]["wished"] is True
+    assert calls[0] is None                                 # first batch started from page 1
+
+
 def test_youtube_wishlist_add_single_video(tmp_path):
     client, _ = _make_client(tmp_path)
     r = client.post("/api/video/youtube/wishlist/add", json={

tests/test_video_youtube.py

@@ -456,3 +456,47 @@ def test_innertube_channel_dates_guards():
     assert yt.innertube_channel_dates("not-a-uc-id", post=lambda p: {}) == {}     # only UC… channels
     assert yt.innertube_channel_dates("UCx", post=lambda p: None) == {}           # bad/empty response → {}
     assert yt.innertube_channel_dates("UCx", post=lambda p: (_ for _ in ()).throw(RuntimeError())) == {}
+
+
+def _lk_item_thumb(vid, rel, thumb):
+    """_lk_item plus a contentImage so thumbnail extraction can be exercised."""
+    item = _lk_item(vid, rel)
+    lk = item["richItemRenderer"]["content"]["lockupViewModel"]
+    lk["contentImage"] = {"thumbnailViewModel": {"image": {"sources": [{"url": thumb}]}}}
+    return item
+
+
+def test_innertube_parse_video_items_keeps_title_and_thumbnail():
+    page = _page([
+        _lk_item_thumb("v1", "2 years ago", "https://i.ytimg.com/vi/v1/hq.jpg"),
+        _lk_item("v2", "3 months ago"),                                       # no thumbnail → None
+        _lk_item("p1", "1 day ago", ctype="LOCKUP_CONTENT_TYPE_PLAYLIST"),    # not a video → skip
+    ])
+    items = yt.innertube_parse_video_items(page)
+    assert [it["youtube_id"] for it in items] == ["v1", "v2"]                 # playlist filtered out
+    assert items[0] == {"youtube_id": "v1", "title": "Title v1",
+                        "thumbnail_url": "https://i.ytimg.com/vi/v1/hq.jpg", "relative": "2 years ago"}
+    assert items[1]["title"] == "Title v2" and items[1]["thumbnail_url"] is None
+
+
+def test_innertube_channel_videos_page_converts_and_pages():
+    now = date(2026, 6, 17)
+    page1 = _page([_lk_item_thumb("v1", "1 year ago", "t1"), _lk_item("v2", "2 years ago")], token="TOK")
+    got = yt.innertube_channel_videos_page("UCxxxx", now=now, post=lambda p: page1)
+    assert got["continuation"] == "TOK"
+    assert got["videos"][0] == {"youtube_id": "v1", "title": "Title v1",
+                                "thumbnail_url": "t1", "published_at": "2025-06-17"}
+    assert got["videos"][1]["published_at"] == "2024-06-17"
+    # a continuation request carries the token (not browseId) and ends when none returned
+    end = yt.innertube_channel_videos_page("UCxxxx", continuation="TOK", now=now,
+                                           post=lambda p: (_page([_lk_item("v3", "5 days ago")])
+                                                           if p.get("continuation") == "TOK" else None))
+    assert [v["youtube_id"] for v in end["videos"]] == ["v3"] and end["continuation"] is None
+
+
+def test_innertube_channel_videos_page_guards():
+    assert yt.innertube_channel_videos_page("not-uc", post=lambda p: {}) == {"videos": [], "continuation": None}
+    assert yt.innertube_channel_videos_page("UCx", post=lambda p: None) == {"videos": [], "continuation": None}
+    # a continuation token works even without a UC id (it encodes the channel itself)
+    assert yt.innertube_channel_videos_page("", continuation="TOK",
+                                            post=lambda p: _page([_lk_item("v9", "1 day ago")]))["videos"]

webui/static/video/video-detail.js

@@ -1348,44 +1348,62 @@
             episode_total: (ch.videos || []).length, episode_owned: 0 };
     }
 
-    // Freshly-followed channels get their upload dates filled in over the next
-    // minute+ by the background enricher. Re-fetch a few times so new year-seasons
-    // pop in WITHOUT a manual reload (only re-renders if more years appeared).
-    var ytRepollTimers = [];
-    function ytClearRepoll() { ytRepollTimers.forEach(function (t) { clearTimeout(t); }); ytRepollTimers = []; }
-    function _undatedCount(seasons) {
-        var n = 0;
-        (seasons || []).forEach(function (s) { if (s.season_number === 0) n = s.episodes.length; });
-        return n;
-    }
-    function ytRefetch(id) {
-        var before = ((data && data.seasons) || []).length;
-        var undatedBefore = _undatedCount(data && data.seasons);
-        fetch('/api/video/youtube/channel/' + encodeURIComponent(id) + '?limit=90', { headers: { 'Accept': 'application/json' } })
-            .then(function (r) { return r.ok ? r.json() : null; })
-            .then(function (resp) {
-                if (!resp || !resp.success || currentId !== id || currentSource !== 'youtube') return;
-                var nd = ytToShow(resp);
-                // Re-render when dating IMPROVED — new year-seasons appeared OR the
-                // "Earlier videos" (undated) bucket shrank. (Season count alone misses
-                // the common case where many recent videos all collapse into one year.)
-                if (nd.seasons.length <= before && _undatedCount(nd.seasons) >= undatedBefore) return;
-                data = nd;
-                if (!seasonByNum(selectedSeason)) selectedSeason = nd.seasons.length ? nd.seasons[0].season_number : null;
-                renderBillboard(nd); renderSeasonNav(); renderEpisodes();
-                if (!nd.seasons.some(function (s) { return s.season_number === 0; })) { showEpSyncing(false); ytClearRepoll(); }
-            })
-            .catch(function () { /* ignore */ });
-    }
-    function ytScheduleRepoll(id) {
-        ytClearRepoll();
-        // Tell the user dates are being fetched (the enricher takes ~30-45s, more
-        // if the channel is queued behind others), and poll until they land.
-        showEpSyncing(true, 'Fetching upload dates from YouTube… your year-seasons will fill in shortly.');
-        [20000, 45000, 80000, 120000, 160000].forEach(function (ms) {
-            ytRepollTimers.push(setTimeout(function () { ytRefetch(id); }, ms));
-        });
-        ytRepollTimers.push(setTimeout(function () { if (currentId === id) showEpSyncing(false); }, 175000));
+    // Stream the channel's FULL video catalog in batches via InnerTube (each page
+    // fetched once, light on rate limits) and fold it into the year-seasons live:
+    // each batch fills missing upload dates on the videos already shown AND appends
+    // older ones, re-rendering after every batch. Replaces the old date-only re-poll
+    // — this both DATES the recent videos and EXPANDS past the initial ~90 cap.
+    var ytLoadAllToken = 0;
+    function ytCancelLoad() { ytLoadAllToken++; }
+    function ytLoadAllVideos(id) {
+        var token = ++ytLoadAllToken;
+        var byId = {};
+        ((data && data._channel && data._channel.videos) || []).forEach(function (v) { byId[v.youtube_id] = v; });
+        var cont = null, MAX = 2000;   // safety ceiling for pathological channels
+        function step() {
+            if (token !== ytLoadAllToken || currentId !== id || currentSource !== 'youtube') return;
+            var url = '/api/video/youtube/channel/' + encodeURIComponent(id) + '/videos' +
+                (cont ? '?continuation=' + encodeURIComponent(cont) : '');
+            fetch(url, { headers: { 'Accept': 'application/json' } })
+                .then(function (r) { return r.ok ? r.json() : null; })
+                .then(function (resp) {
+                    if (token !== ytLoadAllToken || currentId !== id || currentSource !== 'youtube') return;
+                    if (!resp || !resp.success) { showEpSyncing(false); return; }
+                    var prevSel = selectedSeason;
+                    var selObj = seasonByNum(prevSel);
+                    var prevEp = selObj ? selObj.episodes.length : -1;
+                    var changed = false;
+                    (resp.videos || []).forEach(function (v) {
+                        if (!v.youtube_id) return;
+                        var ex = byId[v.youtube_id];
+                        if (ex) {                                   // already shown → backfill a missing date
+                            if (!ex.published_at && v.published_at) { ex.published_at = v.published_at; changed = true; }
+                        } else {                                    // older video → add it
+                            byId[v.youtube_id] = v; data._channel.videos.push(v); changed = true;
+                        }
+                    });
+                    if (changed) {
+                        var nd = ytToShow({ channel: data._channel, following: data.following });
+                        data = nd;
+                        if (!seasonByNum(prevSel)) selectedSeason = nd.seasons.length ? nd.seasons[0].season_number : null;
+                        renderBillboard(nd); renderSeasonNav();
+                        // Only re-render the episode grid if the season you're viewing
+                        // actually changed (most batches add OLDER years, not yours).
+                        var nowObj = seasonByNum(selectedSeason);
+                        if (selectedSeason !== prevSel || !nowObj || nowObj.episodes.length !== prevEp) renderEpisodes();
+                    }
+                    cont = resp.continuation;
+                    if (cont && data._channel.videos.length < MAX) {
+                        showEpSyncing(true, 'Loading the channel’s full video history… ' +
+                            data._channel.videos.length + ' videos so far.');
+                        setTimeout(step, 120);
+                    } else {
+                        showEpSyncing(false);
+                    }
+                })
+                .catch(function () { showEpSyncing(false); });
+        }
+        step();
     }
 
     function loadChannel(id) {
@@ -1393,7 +1411,7 @@
         if (currentId !== id) artAttemptedFor = null;
         currentId = id;
         if (!root()) return;
-        ytClearRepoll();
+        ytCancelLoad();
         showLoading(true); resetExtras(); showEpSyncing(false);
         ['[data-vd-episodes]', '[data-vd-season-nav]'].forEach(function (s) { var n = q(s); if (n) n.innerHTML = ''; });
         var r0 = root(); if (r0) r0.style.removeProperty('--vd-accent-rgb');
@@ -1412,9 +1430,8 @@
                 var sub = document.querySelector('.video-subpage[data-video-subpage="video-show-detail"]');
                 if (sub) sub.scrollTop = 0;
                 ytLoadPlaylists(id);
-                // If videos are still undated, the enricher is (or will be) filling
-                // them in — poll a few times so the years appear live.
-                if (data.seasons.some(function (s) { return s.season_number === 0; })) ytScheduleRepoll(id);
+                // Stream the rest of the catalog (and fill upload dates) in batches.
+                ytLoadAllVideos(id);
             })
             .catch(function () { showLoading(false); setText('[data-vd-title]', 'Could not load channel'); });
     }