diff --git a/api/video/youtube.py b/api/video/youtube.py index 771755ea..dfb92a77 100644 --- a/api/video/youtube.py +++ b/api/video/youtube.py @@ -220,28 +220,22 @@ def register_routes(bp): logger.exception("youtube channel detail failed for %r", channel_id) return jsonify({"success": False, "error": "Could not load channel"}), 500 - @bp.route("/youtube/channel//videos", methods=["GET"]) + @bp.route("/youtube/channel//videos", methods=["POST"]) def video_youtube_channel_videos(channel_id): - """A batch of a channel's videos via InnerTube — the channel page streams the - WHOLE catalog by calling this with the continuation token from each response - (each page fetched once; no yt-dlp re-scan). Returns ~a hundred videos with - approximate upload dates (refined from the date cache) + next ``continuation`` - (null = no more).""" + """ONE InnerTube page of a channel's videos — the channel page streams the + WHOLE catalog by re-POSTing with the continuation token from each response + (each page fetched once; no yt-dlp re-scan). POST (not GET) keeps the giant + continuation token out of the URL/access logs; the frontend paces the calls, + so each is fast and never trips the slow-request warning. Returns the videos + (dates refined from cache) + next ``continuation`` (null = no more).""" from . import get_video_db from core.video import youtube as yt - import time - cont = (request.args.get("continuation") or "").strip() or None + body = request.get_json(silent=True) or {} + cont = (body.get("continuation") or "").strip() or None try: db = get_video_db() - videos, token = [], cont - # ~3 InnerTube pages (~90 videos) per request, gently throttled. - for i in range(3): - page = yt.innertube_channel_videos_page(channel_id, continuation=token) - videos.extend(page.get("videos") or []) - token = page.get("continuation") - if not token or len(videos) >= 90: - break - time.sleep(0.2) + page = yt.innertube_channel_videos_page(channel_id, continuation=cont) + videos, token = page.get("videos") or [], page.get("continuation") ids = [v.get("youtube_id") for v in videos if v.get("youtube_id")] cached = db.get_video_dates(ids) wished = db.youtube_video_wish_state(ids) diff --git a/tests/test_video_api.py b/tests/test_video_api.py index 567e6908..8d7731ee 100644 --- a/tests/test_video_api.py +++ b/tests/test_video_api.py @@ -660,21 +660,14 @@ def test_youtube_channel_detail_404_on_unresolvable(tmp_path, monkeypatch): def test_youtube_channel_videos_batch_streams_and_merges(tmp_path, monkeypatch): client, videoapi = _make_client(tmp_path) import core.video.youtube as ytmod - # First call (no continuation) returns two InnerTube videos + a token; a second - # call with that token would return more — but the endpoint stops at the token. + # One POST = one InnerTube page; the token rides in the body (not the URL). calls = [] def fake_page(cid, continuation=None, now=None, post=None): calls.append(continuation) - if continuation is None: - # A full page (>=90) so the endpoint stops AT the token (one batch) rather - # than draining it; a1/a2 are the rows we assert on, the rest are filler. - vids = [{"youtube_id": "a1", "title": "A1", "thumbnail_url": "ta", "published_at": "2020-01-01"}, - {"youtube_id": "a2", "title": "A2", "thumbnail_url": "tb", "published_at": "2019-01-01"}] - vids += [{"youtube_id": "f%d" % i, "title": "F%d" % i, "thumbnail_url": "t", - "published_at": "2018-01-01"} for i in range(95)] - return {"videos": vids, "continuation": "NEXT"} - return {"videos": [], "continuation": None} + return {"videos": [{"youtube_id": "a1", "title": "A1", "thumbnail_url": "ta", "published_at": "2020-01-01"}, + {"youtube_id": "a2", "title": "A2", "thumbnail_url": "tb", "published_at": "2019-01-01"}], + "continuation": "NEXT" if continuation is None else None} monkeypatch.setattr(ytmod, "innertube_channel_videos_page", fake_page) # Seed a cached (exact) date for a1 + wish a2 so the merge is observable. @@ -682,13 +675,18 @@ def test_youtube_channel_videos_batch_streams_and_merges(tmp_path, monkeypatch): videoapi._video_db.add_videos_to_wishlist({"youtube_id": "UCx", "title": "X"}, [{"youtube_id": "a2", "title": "A2"}]) - r = client.get("/api/video/youtube/channel/UCx/videos").get_json() + r = client.post("/api/video/youtube/channel/UCx/videos", json={}).get_json() assert r["success"] is True assert r["continuation"] == "NEXT" # token passed back for the next batch vids = {v["youtube_id"]: v for v in r["videos"]} assert vids["a1"]["published_at"] == "2020-03-15" # cached date wins over InnerTube's assert vids["a1"]["wished"] is False and vids["a2"]["wished"] is True assert calls[0] is None # first batch started from page 1 + # the streamed list is remembered for instant re-open + assert {v["youtube_id"] for v in videoapi._video_db.get_channel_videos("UCx")} == {"a1", "a2"} + # the next page is fetched by POSTing the returned token (kept out of the URL) + r2 = client.post("/api/video/youtube/channel/UCx/videos", json={"continuation": "NEXT"}).get_json() + assert r2["continuation"] is None and calls[1] == "NEXT" def test_youtube_wishlist_add_single_video(tmp_path): diff --git a/webui/static/video/video-detail.js b/webui/static/video/video-detail.js index 232c5f7b..1ec72133 100644 --- a/webui/static/video/video-detail.js +++ b/webui/static/video/video-detail.js @@ -1362,9 +1362,11 @@ var cont = null, MAX = 2000; // safety ceiling for pathological channels function step() { if (token !== ytLoadAllToken || currentId !== id || currentSource !== 'youtube') return; - var url = '/api/video/youtube/channel/' + encodeURIComponent(id) + '/videos' + - (cont ? '?continuation=' + encodeURIComponent(cont) : ''); - fetch(url, { headers: { 'Accept': 'application/json' } }) + // POST so the (huge) continuation token rides in the body, not the URL. + fetch('/api/video/youtube/channel/' + encodeURIComponent(id) + '/videos', { + method: 'POST', headers: { 'Content-Type': 'application/json', 'Accept': 'application/json' }, + body: JSON.stringify({ continuation: cont || null }), + }) .then(function (r) { return r.ok ? r.json() : null; }) .then(function (resp) { if (token !== ytLoadAllToken || currentId !== id || currentSource !== 'youtube') return;