danvics/soulsync
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 1

Channel videos: POST + one page per request (kill the giant slow-request log spam)

Browse source 
Each /videos batch was 3 InnerTube pages + 0.4s sleeps (>1s → tripped the
slow-request WARNING) and carried the ~2KB continuation token in the URL, so
every batch dumped a giant warning line — a dozen per channel open. Now it's a
POST (token in the body, never the URL) fetching ONE page per call with no
server sleep, so each request is fast (<1s, no warning) and the frontend's 120ms
pacing keeps it polite. 17 youtube API tests green.
This commit is contained in:
BoulderBadgeDad 2026-06-17 18:19:32 -07:00
parent 267f11c848
commit 66f3f97460
3 changed files with 26 additions and 32 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

28
api/video/youtube.py
View file

@ -220,28 +220,22 @@ def register_routes(bp):
logger.exception("youtube channel detail failed for %r", channel_id)
return jsonify({"success": False, "error": "Could not load channel"}), 500
@bp.route("/youtube/channel/<channel_id>/videos", methods=["GET"])
@bp.route("/youtube/channel/<channel_id>/videos", methods=["POST"])
def video_youtube_channel_videos(channel_id):
"""A batch of a channel's videos via InnerTube — the channel page streams the
WHOLE catalog by calling this with the continuation token from each response
(each page fetched once; no yt-dlp re-scan). Returns ~a hundred videos with
approximate upload dates (refined from the date cache) + next ``continuation``
(null = no more)."""
"""ONE InnerTube page of a channel's videos — the channel page streams the
WHOLE catalog by re-POSTing with the continuation token from each response
(each page fetched once; no yt-dlp re-scan). POST (not GET) keeps the giant
continuation token out of the URL/access logs; the frontend paces the calls,
so each is fast and never trips the slow-request warning. Returns the videos
(dates refined from cache) + next ``continuation`` (null = no more)."""
from . import get_video_db
from core.video import youtube as yt
import time
cont = (request.args.get("continuation") or "").strip() or None
body = request.get_json(silent=True) or {}
cont = (body.get("continuation") or "").strip() or None
try:
db = get_video_db()
videos, token = [], cont
# ~3 InnerTube pages (~90 videos) per request, gently throttled.
for i in range(3):
page = yt.innertube_channel_videos_page(channel_id, continuation=token)
videos.extend(page.get("videos") or [])
token = page.get("continuation")
if not token or len(videos) >= 90:
break
time.sleep(0.2)
page = yt.innertube_channel_videos_page(channel_id, continuation=cont)
videos, token = page.get("videos") or [], page.get("continuation")
ids = [v.get("youtube_id") for v in videos if v.get("youtube_id")]
cached = db.get_video_dates(ids)
wished = db.youtube_video_wish_state(ids)

22
tests/test_video_api.py
View file

@ -660,21 +660,14 @@ def test_youtube_channel_detail_404_on_unresolvable(tmp_path, monkeypatch):
def test_youtube_channel_videos_batch_streams_and_merges(tmp_path, monkeypatch):
client, videoapi = _make_client(tmp_path)
import core.video.youtube as ytmod
# First call (no continuation) returns two InnerTube videos + a token; a second
# call with that token would return more — but the endpoint stops at the token.
# One POST = one InnerTube page; the token rides in the body (not the URL).
calls = []
def fake_page(cid, continuation=None, now=None, post=None):
calls.append(continuation)
if continuation is None:
# A full page (>=90) so the endpoint stops AT the token (one batch) rather
# than draining it; a1/a2 are the rows we assert on, the rest are filler.
vids = [{"youtube_id": "a1", "title": "A1", "thumbnail_url": "ta", "published_at": "2020-01-01"},
{"youtube_id": "a2", "title": "A2", "thumbnail_url": "tb", "published_at": "2019-01-01"}]
vids += [{"youtube_id": "f%d" % i, "title": "F%d" % i, "thumbnail_url": "t",
"published_at": "2018-01-01"} for i in range(95)]
return {"videos": vids, "continuation": "NEXT"}
return {"videos": [], "continuation": None}
return {"videos": [{"youtube_id": "a1", "title": "A1", "thumbnail_url": "ta", "published_at": "2020-01-01"},
{"youtube_id": "a2", "title": "A2", "thumbnail_url": "tb", "published_at": "2019-01-01"}],
"continuation": "NEXT" if continuation is None else None}
monkeypatch.setattr(ytmod, "innertube_channel_videos_page", fake_page)
# Seed a cached (exact) date for a1 + wish a2 so the merge is observable.
@ -682,13 +675,18 @@ def test_youtube_channel_videos_batch_streams_and_merges(tmp_path, monkeypatch):
videoapi._video_db.add_videos_to_wishlist({"youtube_id": "UCx", "title": "X"},
[{"youtube_id": "a2", "title": "A2"}])
r = client.get("/api/video/youtube/channel/UCx/videos").get_json()
r = client.post("/api/video/youtube/channel/UCx/videos", json={}).get_json()
assert r["success"] is True
assert r["continuation"] == "NEXT" # token passed back for the next batch
vids = {v["youtube_id"]: v for v in r["videos"]}
assert vids["a1"]["published_at"] == "2020-03-15" # cached date wins over InnerTube's
assert vids["a1"]["wished"] is False and vids["a2"]["wished"] is True
assert calls[0] is None # first batch started from page 1
# the streamed list is remembered for instant re-open
assert {v["youtube_id"] for v in videoapi._video_db.get_channel_videos("UCx")} == {"a1", "a2"}
# the next page is fetched by POSTing the returned token (kept out of the URL)
r2 = client.post("/api/video/youtube/channel/UCx/videos", json={"continuation": "NEXT"}).get_json()
assert r2["continuation"] is None and calls[1] == "NEXT"
def test_youtube_wishlist_add_single_video(tmp_path):

8
webui/static/video/video-detail.js
View file

@ -1362,9 +1362,11 @@
var cont = null, MAX = 2000; // safety ceiling for pathological channels
function step() {
if (token !== ytLoadAllToken || currentId !== id || currentSource !== 'youtube') return;
var url = '/api/video/youtube/channel/' + encodeURIComponent(id) + '/videos' +
(cont ? '?continuation=' + encodeURIComponent(cont) : '');
fetch(url, { headers: { 'Accept': 'application/json' } })
// POST so the (huge) continuation token rides in the body, not the URL.
fetch('/api/video/youtube/channel/' + encodeURIComponent(id) + '/videos', {
method: 'POST', headers: { 'Content-Type': 'application/json', 'Accept': 'application/json' },
body: JSON.stringify({ continuation: cont || null }),
})
.then(function (r) { return r.ok ? r.json() : null; })
.then(function (resp) {
if (token !== ytLoadAllToken || currentId !== id || currentSource !== 'youtube') return;