From a83df05ce25a384993bf9621b0f62b926b253bed Mon Sep 17 00:00:00 2001 From: BoulderBadgeDad Date: Wed, 17 Jun 2026 12:30:21 -0700 Subject: [PATCH] YouTube dates: proxy is opt-in (public instances are dead); yt-dlp is the default MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Live-tested the proxy against real channels — it does not work: Invidious public APIs are 401/403 (auth-walled/disabled), and the one reachable Piped instance returns EMPTY video lists for every channel (MrBeast/LTT/Veritasium all 0). Also fixed a real bug: _harvest bailed on an empty first page instead of following the nextpage token. So the proxy is now OPT-IN via a youtube_proxy_instances setting (empty by default → skipped entirely, no wasted dead calls). The default path is RSS + the parallelized yt-dlp fallback, which actually works (your log: 9/36/25 dates cached). Power users can point the setting at a live instance if they find one. --- core/video/youtube.py | 13 +++++---- core/video/youtube_enrichment.py | 40 ++++++++++++++++++++++---- tests/test_video_youtube_enrichment.py | 13 +++++++++ 3 files changed, 55 insertions(+), 11 deletions(-) diff --git a/core/video/youtube.py b/core/video/youtube.py index 75c0afb8..ca3628ff 100644 --- a/core/video/youtube.py +++ b/core/video/youtube.py @@ -316,7 +316,9 @@ def _proxy_get(url, fetch): def _harvest(kind, base, cid, pages, fetch): - """Walk one instance's channel pages, accumulating {video_id: date}.""" + """Walk one instance's channel pages, accumulating {video_id: date}. Follows + pagination even when an early page is empty-but-has-a-next-token (some Piped + instances return an empty first page), stopping when there's no token left.""" from urllib.parse import quote out, token = {}, None for i in range(max(1, pages)): @@ -327,11 +329,10 @@ def _harvest(kind, base, cid, pages, fetch): url = base + "/api/v1/channels/" + cid + "/videos" + \ (("?continuation=" + quote(token, safe="")) if token else "") obj = _proxy_get(url, fetch) - page = parse_proxy_dates(obj if isinstance(obj, dict) else {}) - if not page and i == 0: - return {} # instance gave nothing → caller tries next - out.update(page) - token = (obj.get("nextpage") if kind == "piped" else obj.get("continuation")) if isinstance(obj, dict) else None + if not isinstance(obj, dict): + break + out.update(parse_proxy_dates(obj)) + token = obj.get("nextpage") if kind == "piped" else obj.get("continuation") if not token: break return out diff --git a/core/video/youtube_enrichment.py b/core/video/youtube_enrichment.py index d36947ac..b73d3583 100644 --- a/core/video/youtube_enrichment.py +++ b/core/video/youtube_enrichment.py @@ -46,6 +46,30 @@ class YoutubeDateEnricher: from database.video_database import VideoDatabase return VideoDatabase() + @staticmethod + def _proxy_instances(db): + """Parse the optional youtube_proxy_instances setting into [(kind, url), …]. + Empty (the default) → proxy skipped entirely. Format is comma/newline + separated 'kind|url' (kind inferred from the url if omitted).""" + try: + raw = db.get_setting("youtube_proxy_instances") or "" + except Exception: + return [] + out = [] + for part in raw.replace("\n", ",").split(","): + part = part.strip() + if not part: + continue + if "|" in part: + kind, url = part.split("|", 1) + kind, url = kind.strip().lower(), url.strip() + else: + url = part + kind = "invidious" if "invidious" in url or "/api/v1" in url else "piped" + if url.startswith("http"): + out.append((kind, url.rstrip("/"))) + return out + def enqueue(self, channel_id, title=None): """Queue a followed channel for full date enrichment (deduped; starts the worker thread on first use).""" @@ -119,12 +143,18 @@ class YoutubeDateEnricher: self._current = self._titles.get(cid) or cid logger.debug("Enriching dates for %s (%s)", self._current, cid) + # Bulk no-key proxy (Piped/Invidious) is OPT-IN via a setting — the public + # instances are unreliable/API-disabled, so by default we skip straight to + # the yt-dlp path. Power users can point youtube_proxy_instances at a live + # instance ("piped|https://…, invidious|https://…"). dates = {} - try: - dates = yt.proxy_channel_dates(cid) or {} - except Exception: - logger.debug("proxy date fetch failed for %s", cid, exc_info=True) - logger.debug("proxy returned %d dates for %s", len(dates), cid) + instances = self._proxy_instances(db) + if instances: + try: + dates = yt.proxy_channel_dates(cid, instances=instances) or {} + except Exception: + logger.debug("proxy date fetch failed for %s", cid, exc_info=True) + logger.debug("proxy returned %d dates for %s", len(dates), cid) if dates: db.cache_video_dates([{"youtube_id": k, "published_at": v} for k, v in dates.items()]) for k, v in dates.items(): # per-item INFO, like the other workers diff --git a/tests/test_video_youtube_enrichment.py b/tests/test_video_youtube_enrichment.py index 0fa71ea3..01d98b5c 100644 --- a/tests/test_video_youtube_enrichment.py +++ b/tests/test_video_youtube_enrichment.py @@ -19,6 +19,7 @@ def test_enrich_caches_proxy_dates_and_marks_done(db, monkeypatch): lambda cid, *a, **k: {"v1": "2024-06-01", "v2": "2023-02-02"}) # if proxy covers everything, the per-video fallback must NOT run monkeypatch.setattr(yt, "video_detail", lambda vid: (_ for _ in ()).throw(AssertionError("no fallback"))) + db.set_setting("youtube_proxy_instances", "piped|https://example.test") # proxy is opt-in db.add_videos_to_wishlist({"youtube_id": "UCx", "title": "X"}, [{"youtube_id": "v1", "title": "A"}]) e = YoutubeDateEnricher(db_factory=lambda: db) @@ -62,3 +63,15 @@ def test_enricher_stats_shape_and_pause(): assert s["current_item"] is None and "progress" in s e.pause(); assert e.stats()["paused"] is True e.resume(); assert e.stats()["paused"] is False + + +def test_proxy_instances_setting_parsing(db): + e = YoutubeDateEnricher(db_factory=lambda: db) + assert e._proxy_instances(db) == [] # unset → proxy off by default + db.set_setting("youtube_proxy_instances", + "piped|https://a.test, https://invidious.b.test/, junk, https://c.test") + got = e._proxy_instances(db) + assert ("piped", "https://a.test") in got + assert ("invidious", "https://invidious.b.test") in got # kind inferred + trailing / stripped + assert ("piped", "https://c.test") in got + assert all(u.startswith("http") for _, u in got) # 'junk' dropped