YouTube dates: proxy is opt-in (public instances are dead); yt-dlp is the default

Live-tested the proxy against real channels — it does not work: Invidious public
APIs are 401/403 (auth-walled/disabled), and the one reachable Piped instance
returns EMPTY video lists for every channel (MrBeast/LTT/Veritasium all 0). Also
fixed a real bug: _harvest bailed on an empty first page instead of following the
nextpage token.

So the proxy is now OPT-IN via a youtube_proxy_instances setting (empty by
default → skipped entirely, no wasted dead calls). The default path is RSS +
the parallelized yt-dlp fallback, which actually works (your log: 9/36/25 dates
cached). Power users can point the setting at a live instance if they find one.

core/video/youtube.py

@@ -316,7 +316,9 @@ def _proxy_get(url, fetch):
 
 
 def _harvest(kind, base, cid, pages, fetch):
-    """Walk one instance's channel pages, accumulating {video_id: date}."""
+    """Walk one instance's channel pages, accumulating {video_id: date}. Follows
+    pagination even when an early page is empty-but-has-a-next-token (some Piped
+    instances return an empty first page), stopping when there's no token left."""
     from urllib.parse import quote
     out, token = {}, None
     for i in range(max(1, pages)):
@@ -327,11 +329,10 @@ def _harvest(kind, base, cid, pages, fetch):
             url = base + "/api/v1/channels/" + cid + "/videos" + \
                 (("?continuation=" + quote(token, safe="")) if token else "")
         obj = _proxy_get(url, fetch)
-        page = parse_proxy_dates(obj if isinstance(obj, dict) else {})
-        if not page and i == 0:
-            return {}                                # instance gave nothing → caller tries next
-        out.update(page)
-        token = (obj.get("nextpage") if kind == "piped" else obj.get("continuation")) if isinstance(obj, dict) else None
+        if not isinstance(obj, dict):
+            break
+        out.update(parse_proxy_dates(obj))
+        token = obj.get("nextpage") if kind == "piped" else obj.get("continuation")
         if not token:
             break
     return out

core/video/youtube_enrichment.py

@@ -46,6 +46,30 @@ class YoutubeDateEnricher:
         from database.video_database import VideoDatabase
         return VideoDatabase()
 
+    @staticmethod
+    def _proxy_instances(db):
+        """Parse the optional youtube_proxy_instances setting into [(kind, url), …].
+        Empty (the default) → proxy skipped entirely. Format is comma/newline
+        separated 'kind|url' (kind inferred from the url if omitted)."""
+        try:
+            raw = db.get_setting("youtube_proxy_instances") or ""
+        except Exception:
+            return []
+        out = []
+        for part in raw.replace("\n", ",").split(","):
+            part = part.strip()
+            if not part:
+                continue
+            if "|" in part:
+                kind, url = part.split("|", 1)
+                kind, url = kind.strip().lower(), url.strip()
+            else:
+                url = part
+                kind = "invidious" if "invidious" in url or "/api/v1" in url else "piped"
+            if url.startswith("http"):
+                out.append((kind, url.rstrip("/")))
+        return out
+
     def enqueue(self, channel_id, title=None):
         """Queue a followed channel for full date enrichment (deduped; starts the
         worker thread on first use)."""
@@ -119,12 +143,18 @@ class YoutubeDateEnricher:
         self._current = self._titles.get(cid) or cid
         logger.debug("Enriching dates for %s (%s)", self._current, cid)
 
+        # Bulk no-key proxy (Piped/Invidious) is OPT-IN via a setting — the public
+        # instances are unreliable/API-disabled, so by default we skip straight to
+        # the yt-dlp path. Power users can point youtube_proxy_instances at a live
+        # instance ("piped|https://…, invidious|https://…").
         dates = {}
-        try:
-            dates = yt.proxy_channel_dates(cid) or {}
-        except Exception:
-            logger.debug("proxy date fetch failed for %s", cid, exc_info=True)
-        logger.debug("proxy returned %d dates for %s", len(dates), cid)
+        instances = self._proxy_instances(db)
+        if instances:
+            try:
+                dates = yt.proxy_channel_dates(cid, instances=instances) or {}
+            except Exception:
+                logger.debug("proxy date fetch failed for %s", cid, exc_info=True)
+            logger.debug("proxy returned %d dates for %s", len(dates), cid)
         if dates:
             db.cache_video_dates([{"youtube_id": k, "published_at": v} for k, v in dates.items()])
             for k, v in dates.items():   # per-item INFO, like the other workers

tests/test_video_youtube_enrichment.py

@@ -19,6 +19,7 @@ def test_enrich_caches_proxy_dates_and_marks_done(db, monkeypatch):
                         lambda cid, *a, **k: {"v1": "2024-06-01", "v2": "2023-02-02"})
     # if proxy covers everything, the per-video fallback must NOT run
     monkeypatch.setattr(yt, "video_detail", lambda vid: (_ for _ in ()).throw(AssertionError("no fallback")))
+    db.set_setting("youtube_proxy_instances", "piped|https://example.test")   # proxy is opt-in
     db.add_videos_to_wishlist({"youtube_id": "UCx", "title": "X"}, [{"youtube_id": "v1", "title": "A"}])
 
     e = YoutubeDateEnricher(db_factory=lambda: db)
@@ -62,3 +63,15 @@ def test_enricher_stats_shape_and_pause():
     assert s["current_item"] is None and "progress" in s
     e.pause(); assert e.stats()["paused"] is True
     e.resume(); assert e.stats()["paused"] is False
+
+
+def test_proxy_instances_setting_parsing(db):
+    e = YoutubeDateEnricher(db_factory=lambda: db)
+    assert e._proxy_instances(db) == []                       # unset → proxy off by default
+    db.set_setting("youtube_proxy_instances",
+                   "piped|https://a.test, https://invidious.b.test/, junk, https://c.test")
+    got = e._proxy_instances(db)
+    assert ("piped", "https://a.test") in got
+    assert ("invidious", "https://invidious.b.test") in got   # kind inferred + trailing / stripped
+    assert ("piped", "https://c.test") in got
+    assert all(u.startswith("http") for _, u in got)          # 'junk' dropped