YouTube enricher: reliable full coverage without a proxy + sweep existing follows
Two gaps from real use:
- When all proxy instances are down (common), the fallback only dated WISHED
videos, leaving most as 'Earlier videos'. Now on proxy failure it flat-resolves
the channel's recent uploads and dates THOSE via yt-dlp (throttled, cached) —
so years populate fully even with no working proxy.
- A channel followed before the enricher existed was never queued ('nothing to
process'). /youtube/channels now sweeps: any followed channel not enriched
recently gets enqueued — so viewing the Channels tab (or the wishlist badge
refresh) picks up existing follows.
The detail page still reads cache+RSS (fast); the enricher fills the cache. 51
tests green.
This commit is contained in:
parent
b6d8f07516
commit
9524a40615
3 changed files with 31 additions and 6 deletions
|
|
@ -111,11 +111,22 @@ def register_routes(bp):
|
|||
|
||||
@bp.route("/youtube/channels", methods=["GET"])
|
||||
def video_youtube_channels():
|
||||
"""Followed channels (newest first) for the watchlist page."""
|
||||
"""Followed channels (newest first) for the watchlist page. Also sweeps:
|
||||
any followed channel not date-enriched recently gets queued for the
|
||||
background enricher (so existing follows get picked up, not just new ones)."""
|
||||
from . import get_video_db
|
||||
try:
|
||||
db = get_video_db()
|
||||
return jsonify({"success": True, "channels": db.list_watchlist_channels(),
|
||||
channels = db.list_watchlist_channels()
|
||||
try:
|
||||
from core.video.youtube_enrichment import get_youtube_date_enricher
|
||||
enr = get_youtube_date_enricher()
|
||||
for c in channels:
|
||||
if not db.channel_dates_enriched_recently(c["youtube_id"]):
|
||||
enr.enqueue(c["youtube_id"], c.get("title"))
|
||||
except Exception:
|
||||
pass
|
||||
return jsonify({"success": True, "channels": channels,
|
||||
"counts": db.youtube_wishlist_counts()})
|
||||
except Exception:
|
||||
logger.exception("youtube channels list failed")
|
||||
|
|
|
|||
|
|
@ -121,8 +121,18 @@ class YoutubeDateEnricher:
|
|||
db.cache_video_dates([{"youtube_id": k, "published_at": v} for k, v in dates.items()])
|
||||
logger.info("YouTube dates: %d cached for %s (proxy)", len(dates), cid)
|
||||
|
||||
# Fallback: per-video for the channel's wished videos still lacking a date.
|
||||
ids = db.wishlisted_video_ids_for_channel(cid)
|
||||
# Fallback (proxy down): date the channel's RECENT UPLOADS via yt-dlp, not
|
||||
# just wished videos — so years populate fully even with no working proxy.
|
||||
ids = set(db.wishlisted_video_ids_for_channel(cid))
|
||||
if not dates:
|
||||
try:
|
||||
ch = yt.resolve_channel("https://www.youtube.com/channel/" + cid, limit=60)
|
||||
for v in (ch or {}).get("videos") or []:
|
||||
if v.get("youtube_id"):
|
||||
ids.add(v["youtube_id"])
|
||||
except Exception:
|
||||
logger.info("flat resolve for date fallback failed for %s", cid, exc_info=True)
|
||||
ids = list(ids)
|
||||
have = db.get_video_dates(ids)
|
||||
missing = [i for i in ids if i not in have and i not in dates]
|
||||
filled = 0
|
||||
|
|
|
|||
|
|
@ -33,8 +33,11 @@ def test_enrich_caches_proxy_dates_and_marks_done(db, monkeypatch):
|
|||
def test_enrich_falls_back_to_per_video_when_proxy_empty(db, monkeypatch):
|
||||
import core.video.youtube as yt
|
||||
monkeypatch.setattr(yt, "proxy_channel_dates", lambda cid, *a, **k: {}) # all proxies down
|
||||
# flat resolve adds a recent upload r1 to the date-fallback set (besides wished w1/w2)
|
||||
monkeypatch.setattr(yt, "resolve_channel",
|
||||
lambda url, **k: {"videos": [{"youtube_id": "r1", "title": "Recent"}]})
|
||||
monkeypatch.setattr(yt, "video_detail",
|
||||
lambda vid: {"youtube_id": vid, "published_at": "2022-03-03"} if vid == "w1" else None)
|
||||
lambda vid: {"youtube_id": vid, "published_at": "2022-03-03"} if vid in ("w1", "r1") else None)
|
||||
import core.video.youtube_enrichment as mod
|
||||
monkeypatch.setattr(mod.time, "sleep", lambda s: None) # no real throttle delay in tests
|
||||
db.add_videos_to_wishlist({"youtube_id": "UCx", "title": "X"},
|
||||
|
|
@ -42,7 +45,8 @@ def test_enrich_falls_back_to_per_video_when_proxy_empty(db, monkeypatch):
|
|||
|
||||
e = YoutubeDateEnricher(db_factory=lambda: db)
|
||||
e._enrich("UCx")
|
||||
assert db.get_video_dates(["w1", "w2"]) == {"w1": "2022-03-03"} # only w1 had a date
|
||||
# w1 (wished) + r1 (recent upload from flat resolve) got dates; w2 had none
|
||||
assert db.get_video_dates(["w1", "w2", "r1"]) == {"w1": "2022-03-03", "r1": "2022-03-03"}
|
||||
assert db.channel_dates_enriched_recently("UCx") is True
|
||||
|
||||
|
||||
|
|
|
|||
Loading…
Reference in a new issue