30dc587ebf
Follow a channel → a background job fetches its FULL upload-date catalog so the channel page's year-seasons populate fully, cached permanently (one-time per channel, instant after). - core/video/youtube.py: proxy_channel_dates() — no-key BULK dates via Piped/ Invidious instances (tries several, paginates); parse_proxy_dates handles both shapes. The clean path the user wanted. - core/video/youtube_enrichment.py: YoutubeDateEnricher daemon — enqueue(channel) → proxy bulk → cache; per-video yt-dlp only as a throttled fallback (cap 60, 0.4s) for wished videos when every proxy is down. Skips channels enriched in the last 24h. - db: youtube_channel_enrichment table + mark/check + wishlisted_video_ids_for_ channel. Enqueued from /youtube/follow and on opening a followed channel. Scoped to FOLLOWED channels only (per request). 174 tests green; enricher imports nothing from music.
456 lines
17 KiB
Python
456 lines
17 KiB
Python
"""Resolve a YouTube *channel* reference into a followable channel + its recent
|
|
uploads — the data source behind "follow a YouTube channel" on the video side.
|
|
|
|
yt-dlp only (no API key). The upload list is fetched with ``extract_flat`` so it
|
|
stays cheap even for channels with thousands of videos; the trade-off is that
|
|
per-video publish dates are sparse in flat mode, so they get backfilled later
|
|
(same pattern as the wishlist art backfill). Real *downloads* will additionally
|
|
need Deno + ffmpeg; flat *listing* does not.
|
|
|
|
The URL parsing and the yt-dlp-dict → our-shape mapping are pure functions so
|
|
they're unit-testable without touching the network; the one network call goes
|
|
through ``_extract`` which accepts an injectable factory for tests.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import logging
|
|
import re
|
|
from datetime import datetime, timezone
|
|
from urllib.parse import urlparse
|
|
|
|
try:
|
|
import yt_dlp
|
|
except Exception: # pragma: no cover - yt-dlp is a hard dep, but stay import-safe
|
|
yt_dlp = None
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
# Path prefixes that identify a channel (everything after is the channel key).
|
|
# Tabs like /videos, /streams, /shorts, /featured, /playlists are stripped.
|
|
_CHANNEL_TABS = {"videos", "streams", "shorts", "featured", "playlists", "community", "about"}
|
|
_HANDLE_RE = re.compile(r"^@?[A-Za-z0-9._-]{2,}$")
|
|
_UA = ("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
|
|
"(KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36")
|
|
|
|
|
|
def parse_channel_url(raw):
|
|
"""Normalize a pasted channel reference to a canonical uploads URL, or return
|
|
None if it isn't a channel (a /watch video, a playlist, the YT home, etc.).
|
|
|
|
Accepts: ``https://www.youtube.com/@PlayStation`` (+ /videos etc.),
|
|
``/channel/UC...``, ``/c/Name``, ``/user/Name``, a bare ``@handle``, or a
|
|
bare ``handle``. Returns ``https://www.youtube.com/<base>/videos``.
|
|
"""
|
|
if not raw or not str(raw).strip():
|
|
return None
|
|
raw = str(raw).strip()
|
|
|
|
# Bare handle (no scheme, no slash) → treat as @handle.
|
|
if "/" not in raw and "." not in raw and " " not in raw:
|
|
if _HANDLE_RE.match(raw):
|
|
handle = raw if raw.startswith("@") else "@" + raw
|
|
return "https://www.youtube.com/" + handle + "/videos"
|
|
return None
|
|
|
|
# Give bare "youtube.com/..." a scheme so urlparse populates netloc.
|
|
parse_target = raw if "://" in raw else "https://" + raw
|
|
try:
|
|
u = urlparse(parse_target)
|
|
except Exception:
|
|
return None
|
|
host = (u.netloc or "").lower()
|
|
if host.startswith("www."):
|
|
host = host[4:]
|
|
if host not in ("youtube.com", "m.youtube.com", "music.youtube.com"):
|
|
return None
|
|
|
|
segs = [s for s in (u.path or "").split("/") if s]
|
|
if not segs:
|
|
return None
|
|
|
|
first = segs[0]
|
|
if first.startswith("@"):
|
|
base = first # /@handle
|
|
elif first in ("channel", "c", "user") and len(segs) >= 2:
|
|
base = first + "/" + segs[1] # /channel/UC.., /c/Name, /user/Name
|
|
else:
|
|
return None # /watch, /playlist, /shorts/<id>, home…
|
|
|
|
return "https://www.youtube.com/" + base + "/videos"
|
|
|
|
|
|
def _best_thumb(thumbs):
|
|
"""Pick the highest-resolution thumbnail URL from a yt-dlp thumbnails list."""
|
|
if not thumbs:
|
|
return None
|
|
best, best_area = None, -1
|
|
for t in thumbs:
|
|
if not isinstance(t, dict) or not t.get("url"):
|
|
continue
|
|
area = (t.get("width") or 0) * (t.get("height") or 0)
|
|
# preference value as a tie-breaker when dimensions are absent
|
|
score = area if area else (t.get("preference") or 0)
|
|
if score >= best_area:
|
|
best, best_area = t.get("url"), score
|
|
return best
|
|
|
|
|
|
def _thumb_by_id(thumbs, keyword):
|
|
"""The highest-res thumbnail whose yt-dlp id mentions ``keyword`` (e.g.
|
|
'avatar' / 'banner'), or None — channels expose both in one thumbnails list."""
|
|
hits = [t for t in (thumbs or []) if isinstance(t, dict) and t.get("url")
|
|
and keyword in str(t.get("id", "")).lower()]
|
|
return _best_thumb(hits)
|
|
|
|
|
|
def _entry_date(e):
|
|
"""ISO date (YYYY-MM-DD) for a flat entry, or None — flat mode often omits it."""
|
|
ts = e.get("timestamp")
|
|
if isinstance(ts, (int, float)):
|
|
try:
|
|
return datetime.fromtimestamp(ts, tz=timezone.utc).date().isoformat()
|
|
except Exception:
|
|
pass
|
|
ud = e.get("upload_date") # 'YYYYMMDD'
|
|
if isinstance(ud, str) and len(ud) == 8 and ud.isdigit():
|
|
return f"{ud[0:4]}-{ud[4:6]}-{ud[6:8]}"
|
|
return None
|
|
|
|
|
|
def _shape_entries(entries, limit):
|
|
"""Flat playlist/channel entries → our lightweight video shape (id-less and
|
|
null entries dropped)."""
|
|
out = []
|
|
for e in [x for x in (entries or []) if isinstance(x, dict)][:limit]:
|
|
vid = e.get("id")
|
|
if not vid:
|
|
continue
|
|
dur = e.get("duration")
|
|
out.append({
|
|
"youtube_id": vid,
|
|
"title": e.get("title") or "",
|
|
"published_at": _entry_date(e),
|
|
"duration_seconds": int(dur) if isinstance(dur, (int, float)) else None,
|
|
"thumbnail_url": _best_thumb(e.get("thumbnails")) or e.get("thumbnail"),
|
|
"view_count": e.get("view_count"),
|
|
"description": e.get("description"),
|
|
})
|
|
return out
|
|
|
|
|
|
def shape_channel(info, limit=30):
|
|
"""Map a yt-dlp channel info dict to our followable-channel shape."""
|
|
info = info or {}
|
|
videos = _shape_entries(info.get("entries"), limit)
|
|
handle = info.get("uploader_id") or info.get("channel_id_handle")
|
|
if handle and not str(handle).startswith("@"):
|
|
handle = None # uploader_id is sometimes the UC id, not a handle
|
|
thumbs = info.get("thumbnails")
|
|
return {
|
|
"youtube_id": info.get("channel_id") or info.get("id"),
|
|
"title": info.get("channel") or info.get("uploader") or info.get("title") or "",
|
|
"handle": handle,
|
|
"avatar_url": _thumb_by_id(thumbs, "avatar") or _best_thumb(thumbs),
|
|
"banner_url": _thumb_by_id(thumbs, "banner"),
|
|
"description": info.get("description"),
|
|
"subscriber_count": info.get("channel_follower_count"),
|
|
"view_count": info.get("view_count"),
|
|
"tags": (info.get("tags") or [])[:12],
|
|
"video_count": info.get("playlist_count") or len(videos),
|
|
"videos": videos,
|
|
}
|
|
|
|
|
|
def _cookie_opts():
|
|
"""The music client's cookie convention so age/region-gated content works."""
|
|
try:
|
|
from config.settings import config_manager
|
|
cb = config_manager.get("youtube.cookies_browser", "")
|
|
if cb:
|
|
return {"cookiesfrombrowser": (cb,)}
|
|
except Exception:
|
|
pass
|
|
return {}
|
|
|
|
|
|
def _ydl_opts(limit, db=None):
|
|
opts = {
|
|
"quiet": True,
|
|
"no_warnings": True,
|
|
"extract_flat": True, # fast: enumerate uploads without per-video format probing
|
|
"skip_download": True,
|
|
"playlistend": int(limit),
|
|
"user_agent": _UA,
|
|
}
|
|
opts.update(_cookie_opts())
|
|
return opts
|
|
|
|
|
|
def shape_video(info):
|
|
"""Map yt-dlp's FULL single-video info dict to our rich video shape (the data
|
|
flat-listing can't give: description, views, likes, duration, tags)."""
|
|
info = info or {}
|
|
dur = info.get("duration")
|
|
return {
|
|
"youtube_id": info.get("id"),
|
|
"title": info.get("title") or "",
|
|
"description": info.get("description"),
|
|
"duration_seconds": int(dur) if isinstance(dur, (int, float)) else None,
|
|
"view_count": info.get("view_count"),
|
|
"like_count": info.get("like_count"),
|
|
"published_at": _entry_date(info),
|
|
"thumbnail_url": _best_thumb(info.get("thumbnails")) or info.get("thumbnail"),
|
|
"channel_title": info.get("channel") or info.get("uploader"),
|
|
"channel_id": info.get("channel_id"),
|
|
"webpage_url": info.get("webpage_url") or ("https://www.youtube.com/watch?v=" + (info.get("id") or "")),
|
|
"tags": info.get("tags") or [],
|
|
}
|
|
|
|
|
|
def video_detail(video_id, ydl_factory=None):
|
|
"""Full metadata for ONE video (non-flat extract) — done lazily on click, the
|
|
way the TV nebula lazy-loads guest stars. Accepts a raw id or a watch URL."""
|
|
if not video_id:
|
|
return None
|
|
vid = str(video_id).strip()
|
|
url = vid if vid.startswith("http") else "https://www.youtube.com/watch?v=" + vid
|
|
opts = {"quiet": True, "no_warnings": True, "skip_download": True, "user_agent": _UA}
|
|
opts.update(_cookie_opts())
|
|
info = _extract(url, opts, ydl_factory)
|
|
if not info or not info.get("id"):
|
|
return None
|
|
return shape_video(info)
|
|
|
|
|
|
def _extract(url, opts, ydl_factory=None):
|
|
"""Run yt-dlp's extract_info; isolated for test injection. Returns a dict or None."""
|
|
factory = ydl_factory or (yt_dlp.YoutubeDL if yt_dlp else None)
|
|
if factory is None:
|
|
logger.warning("yt-dlp unavailable; cannot resolve YouTube channel")
|
|
return None
|
|
try:
|
|
with factory(opts) as ydl:
|
|
return ydl.extract_info(url, download=False)
|
|
except Exception as e:
|
|
logger.info("YouTube channel resolve failed for %s: %s", url, e)
|
|
return None
|
|
|
|
|
|
def resolve_channel(raw, limit=30, ydl_factory=None, db=None):
|
|
"""Resolve a pasted channel reference to ``{youtube_id, title, handle,
|
|
avatar_url, videos:[...], ...}``, or None if it isn't a resolvable channel."""
|
|
url = parse_channel_url(raw)
|
|
if not url:
|
|
return None
|
|
info = _extract(url, _ydl_opts(limit, db), ydl_factory)
|
|
if not info:
|
|
return None
|
|
shaped = shape_channel(info, limit)
|
|
return shaped if shaped.get("youtube_id") else None
|
|
|
|
|
|
# No-key bulk date sources: community Piped/Invidious instances. Flaky (they go
|
|
# up/down), so we try several and fall back to per-video yt-dlp if none answer.
|
|
_PROXY_INSTANCES = [
|
|
("piped", "https://pipedapi.kavin.rocks"),
|
|
("piped", "https://pipedapi.adminforge.de"),
|
|
("piped", "https://api.piped.private.coffee"),
|
|
("invidious", "https://invidious.nerdvpn.de"),
|
|
("invidious", "https://inv.nadeko.net"),
|
|
]
|
|
|
|
|
|
def _epoch_to_date(v):
|
|
"""epoch seconds OR milliseconds → 'YYYY-MM-DD', or None."""
|
|
try:
|
|
n = float(v)
|
|
except (TypeError, ValueError):
|
|
return None
|
|
if n <= 0:
|
|
return None
|
|
if n > 1e12: # milliseconds
|
|
n /= 1000.0
|
|
try:
|
|
return datetime.fromtimestamp(n, tz=timezone.utc).date().isoformat()
|
|
except Exception:
|
|
return None
|
|
|
|
|
|
def _vid_from_url(u):
|
|
m = re.search(r"[?&]v=([\w-]+)", u or "")
|
|
return m.group(1) if m else None
|
|
|
|
|
|
def parse_proxy_dates(obj):
|
|
"""{video_id: 'YYYY-MM-DD'} from a Piped (relatedStreams) or Invidious (videos)
|
|
channel JSON response — handles both shapes."""
|
|
out = {}
|
|
if not isinstance(obj, dict):
|
|
return out
|
|
for s in (obj.get("relatedStreams") or []): # Piped
|
|
if not isinstance(s, dict):
|
|
continue
|
|
vid = _vid_from_url(s.get("url")) or s.get("videoId")
|
|
d = _epoch_to_date(s.get("uploaded"))
|
|
if vid and d:
|
|
out[vid] = d
|
|
for v in (obj.get("videos") or []): # Invidious
|
|
if not isinstance(v, dict):
|
|
continue
|
|
vid = v.get("videoId")
|
|
d = _epoch_to_date(v.get("published"))
|
|
if vid and d:
|
|
out[vid] = d
|
|
return out
|
|
|
|
|
|
def _proxy_get(url, fetch):
|
|
if fetch is not None:
|
|
return fetch(url)
|
|
import requests
|
|
r = requests.get(url, timeout=8, headers={"User-Agent": _UA, "Accept": "application/json"})
|
|
return r.json() if r.status_code == 200 else None
|
|
|
|
|
|
def _harvest(kind, base, cid, pages, fetch):
|
|
"""Walk one instance's channel pages, accumulating {video_id: date}."""
|
|
from urllib.parse import quote
|
|
out, token = {}, None
|
|
for i in range(max(1, pages)):
|
|
if kind == "piped":
|
|
url = (base + "/channel/" + cid) if token is None \
|
|
else (base + "/nextpage/channel/" + cid + "?nextpage=" + quote(token, safe=""))
|
|
else: # invidious
|
|
url = base + "/api/v1/channels/" + cid + "/videos" + \
|
|
(("?continuation=" + quote(token, safe="")) if token else "")
|
|
obj = _proxy_get(url, fetch)
|
|
page = parse_proxy_dates(obj if isinstance(obj, dict) else {})
|
|
if not page and i == 0:
|
|
return {} # instance gave nothing → caller tries next
|
|
out.update(page)
|
|
token = (obj.get("nextpage") if kind == "piped" else obj.get("continuation")) if isinstance(obj, dict) else None
|
|
if not token:
|
|
break
|
|
return out
|
|
|
|
|
|
def proxy_channel_dates(channel_id, pages=6, fetch=None, instances=None):
|
|
"""Bulk upload dates for a whole channel via a no-key proxy (Piped/Invidious),
|
|
paginated. Tries instances until one answers. {video_id: 'YYYY-MM-DD'} (empty
|
|
if all are down → caller falls back to per-video yt-dlp)."""
|
|
cid = str(channel_id or "").strip()
|
|
if not cid:
|
|
return {}
|
|
for kind, base in (instances or _PROXY_INSTANCES):
|
|
try:
|
|
dates = _harvest(kind, base, cid, pages, fetch)
|
|
if dates:
|
|
return dates
|
|
except Exception:
|
|
continue
|
|
return {}
|
|
|
|
|
|
def parse_rss_dates(xml_text):
|
|
"""{video_id: 'YYYY-MM-DD'} from a YouTube channel RSS feed (pure, testable)."""
|
|
import xml.etree.ElementTree as ET
|
|
out = {}
|
|
try:
|
|
root = ET.fromstring(xml_text)
|
|
except Exception:
|
|
return out
|
|
ns = {"a": "http://www.w3.org/2005/Atom", "yt": "http://www.youtube.com/xml/schemas/2015"}
|
|
for entry in root.findall("a:entry", ns):
|
|
vid = entry.find("yt:videoId", ns)
|
|
pub = entry.find("a:published", ns)
|
|
if vid is not None and vid.text and pub is not None and pub.text:
|
|
out[vid.text.strip()] = pub.text.strip()[:10] # ISO datetime → YYYY-MM-DD
|
|
return out
|
|
|
|
|
|
def channel_recent_dates(channel_id, fetch=None):
|
|
"""Real upload dates for a channel's ~15 most-recent videos via its public RSS
|
|
feed — one cheap GET, no yt-dlp, no bot risk. {video_id: 'YYYY-MM-DD'}."""
|
|
cid = str(channel_id or "").strip()
|
|
if not cid:
|
|
return {}
|
|
url = "https://www.youtube.com/feeds/videos.xml?channel_id=" + cid
|
|
try:
|
|
if fetch is not None:
|
|
return parse_rss_dates(fetch(url) or "")
|
|
import requests
|
|
r = requests.get(url, timeout=10, headers={"User-Agent": _UA})
|
|
return parse_rss_dates(r.text) if r.status_code == 200 else {}
|
|
except Exception as e:
|
|
logger.info("YouTube RSS dates failed for %s: %s", cid, e)
|
|
return {}
|
|
|
|
|
|
def search_channels(query, limit=6, ydl_factory=None):
|
|
"""Search YouTube for CHANNELS (the results page filtered to channels) → a few
|
|
{youtube_id, title, handle, avatar_url, subscriber_count} for the search page.
|
|
Best-effort; entries that aren't channels are skipped."""
|
|
from urllib.parse import quote
|
|
q = (query or "").strip()
|
|
if not q:
|
|
return []
|
|
# sp=EgIQAg%3D%3D is YouTube's "Type: Channel" search filter.
|
|
url = "https://www.youtube.com/results?search_query=" + quote(q) + "&sp=EgIQAg%3D%3D"
|
|
info = _extract(url, _ydl_opts(limit * 3), ydl_factory) # over-fetch; some entries may be videos
|
|
out = []
|
|
for e in [x for x in ((info or {}).get("entries") or []) if isinstance(x, dict)]:
|
|
cid = e.get("channel_id")
|
|
if not cid and str(e.get("id", "")).startswith("UC"):
|
|
cid = e.get("id")
|
|
if not cid:
|
|
m = re.search(r"/channel/(UC[\w-]+)", e.get("url") or "")
|
|
if m:
|
|
cid = m.group(1)
|
|
if not cid or not str(cid).startswith("UC"):
|
|
continue
|
|
title = e.get("channel") or e.get("title") or e.get("uploader") or ""
|
|
if not title:
|
|
continue
|
|
uid = e.get("uploader_id")
|
|
out.append({
|
|
"youtube_id": cid,
|
|
"title": title,
|
|
"handle": uid if str(uid or "").startswith("@") else None,
|
|
"avatar_url": _best_thumb(e.get("thumbnails")) or e.get("thumbnail"),
|
|
"subscriber_count": e.get("channel_follower_count"),
|
|
"video_count": e.get("playlist_count"),
|
|
})
|
|
if len(out) >= limit:
|
|
break
|
|
return out
|
|
|
|
|
|
def channel_playlists(channel_id, limit=50, ydl_factory=None):
|
|
"""The channel's playlists (flat) → [{playlist_id, title, video_count,
|
|
thumbnail_url}] — rendered as "seasons" on the channel page. Lazy-loaded."""
|
|
if not channel_id:
|
|
return []
|
|
url = "https://www.youtube.com/channel/" + str(channel_id) + "/playlists"
|
|
info = _extract(url, _ydl_opts(limit), ydl_factory)
|
|
out = []
|
|
for e in [x for x in ((info or {}).get("entries") or []) if isinstance(x, dict)][:limit]:
|
|
pid = e.get("id")
|
|
if not pid:
|
|
continue
|
|
out.append({
|
|
"playlist_id": pid,
|
|
"title": e.get("title") or "",
|
|
"video_count": e.get("playlist_count") or e.get("video_count"),
|
|
"thumbnail_url": _best_thumb(e.get("thumbnails")) or e.get("thumbnail"),
|
|
})
|
|
return out
|
|
|
|
|
|
def playlist_videos(playlist_id, limit=50, ydl_factory=None):
|
|
"""A playlist's videos (flat), in the same shape as channel uploads."""
|
|
if not playlist_id:
|
|
return []
|
|
url = "https://www.youtube.com/playlist?list=" + str(playlist_id)
|
|
info = _extract(url, _ydl_opts(limit), ydl_factory)
|
|
return _shape_entries((info or {}).get("entries"), limit)
|