danvics/soulsync
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 1

Channel year-seasons: RSS dates + persistent date cache (real years that fill in)

Browse source 
Flat listing has no upload dates, so channels showed one 'All Videos' season.
Now real year-seasons, filled from cheap sources and cached so they grow:
- core: parse_rss_dates / channel_recent_dates — one public RSS GET dates the
  ~15 most-recent uploads (no yt-dlp, no bot risk).
- db: youtube_video_dates cache table + cache_video_dates / get_video_dates.
- /youtube/channel merges cached + RSS dates onto the flat videos (year-seasons)
  and caches what it learns; /youtube/video caches each fetched date too — so
  expanding/wishing videos progressively fills the catalog's years, instant on
  repeat. Dateless tail groups as 'Earlier videos'. Missing-only toggle hidden
  for channels. 84 db + 84 youtube/api tests green.

Full day-one coverage still needs the YouTube Data API's publishedAt (optional,
yt-dlp can't match it cheaply) — RSS + cache is the no-key best.
This commit is contained in:
BoulderBadgeDad 2026-06-17 10:30:54 -07:00
parent ece4fbc792
commit ec5af17de7
9 changed files with 144 additions and 4 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

BIN
Issue/1.png
View file

Binary file not shown.
Side by side Swipe Overlay

Before

Width:  |  Height:  |  Size: 440 KiB

After

Width:  |  Height:  |  Size: 563 KiB

30
api/video/youtube.py
View file

@ -153,9 +153,27 @@ def register_routes(bp):
db.set_wishlist_channel_poster(cid, channel["avatar_url"])
except Exception:
pass
wished = db.youtube_video_wish_state([v.get("youtube_id") for v in channel.get("videos") or []])
for v in channel.get("videos") or []:
vids = channel.get("videos") or []
ids = [v.get("youtube_id") for v in vids]
wished = db.youtube_video_wish_state(ids)
# Upload dates → real year-seasons. Merge the persistent cache with a
# fresh RSS fetch (recent ~15); cache anything new so it fills in over time.
dates = db.get_video_dates(ids)
try:
rss = yt.channel_recent_dates(cid)
except Exception:
rss = {}
if rss:
dates.update(rss)
for v in vids:
v["wished"] = v.get("youtube_id") in wished
if not v.get("published_at") and dates.get(v.get("youtube_id")):
v["published_at"] = dates[v["youtube_id"]]
try:
db.cache_video_dates([{"youtube_id": v["youtube_id"], "published_at": v.get("published_at")}
for v in vids if v.get("published_at")])
except Exception:
pass
return jsonify({"success": True, "kind": "channel", "source": "youtube",
"channel": channel, "following": following})
except Exception:
@ -173,11 +191,17 @@ def register_routes(bp):
v = yt.video_detail(video_id)
if not v or not v.get("youtube_id"):
return jsonify({"success": False, "error": "Video not found"}), 404
db = get_video_db()
if v.get("description"):
try:
get_video_db().set_wishlist_video_overview(v["youtube_id"], v["description"])
db.set_wishlist_video_overview(v["youtube_id"], v["description"])
except Exception:
pass # persistence is best-effort; the detail still returns
if v.get("published_at"): # learned a real date → cache it for year-seasons
try:
db.cache_video_dates([{"youtube_id": v["youtube_id"], "published_at": v["published_at"]}])
except Exception:
pass
return jsonify({"success": True, "video": v})
except Exception:
logger.exception("youtube video detail failed for %r", video_id)

35
core/video/youtube.py
View file

@ -250,6 +250,41 @@ def resolve_channel(raw, limit=30, ydl_factory=None, db=None):
return shaped if shaped.get("youtube_id") else None
def parse_rss_dates(xml_text):
"""{video_id: 'YYYY-MM-DD'} from a YouTube channel RSS feed (pure, testable)."""
import xml.etree.ElementTree as ET
out = {}
try:
root = ET.fromstring(xml_text)
except Exception:
return out
ns = {"a": "http://www.w3.org/2005/Atom", "yt": "http://www.youtube.com/xml/schemas/2015"}
for entry in root.findall("a:entry", ns):
vid = entry.find("yt:videoId", ns)
pub = entry.find("a:published", ns)
if vid is not None and vid.text and pub is not None and pub.text:
out[vid.text.strip()] = pub.text.strip()[:10] # ISO datetime → YYYY-MM-DD
return out
def channel_recent_dates(channel_id, fetch=None):
"""Real upload dates for a channel's ~15 most-recent videos via its public RSS
feed — one cheap GET, no yt-dlp, no bot risk. {video_id: 'YYYY-MM-DD'}."""
cid = str(channel_id or "").strip()
if not cid:
return {}
url = "https://www.youtube.com/feeds/videos.xml?channel_id=" + cid
try:
if fetch is not None:
return parse_rss_dates(fetch(url) or "")
import requests
r = requests.get(url, timeout=10, headers={"User-Agent": _UA})
return parse_rss_dates(r.text) if r.status_code == 200 else {}
except Exception as e:
logger.info("YouTube RSS dates failed for %s: %s", cid, e)
return {}
def search_channels(query, limit=6, ydl_factory=None):
"""Search YouTube for CHANNELS (the results page filtered to channels) → a few
{youtube_id, title, handle, avatar_url, subscriber_count} for the search page.

36
database/video_database.py
View file

@ -1991,6 +1991,42 @@ class VideoDatabase:
finally:
conn.close()
def cache_video_dates(self, pairs) -> int:
"""Persist learned YouTube upload dates ([{youtube_id, published_at}, …]).
Idempotent; only stores non-empty dates. Returns rows written."""
rows = [(p.get("youtube_id"), p.get("published_at")) for p in (pairs or [])
if p.get("youtube_id") and p.get("published_at")]
if not rows:
return 0
conn = self._get_connection()
try:
conn.executemany(
"INSERT INTO youtube_video_dates (youtube_id, published_at) VALUES (?, ?) "
"ON CONFLICT(youtube_id) DO UPDATE SET published_at=excluded.published_at", rows)
conn.commit()
return len(rows)
finally:
conn.close()
def get_video_dates(self, video_ids) -> dict:
"""{youtube_id: published_at} for cached ids — hydrates channel year-seasons."""
out: dict = {}
ids = [str(x) for x in (video_ids or []) if x]
if not ids:
return out
conn = self._get_connection()
try:
for i in range(0, len(ids), 400):
chunk = ids[i:i + 400]
ph = ",".join("?" * len(chunk))
for r in conn.execute(
f"SELECT youtube_id, published_at FROM youtube_video_dates WHERE youtube_id IN ({ph})", chunk):
if r["published_at"]:
out[r["youtube_id"]] = r["published_at"]
return out
finally:
conn.close()
def youtube_wishlist_counts(self) -> dict:
"""{'channel': n distinct channels, 'video': n videos} in the wishlist."""
conn = self._get_connection()

10
database/video_schema.sql
View file

@ -261,6 +261,16 @@ CREATE INDEX IF NOT EXISTS idx_channel_videos_channel ON channel_videos(channe
CREATE INDEX IF NOT EXISTS idx_channel_videos_published ON channel_videos(published_at);
CREATE INDEX IF NOT EXISTS idx_channel_videos_wanted ON channel_videos(monitored, has_file);
-- Cheap persistent cache of YouTube video upload dates (the flat listing omits
-- them). Filled from the channel RSS feed + any per-video metadata fetch, so the
-- channel page's year-seasons fill in over time without re-fetching. Standalone
-- (no channels FK) since the bridge stores channels in video_watchlist.
CREATE TABLE IF NOT EXISTS youtube_video_dates (
youtube_id TEXT PRIMARY KEY,
published_at TEXT,
cached_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP
);
-- ── Owned media files (the Library = content that has a file) ────────────────
-- Exactly one owner FK is set (no polymorphic id). 1 row per physical file;
-- usually 1:1 with its content, but the table allows history/extras.

3
tests/test_video_api.py
View file

@ -617,6 +617,7 @@ def test_youtube_channel_detail_hydrates_follow_and_wished(tmp_path, monkeypatch
client, videoapi = _make_client(tmp_path)
import core.video.youtube as ytmod
monkeypatch.setattr(ytmod, "resolve_channel", lambda url, limit=60: dict(_CHANNEL))
monkeypatch.setattr(ytmod, "channel_recent_dates", lambda cid: {}) # no network in tests
# follow first so detail reports following + one wished video
client.post("/api/video/youtube/follow", json={"channel": _CHANNEL})
videoapi._video_db.remove_one_video_from_wishlist("v2") # leave only v1 wished
@ -624,6 +625,8 @@ def test_youtube_channel_detail_hydrates_follow_and_wished(tmp_path, monkeypatch
assert d["success"] is True and d["kind"] == "channel" and d["following"] is True
wished = {v["youtube_id"]: v["wished"] for v in d["channel"]["videos"]}
assert wished == {"v1": True, "v2": False}
# the videos' dates got cached for year-seasons
assert videoapi._video_db.get_video_dates(["v1"]) == {"v1": "2024-06-01"}
def test_youtube_channel_detail_404_on_unresolvable(tmp_path, monkeypatch):

10
tests/test_video_database.py
View file

@ -1157,3 +1157,13 @@ def test_set_wishlist_channel_poster_backfills_avatar(db):
assert db.set_wishlist_channel_poster("UCx", "http://yt/avatar.jpg") == 2
grp = db.query_youtube_wishlist()["items"][0]
assert grp["poster_url"] == "http://yt/avatar.jpg"
def test_video_date_cache_roundtrip(db):
assert db.cache_video_dates([{"youtube_id": "a", "published_at": "2024-06-01"},
{"youtube_id": "b", "published_at": "2023-01-01"},
{"youtube_id": "c", "published_at": ""}]) == 2 # blank skipped
assert db.get_video_dates(["a", "b", "c", "d"]) == {"a": "2024-06-01", "b": "2023-01-01"}
# upsert refreshes
db.cache_video_dates([{"youtube_id": "a", "published_at": "2025-12-31"}])
assert db.get_video_dates(["a"]) == {"a": "2025-12-31"}

22
tests/test_video_youtube.py
View file

@ -316,3 +316,25 @@ def test_search_channels_extracts_channel_results():
def test_search_channels_empty_query():
assert yt.search_channels(" ", ydl_factory=_SearchYDL) == []
# ── RSS upload-date enrichment ───────────────────────────────────────────────
_RSS = """<?xml version="1.0"?>
<feed xmlns:yt="http://www.youtube.com/xml/schemas/2015" xmlns="http://www.w3.org/2005/Atom">
<entry><yt:videoId>v1</yt:videoId><published>2024-06-01T10:00:00+00:00</published></entry>
<entry><yt:videoId>v2</yt:videoId><published>2023-02-15T08:30:00+00:00</published></entry>
<entry><title>no id or date</title></entry>
</feed>"""
def test_parse_rss_dates():
out = yt.parse_rss_dates(_RSS)
assert out == {"v1": "2024-06-01", "v2": "2023-02-15"}
assert yt.parse_rss_dates("not xml") == {}
def test_channel_recent_dates_via_injected_fetch():
out = yt.channel_recent_dates("UCx", fetch=lambda url: _RSS)
assert out["v1"] == "2024-06-01"
assert yt.channel_recent_dates("", fetch=lambda url: _RSS) == {}

2
webui/static/video/video-detail.js
View file

@ -1326,7 +1326,7 @@
var wishedN = eps.filter(function (e) { return e.owned; }).length;
// No upload dates (flat listing omits them) → a single "All Videos"
// season instead of a lone "Undated"; mixed → label the dateless bucket.
var label = yr ? String(yr) : (years.length === 1 ? 'All Videos' : 'Unknown date');
var label = yr ? String(yr) : (years.length === 1 ? 'All Videos' : 'Earlier videos');
return { season_number: yr, title: label, poster_url: poster || ytProx(ch.avatar_url),
episode_owned: wishedN, episode_total: eps.length, episodes: eps };
});