diff --git a/Issue/1.png b/Issue/1.png index 7e3a4d3c..84e0347f 100644 Binary files a/Issue/1.png and b/Issue/1.png differ diff --git a/api/video/youtube.py b/api/video/youtube.py index 6c805dbe..dae53535 100644 --- a/api/video/youtube.py +++ b/api/video/youtube.py @@ -153,9 +153,27 @@ def register_routes(bp): db.set_wishlist_channel_poster(cid, channel["avatar_url"]) except Exception: pass - wished = db.youtube_video_wish_state([v.get("youtube_id") for v in channel.get("videos") or []]) - for v in channel.get("videos") or []: + vids = channel.get("videos") or [] + ids = [v.get("youtube_id") for v in vids] + wished = db.youtube_video_wish_state(ids) + # Upload dates → real year-seasons. Merge the persistent cache with a + # fresh RSS fetch (recent ~15); cache anything new so it fills in over time. + dates = db.get_video_dates(ids) + try: + rss = yt.channel_recent_dates(cid) + except Exception: + rss = {} + if rss: + dates.update(rss) + for v in vids: v["wished"] = v.get("youtube_id") in wished + if not v.get("published_at") and dates.get(v.get("youtube_id")): + v["published_at"] = dates[v["youtube_id"]] + try: + db.cache_video_dates([{"youtube_id": v["youtube_id"], "published_at": v.get("published_at")} + for v in vids if v.get("published_at")]) + except Exception: + pass return jsonify({"success": True, "kind": "channel", "source": "youtube", "channel": channel, "following": following}) except Exception: @@ -173,11 +191,17 @@ def register_routes(bp): v = yt.video_detail(video_id) if not v or not v.get("youtube_id"): return jsonify({"success": False, "error": "Video not found"}), 404 + db = get_video_db() if v.get("description"): try: - get_video_db().set_wishlist_video_overview(v["youtube_id"], v["description"]) + db.set_wishlist_video_overview(v["youtube_id"], v["description"]) except Exception: pass # persistence is best-effort; the detail still returns + if v.get("published_at"): # learned a real date → cache it for year-seasons + try: + db.cache_video_dates([{"youtube_id": v["youtube_id"], "published_at": v["published_at"]}]) + except Exception: + pass return jsonify({"success": True, "video": v}) except Exception: logger.exception("youtube video detail failed for %r", video_id) diff --git a/core/video/youtube.py b/core/video/youtube.py index 6c0e3acf..d625fc63 100644 --- a/core/video/youtube.py +++ b/core/video/youtube.py @@ -250,6 +250,41 @@ def resolve_channel(raw, limit=30, ydl_factory=None, db=None): return shaped if shaped.get("youtube_id") else None +def parse_rss_dates(xml_text): + """{video_id: 'YYYY-MM-DD'} from a YouTube channel RSS feed (pure, testable).""" + import xml.etree.ElementTree as ET + out = {} + try: + root = ET.fromstring(xml_text) + except Exception: + return out + ns = {"a": "http://www.w3.org/2005/Atom", "yt": "http://www.youtube.com/xml/schemas/2015"} + for entry in root.findall("a:entry", ns): + vid = entry.find("yt:videoId", ns) + pub = entry.find("a:published", ns) + if vid is not None and vid.text and pub is not None and pub.text: + out[vid.text.strip()] = pub.text.strip()[:10] # ISO datetime → YYYY-MM-DD + return out + + +def channel_recent_dates(channel_id, fetch=None): + """Real upload dates for a channel's ~15 most-recent videos via its public RSS + feed — one cheap GET, no yt-dlp, no bot risk. {video_id: 'YYYY-MM-DD'}.""" + cid = str(channel_id or "").strip() + if not cid: + return {} + url = "https://www.youtube.com/feeds/videos.xml?channel_id=" + cid + try: + if fetch is not None: + return parse_rss_dates(fetch(url) or "") + import requests + r = requests.get(url, timeout=10, headers={"User-Agent": _UA}) + return parse_rss_dates(r.text) if r.status_code == 200 else {} + except Exception as e: + logger.info("YouTube RSS dates failed for %s: %s", cid, e) + return {} + + def search_channels(query, limit=6, ydl_factory=None): """Search YouTube for CHANNELS (the results page filtered to channels) → a few {youtube_id, title, handle, avatar_url, subscriber_count} for the search page. diff --git a/database/video_database.py b/database/video_database.py index 2224cb53..94642129 100644 --- a/database/video_database.py +++ b/database/video_database.py @@ -1991,6 +1991,42 @@ class VideoDatabase: finally: conn.close() + def cache_video_dates(self, pairs) -> int: + """Persist learned YouTube upload dates ([{youtube_id, published_at}, …]). + Idempotent; only stores non-empty dates. Returns rows written.""" + rows = [(p.get("youtube_id"), p.get("published_at")) for p in (pairs or []) + if p.get("youtube_id") and p.get("published_at")] + if not rows: + return 0 + conn = self._get_connection() + try: + conn.executemany( + "INSERT INTO youtube_video_dates (youtube_id, published_at) VALUES (?, ?) " + "ON CONFLICT(youtube_id) DO UPDATE SET published_at=excluded.published_at", rows) + conn.commit() + return len(rows) + finally: + conn.close() + + def get_video_dates(self, video_ids) -> dict: + """{youtube_id: published_at} for cached ids — hydrates channel year-seasons.""" + out: dict = {} + ids = [str(x) for x in (video_ids or []) if x] + if not ids: + return out + conn = self._get_connection() + try: + for i in range(0, len(ids), 400): + chunk = ids[i:i + 400] + ph = ",".join("?" * len(chunk)) + for r in conn.execute( + f"SELECT youtube_id, published_at FROM youtube_video_dates WHERE youtube_id IN ({ph})", chunk): + if r["published_at"]: + out[r["youtube_id"]] = r["published_at"] + return out + finally: + conn.close() + def youtube_wishlist_counts(self) -> dict: """{'channel': n distinct channels, 'video': n videos} in the wishlist.""" conn = self._get_connection() diff --git a/database/video_schema.sql b/database/video_schema.sql index e5528bd7..f881c320 100644 --- a/database/video_schema.sql +++ b/database/video_schema.sql @@ -261,6 +261,16 @@ CREATE INDEX IF NOT EXISTS idx_channel_videos_channel ON channel_videos(channe CREATE INDEX IF NOT EXISTS idx_channel_videos_published ON channel_videos(published_at); CREATE INDEX IF NOT EXISTS idx_channel_videos_wanted ON channel_videos(monitored, has_file); +-- Cheap persistent cache of YouTube video upload dates (the flat listing omits +-- them). Filled from the channel RSS feed + any per-video metadata fetch, so the +-- channel page's year-seasons fill in over time without re-fetching. Standalone +-- (no channels FK) since the bridge stores channels in video_watchlist. +CREATE TABLE IF NOT EXISTS youtube_video_dates ( + youtube_id TEXT PRIMARY KEY, + published_at TEXT, + cached_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP +); + -- ── Owned media files (the Library = content that has a file) ──────────────── -- Exactly one owner FK is set (no polymorphic id). 1 row per physical file; -- usually 1:1 with its content, but the table allows history/extras. diff --git a/tests/test_video_api.py b/tests/test_video_api.py index d3e16c7a..6ed45dff 100644 --- a/tests/test_video_api.py +++ b/tests/test_video_api.py @@ -617,6 +617,7 @@ def test_youtube_channel_detail_hydrates_follow_and_wished(tmp_path, monkeypatch client, videoapi = _make_client(tmp_path) import core.video.youtube as ytmod monkeypatch.setattr(ytmod, "resolve_channel", lambda url, limit=60: dict(_CHANNEL)) + monkeypatch.setattr(ytmod, "channel_recent_dates", lambda cid: {}) # no network in tests # follow first so detail reports following + one wished video client.post("/api/video/youtube/follow", json={"channel": _CHANNEL}) videoapi._video_db.remove_one_video_from_wishlist("v2") # leave only v1 wished @@ -624,6 +625,8 @@ def test_youtube_channel_detail_hydrates_follow_and_wished(tmp_path, monkeypatch assert d["success"] is True and d["kind"] == "channel" and d["following"] is True wished = {v["youtube_id"]: v["wished"] for v in d["channel"]["videos"]} assert wished == {"v1": True, "v2": False} + # the videos' dates got cached for year-seasons + assert videoapi._video_db.get_video_dates(["v1"]) == {"v1": "2024-06-01"} def test_youtube_channel_detail_404_on_unresolvable(tmp_path, monkeypatch): diff --git a/tests/test_video_database.py b/tests/test_video_database.py index 82133d15..f620e950 100644 --- a/tests/test_video_database.py +++ b/tests/test_video_database.py @@ -1157,3 +1157,13 @@ def test_set_wishlist_channel_poster_backfills_avatar(db): assert db.set_wishlist_channel_poster("UCx", "http://yt/avatar.jpg") == 2 grp = db.query_youtube_wishlist()["items"][0] assert grp["poster_url"] == "http://yt/avatar.jpg" + + +def test_video_date_cache_roundtrip(db): + assert db.cache_video_dates([{"youtube_id": "a", "published_at": "2024-06-01"}, + {"youtube_id": "b", "published_at": "2023-01-01"}, + {"youtube_id": "c", "published_at": ""}]) == 2 # blank skipped + assert db.get_video_dates(["a", "b", "c", "d"]) == {"a": "2024-06-01", "b": "2023-01-01"} + # upsert refreshes + db.cache_video_dates([{"youtube_id": "a", "published_at": "2025-12-31"}]) + assert db.get_video_dates(["a"]) == {"a": "2025-12-31"} diff --git a/tests/test_video_youtube.py b/tests/test_video_youtube.py index 41421135..e5495160 100644 --- a/tests/test_video_youtube.py +++ b/tests/test_video_youtube.py @@ -316,3 +316,25 @@ def test_search_channels_extracts_channel_results(): def test_search_channels_empty_query(): assert yt.search_channels(" ", ydl_factory=_SearchYDL) == [] + + +# ── RSS upload-date enrichment ─────────────────────────────────────────────── + +_RSS = """ + + v12024-06-01T10:00:00+00:00 + v22023-02-15T08:30:00+00:00 + no id or date +""" + + +def test_parse_rss_dates(): + out = yt.parse_rss_dates(_RSS) + assert out == {"v1": "2024-06-01", "v2": "2023-02-15"} + assert yt.parse_rss_dates("not xml") == {} + + +def test_channel_recent_dates_via_injected_fetch(): + out = yt.channel_recent_dates("UCx", fetch=lambda url: _RSS) + assert out["v1"] == "2024-06-01" + assert yt.channel_recent_dates("", fetch=lambda url: _RSS) == {} diff --git a/webui/static/video/video-detail.js b/webui/static/video/video-detail.js index 7e0cfc28..0f1afae1 100644 --- a/webui/static/video/video-detail.js +++ b/webui/static/video/video-detail.js @@ -1326,7 +1326,7 @@ var wishedN = eps.filter(function (e) { return e.owned; }).length; // No upload dates (flat listing omits them) → a single "All Videos" // season instead of a lone "Undated"; mixed → label the dateless bucket. - var label = yr ? String(yr) : (years.length === 1 ? 'All Videos' : 'Unknown date'); + var label = yr ? String(yr) : (years.length === 1 ? 'All Videos' : 'Earlier videos'); return { season_number: yr, title: label, poster_url: poster || ytProx(ch.avatar_url), episode_owned: wishedN, episode_total: eps.length, episodes: eps }; });