From fb9459fb5c608cdd95e1682bcb7f8377f9c17bc5 Mon Sep 17 00:00:00 2001 From: BoulderBadgeDad Date: Thu, 25 Jun 2026 21:18:53 -0700 Subject: [PATCH] video automations: scan watchlist YouTube playlists (mirror-all) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit playlists are followable but had no scan. new 'Scan Watchlist Playlists' automation, sibling of the channel scan but a different rule: a playlist is a curated finite set, so MIRROR it — wishlist every long-form video you don't have plus any later additions (no forward-looking baseline, no last-N net). playlist-as-show: videos wishlisted under the playlist's title, so the worker files them as 'Playlist Name / Season YEAR / ... - date - title' (the ytdl-sub tv_show_name-on-a-playlist convention). reuses all the channel plumbing — same wishlist rows, same Download YouTube Wishlist drain, quality + org template. seeded every 6h (Auto-Scan Watchlist Playlists); block + registration + icon + drift test. dedup reuses wishlisted_video_ids_for_channel (parent_source_id=PL). seam-tested; 944-test sweep green. --- core/automation/blocks.py | 2 + core/automation/handlers/registration.py | 7 + .../video_scan_watchlist_playlists.py | 176 ++++++++++++++++++ core/automation_engine.py | 9 + tests/automation/test_handler_registration.py | 1 + tests/test_video_scan_watchlist_playlists.py | 120 ++++++++++++ webui/static/stats-automations.js | 3 +- 7 files changed, 317 insertions(+), 1 deletion(-) create mode 100644 core/automation/handlers/video_scan_watchlist_playlists.py create mode 100644 tests/test_video_scan_watchlist_playlists.py diff --git a/core/automation/blocks.py b/core/automation/blocks.py index 60de924b..728ba579 100644 --- a/core/automation/blocks.py +++ b/core/automation/blocks.py @@ -272,6 +272,8 @@ ACTIONS: list[dict] = [ "config_fields": [ {"key": "backfill_count", "type": "number", "label": "Always keep the last N videos from each channel", "default": 10, "min": 0} ]}, + {"type": "video_scan_watchlist_playlists", "label": "Scan Watchlist Playlists", "icon": "list", "scope": "video", + "description": "For every YouTube playlist you follow, wishlist its videos you don't have yet (and anything later added to it) — mirrors the whole list. Each playlist becomes its own show in the library (playlist-as-show). Shorts excluded. Pair with a schedule.", "available": True}, {"type": "video_process_youtube_wishlist", "label": "Download YouTube Wishlist", "icon": "download", "scope": "video", "description": "Download wished YouTube videos (yt-dlp), organised as a Plex 'TV by date' show (channel/year/date). Queues the WHOLE wishlist — the setting only limits how many download at the same time; each finished one starts the next, so it all drains. A completed download leaves the wishlist. Needs the YouTube library folder set on Settings → Downloads.", "available": True, "config_fields": [ diff --git a/core/automation/handlers/registration.py b/core/automation/handlers/registration.py index a48bbdbe..86214a74 100644 --- a/core/automation/handlers/registration.py +++ b/core/automation/handlers/registration.py @@ -40,6 +40,7 @@ from core.automation.handlers.video_auto_wishlist_airing import auto_video_add_a from core.automation.handlers.video_scan_watchlist_people import auto_video_scan_watchlist_people from core.automation.handlers.video_scan_watchlist_channels import auto_video_scan_watchlist_channels from core.automation.handlers.video_process_youtube_wishlist import auto_video_process_youtube_wishlist +from core.automation.handlers.video_scan_watchlist_playlists import auto_video_scan_watchlist_playlists from core.automation.handlers.video_scan_library import ( auto_video_scan_library, auto_video_scan_server, auto_video_update_database, ) @@ -242,6 +243,12 @@ def register_all(deps: AutomationDeps) -> None: 'video_scan_watchlist_channels', lambda config: auto_video_scan_watchlist_channels(config, deps), ) + # Watchlist-playlists scan: mirror followed YouTube playlists into the wishlist + # (whole list + new additions; playlist-as-show). + engine.register_action_handler( + 'video_scan_watchlist_playlists', + lambda config: auto_video_scan_watchlist_playlists(config, deps), + ) # Drain side: enqueue wished YouTube videos into the download queue (yt-dlp worker). engine.register_action_handler( 'video_process_youtube_wishlist', diff --git a/core/automation/handlers/video_scan_watchlist_playlists.py b/core/automation/handlers/video_scan_watchlist_playlists.py new file mode 100644 index 00000000..d06627e7 --- /dev/null +++ b/core/automation/handlers/video_scan_watchlist_playlists.py @@ -0,0 +1,176 @@ +"""Automation handler: ``video_scan_watchlist_playlists`` action. + +Sibling of the watchlist-CHANNELS scan, with a deliberately different rule. A channel is a +creator posting new stuff over time, so that scan is forward-looking (new uploads + a +last-N net). A **playlist** is a curated, finite set someone assembled — the reason you +follow it is "give me this whole list and keep it complete." So this scan MIRRORS the +playlist: it wishlists every long-form video in it you don't already have, plus anything +later added. No follow-date baseline, no last-N net. + +Organisation: the playlist becomes its own "show" (playlist-as-show) — its videos are +wishlisted under the playlist's title, so the download worker files them as +``Playlist Name / Season YEAR / Playlist Name - DATE - Title`` (matches the ytdl-sub +``tv_show_name``-on-a-playlist convention). All other plumbing is shared with channels: the +same wishlist rows, the same "Download YouTube Wishlist" drain, quality + org template. + +Edge: a video in both a followed channel AND a followed playlist is wishlisted once (dedup +by video id); whichever scan touched it last sets its show-name. Accepted. + +Pure selection with all I/O injected; shared automation side; owns its own progress. +""" + +from __future__ import annotations + +from datetime import date +from typing import Any, Callable, Dict, Iterable, List, Optional + +from core.automation.deps import AutomationDeps +from core.automation.handlers.video_scan_watchlist_channels import _day, long_form_uploads + +# how many playlist entries to read per scan (yt-dlp flat). Big enough for almost any real +# playlist; genuinely huge ones truncate (logged) rather than hammering YouTube each scan. +_PLAYLIST_FETCH_LIMIT = 1000 + + +def select_playlist_video_gaps( + videos: List[Dict[str, Any]], + *, + wishlisted_ids: Iterable = (), + downloaded_ids: Iterable = (), + dismissed_ids: Iterable = (), + today: str, + min_seconds: int = 60, +) -> List[Dict[str, Any]]: + """The pure core: every long-form video in a playlist not already wishlisted / + downloaded / dismissed (mirror the whole list). Future-dated (unaired) entries are + skipped. No baseline, no last-N — a curated playlist is wanted in full. No I/O.""" + excluded = set(wishlisted_ids or ()) | set(downloaded_ids or ()) | set(dismissed_ids or ()) + out: List[Dict[str, Any]] = [] + for v in long_form_uploads(videos, min_seconds): + vid = v["youtube_id"] + if vid in excluded: + continue + d = _day(v.get("published_at")) + if d and today and d > today: + continue # unaired premiere — not out yet + out.append(v) + return out + + +# ── production seams ────────────────────────────────────────────────────────── +def _default_fetch_playlists() -> List[Dict[str, Any]]: + from api.video import get_video_db + return get_video_db().list_watchlist_playlists() + + +def _default_fetch_videos(playlist_id: Any) -> List[Dict[str, Any]]: + """A playlist's videos (flat, with durations) with upload dates merged from the cache. + Playlists have no per-channel RSS, so dates come from whatever's cached (filled over + time by the channel/InnerTube enrichers); undated entries organise cleanly.""" + from api.video import get_video_db + from core.video import youtube as yt + db = get_video_db() + vids = yt.playlist_videos(str(playlist_id), limit=_PLAYLIST_FETCH_LIMIT) or [] + ids = [v.get("youtube_id") for v in vids if v.get("youtube_id")] + dates = db.get_video_dates(ids) or {} + for v in vids: + if not v.get("published_at") and dates.get(v.get("youtube_id")): + v["published_at"] = dates[v["youtube_id"]] + return vids + + +def _default_wishlisted_ids(playlist_id: Any) -> List[Any]: + # parent_source_id holds the playlist id for playlist-sourced wishlist videos. + from api.video import get_video_db + return get_video_db().wishlisted_video_ids_for_channel(playlist_id) + + +def _default_downloaded_ids(playlist_id: Any) -> List[Any]: + return [] # populated once the download-history ledger carries youtube grabs + + +def _default_dismissed_ids(playlist_id: Any) -> List[Any]: + return [] # no youtube "dismissed" store yet (see channels-scan note) + + +def _default_add_videos(playlist: Dict[str, Any], videos: List[Dict[str, Any]]) -> int: + """Wishlist under the PLAYLIST as the show (title = playlist name → playlist-as-show).""" + from api.video import get_video_db + from core.video.sources import resolve_video_server + return get_video_db().add_videos_to_wishlist(playlist, videos, server_source=resolve_video_server()) + + +def auto_video_scan_watchlist_playlists( + config: Dict[str, Any], + deps: AutomationDeps, + *, + fetch_playlists: Optional[Callable[[], List[Dict[str, Any]]]] = None, + fetch_videos: Optional[Callable[[Any], List[Dict[str, Any]]]] = None, + wishlisted_ids: Optional[Callable[[Any], Iterable]] = None, + downloaded_ids: Optional[Callable[[Any], Iterable]] = None, + dismissed_ids: Optional[Callable[[Any], Iterable]] = None, + add_videos: Optional[Callable[[Dict[str, Any], List[Dict[str, Any]]], int]] = None, + today_fn: Optional[Callable[[], str]] = None, +) -> Dict[str, Any]: + """Mirror every followed YouTube playlist into the wishlist (whole list + new additions). + + Returns ``{'status': 'completed', 'playlists': int, 'videos_added': int, ...}``.""" + fetch_playlists = fetch_playlists or _default_fetch_playlists + fetch_videos = fetch_videos or _default_fetch_videos + wishlisted_ids = wishlisted_ids or _default_wishlisted_ids + downloaded_ids = downloaded_ids or _default_downloaded_ids + dismissed_ids = dismissed_ids or _default_dismissed_ids + add_videos = add_videos or _default_add_videos + today_fn = today_fn or (lambda: date.today().isoformat()) + automation_id = config.get('_automation_id') + min_seconds = max(0, int(config.get('min_seconds', 60) or 0)) + + try: + today = today_fn() + deps.update_progress(automation_id, phase='Reading your watchlist…', progress=5, + log_line='Loading the playlists you follow', log_type='info') + playlists = fetch_playlists() or [] + if not playlists: + deps.update_progress(automation_id, status='finished', progress=100, phase='Complete', + log_line='No playlists on the watchlist to scan', log_type='info') + return {'status': 'completed', 'playlists': 0, 'videos_added': 0, + '_manages_own_progress': True} + + added = 0 + total = len(playlists) + for i, pl in enumerate(playlists): + pid = pl.get('playlist_id') + ptitle = pl.get('title') or pid + deps.update_progress(automation_id, phase='Scanning playlists…', + progress=10 + int(80 * i / max(total, 1)), + log_line="Mirroring '%s'" % ptitle, log_type='info') + if not pid: + continue + try: + videos = fetch_videos(pid) or [] + except Exception: # noqa: BLE001 - one flaky playlist shouldn't abort the scan + deps.update_progress(automation_id, log_line="Couldn't reach '%s' — skipping" % ptitle, + log_type='warning') + continue + + gaps = select_playlist_video_gaps( + videos, wishlisted_ids=wishlisted_ids(pid), downloaded_ids=downloaded_ids(pid), + dismissed_ids=dismissed_ids(pid), today=today, min_seconds=min_seconds) + if gaps: + n = int(add_videos({'youtube_id': pid, 'title': ptitle, + 'avatar_url': pl.get('poster_url')}, gaps) or 0) + added += n + if n: + deps.update_progress( + automation_id, log_type='success', + log_line="Wishlisted %d new video(s) from '%s'" % (n, ptitle)) + + done = ('Wishlisted %d new video(s) across %d playlist(s)' % (added, total)) if added \ + else ('Playlists are up to date — nothing new across %d playlist(s)' % total) + deps.update_progress(automation_id, status='finished', progress=100, phase='Complete', + log_line=done, log_type='success') + return {'status': 'completed', 'playlists': total, 'videos_added': added, + '_manages_own_progress': True} + except Exception as e: # noqa: BLE001 + deps.update_progress(automation_id, status='error', phase='Error', log_line=str(e), log_type='error') + return {'status': 'error', 'error': str(e), '_manages_own_progress': True} diff --git a/core/automation_engine.py b/core/automation_engine.py index f2686bfb..f4f83e06 100644 --- a/core/automation_engine.py +++ b/core/automation_engine.py @@ -259,6 +259,15 @@ SYSTEM_AUTOMATIONS = [ 'initial_delay': 1200, 'owned_by': 'video', }, + # Playlists: every 6h, mirror the whole list (playlist-as-show). No-ops if you follow none. + { + 'name': 'Auto-Scan Watchlist Playlists', + 'trigger_type': 'schedule', + 'trigger_config': {'interval': 6, 'unit': 'hours'}, + 'action_type': 'video_scan_watchlist_playlists', + 'initial_delay': 1320, + 'owned_by': 'video', + }, # Drain side: download wished YouTube videos hourly (queues the whole wishlist, runs a # few at a time). Skips quietly until a YouTube library folder is set. { diff --git a/tests/automation/test_handler_registration.py b/tests/automation/test_handler_registration.py index b926e72c..d82d93e4 100644 --- a/tests/automation/test_handler_registration.py +++ b/tests/automation/test_handler_registration.py @@ -62,6 +62,7 @@ EXPECTED_ACTION_NAMES = frozenset({ 'video_add_airing_episodes', 'video_scan_watchlist_people', 'video_scan_watchlist_channels', + 'video_scan_watchlist_playlists', 'video_process_youtube_wishlist', 'video_clean_search_history', 'video_clean_completed_downloads', diff --git a/tests/test_video_scan_watchlist_playlists.py b/tests/test_video_scan_watchlist_playlists.py new file mode 100644 index 00000000..e4a6019b --- /dev/null +++ b/tests/test_video_scan_watchlist_playlists.py @@ -0,0 +1,120 @@ +"""Watchlist-playlists scan: MIRROR a followed YouTube playlist — wishlist every long-form +video in it you don't have (plus new additions), filed playlist-as-show. Differs from the +channel scan (no forward-looking baseline, no last-N net). Pure selection, I/O injected. +""" + +from __future__ import annotations + +from core.automation.handlers.video_scan_watchlist_playlists import ( + auto_video_scan_watchlist_playlists, + select_playlist_video_gaps, +) + + +class _Deps: + def __init__(self): + self.progress = [] + + def update_progress(self, automation_id, **kw): + self.progress.append(kw) + + +def _vid(vid, *, date=None, dur=600): + return {"youtube_id": vid, "title": "Video " + vid, "published_at": date, + "duration_seconds": dur, "thumbnail_url": "/t/" + vid + ".jpg"} + + +# ── pure: mirror the whole list (no baseline / no net) ──────────────────────── +def test_mirrors_every_unowned_long_form_video(): + # old AND new alike — a curated playlist is wanted in full + vids = [_vid("a", date="2024-06-01"), _vid("b", date="2018-01-01"), + _vid("c", date="2010-05-05")] + gaps = select_playlist_video_gaps(vids, today="2026-06-25") + assert [g["youtube_id"] for g in gaps] == ["a", "b", "c"] # all of them, even ancient + + +def test_excludes_already_wishlisted_downloaded_dismissed(): + vids = [_vid("a"), _vid("b"), _vid("c")] + gaps = select_playlist_video_gaps( + vids, wishlisted_ids=["a"], downloaded_ids=["b"], dismissed_ids=["c"], today="2026-06-25") + assert gaps == [] + + +def test_excludes_shorts_and_future_premieres(): + vids = [_vid("short", dur=20), _vid("real", dur=600, date="2024-01-01"), + _vid("premiere", dur=600, date="2099-01-01")] + gaps = select_playlist_video_gaps(vids, today="2026-06-25") + assert [g["youtube_id"] for g in gaps] == ["real"] + + +# ── handler ─────────────────────────────────────────────────────────────────── +def _run(playlists, videos_by_pl, *, wished=None, today="2026-06-25"): + adds = [] + + def add_videos(playlist, videos): + adds.append((playlist["youtube_id"], playlist["title"], [v["youtube_id"] for v in videos])) + return len(videos) + + deps = _Deps() + res = auto_video_scan_watchlist_playlists( + {"_automation_id": "a"}, deps, + fetch_playlists=lambda: playlists, + fetch_videos=lambda pid: videos_by_pl.get(pid, []), + wishlisted_ids=lambda pid: (wished or {}).get(pid, []), + downloaded_ids=lambda pid: [], dismissed_ids=lambda pid: [], + add_videos=add_videos, today_fn=lambda: today) + return res, adds, deps + + +def test_wishlists_under_the_playlist_as_the_show(): + playlists = [{"playlist_id": "PL1", "title": "Best Sci-Fi", "poster_url": "/p.jpg"}] + vids = [_vid("a", date="2024-01-01"), _vid("b", date="2023-01-01")] + res, adds, _ = _run(playlists, {"PL1": vids}) + assert res["status"] == "completed" and res["playlists"] == 1 and res["videos_added"] == 2 + # the PLAYLIST id + title travel to add_videos → playlist-as-show organisation + assert adds == [("PL1", "Best Sci-Fi", ["a", "b"])] + + +def test_rerun_only_adds_new_additions(): + playlists = [{"playlist_id": "PL1", "title": "Mix"}] + vids = [_vid("new", date="2026-06-20"), _vid("old1"), _vid("old2")] + res, adds, _ = _run(playlists, {"PL1": vids}, wished={"PL1": ["old1", "old2"]}) + assert adds == [("PL1", "Mix", ["new"])] and res["videos_added"] == 1 + + +def test_multiple_playlists_independent(): + playlists = [{"playlist_id": "PL1", "title": "A"}, {"playlist_id": "PL2", "title": "B"}] + videos = {"PL1": [_vid("a1")], "PL2": [_vid("b1")]} + res, adds, _ = _run(playlists, videos) + assert res["playlists"] == 2 and res["videos_added"] == 2 + + +def test_one_unreachable_playlist_does_not_abort(): + playlists = [{"playlist_id": "PL1", "title": "Breaks"}, {"playlist_id": "PL2", "title": "Works"}] + + def fetch_videos(pid): + if pid == "PL1": + raise RuntimeError("yt-dlp blew up") + return [_vid("ok")] + + adds = [] + res = auto_video_scan_watchlist_playlists( + {"_automation_id": "a"}, _Deps(), + fetch_playlists=lambda: playlists, fetch_videos=fetch_videos, + wishlisted_ids=lambda pid: [], downloaded_ids=lambda pid: [], + dismissed_ids=lambda pid: [], add_videos=lambda pl, v: len(v), today_fn=lambda: "2026-06-25") + assert res["status"] == "completed" and res["videos_added"] == 1 + + +def test_empty_watchlist_is_a_clean_noop(): + res, adds, _ = _run([], {}) + assert res["status"] == "completed" and res["playlists"] == 0 and adds == [] + + +def test_top_level_error_is_caught(): + def boom(): + raise RuntimeError("watchlist read failed") + deps = _Deps() + res = auto_video_scan_watchlist_playlists({"_automation_id": "a"}, deps, fetch_playlists=boom) + assert res["status"] == "error" and "watchlist read failed" in res["error"] + assert any(p.get("status") == "error" for p in deps.progress) diff --git a/webui/static/stats-automations.js b/webui/static/stats-automations.js index a38d74b7..38e571b4 100644 --- a/webui/static/stats-automations.js +++ b/webui/static/stats-automations.js @@ -1777,7 +1777,7 @@ const _autoIcons = { video_scan_library: '\uD83C\uDFAC', video_scan_server: '\uD83D\uDD04', video_update_database: '\uD83D\uDDC4\uFE0F', video_add_airing_episodes: '\uD83D\uDCFA', video_deep_scan_movies: '\uD83C\uDFAC', video_deep_scan_tv: '\uD83D\uDCFA', video_scan_watchlist_people: '\uD83C\uDFAD', video_scan_watchlist_channels: '\uD83D\uDCE1', - video_process_youtube_wishlist: '\u2B07\uFE0F', + video_scan_watchlist_playlists: '\uD83C\uDFB5', video_process_youtube_wishlist: '\u2B07\uFE0F', }; // --- Inspiration Templates --- @@ -3374,6 +3374,7 @@ function _autoFormatAction(type) { video_deep_scan_movies: 'Deep Scan Movie Library', video_deep_scan_tv: 'Deep Scan TV Library', video_scan_watchlist_people: 'Scan Watchlist People', video_scan_watchlist_channels: 'Scan Watchlist Channels', + video_scan_watchlist_playlists: 'Scan Watchlist Playlists', video_process_youtube_wishlist: 'Download YouTube Wishlist', }; return labels[type] || type || 'Unknown';