video automations: scan watchlist YouTube playlists (mirror-all)
playlists are followable but had no scan. new 'Scan Watchlist Playlists' automation, sibling of the channel scan but a different rule: a playlist is a curated finite set, so MIRROR it — wishlist every long-form video you don't have plus any later additions (no forward-looking baseline, no last-N net). playlist-as-show: videos wishlisted under the playlist's title, so the worker files them as 'Playlist Name / Season YEAR / ... - date - title' (the ytdl-sub tv_show_name-on-a-playlist convention). reuses all the channel plumbing — same wishlist rows, same Download YouTube Wishlist drain, quality + org template. seeded every 6h (Auto-Scan Watchlist Playlists); block + registration + icon + drift test. dedup reuses wishlisted_video_ids_for_channel (parent_source_id=PL). seam-tested; 944-test sweep green.
This commit is contained in:
parent
b83f314617
commit
fb9459fb5c
7 changed files with 317 additions and 1 deletions
|
|
@ -272,6 +272,8 @@ ACTIONS: list[dict] = [
|
|||
"config_fields": [
|
||||
{"key": "backfill_count", "type": "number", "label": "Always keep the last N videos from each channel", "default": 10, "min": 0}
|
||||
]},
|
||||
{"type": "video_scan_watchlist_playlists", "label": "Scan Watchlist Playlists", "icon": "list", "scope": "video",
|
||||
"description": "For every YouTube playlist you follow, wishlist its videos you don't have yet (and anything later added to it) — mirrors the whole list. Each playlist becomes its own show in the library (playlist-as-show). Shorts excluded. Pair with a schedule.", "available": True},
|
||||
{"type": "video_process_youtube_wishlist", "label": "Download YouTube Wishlist", "icon": "download", "scope": "video",
|
||||
"description": "Download wished YouTube videos (yt-dlp), organised as a Plex 'TV by date' show (channel/year/date). Queues the WHOLE wishlist — the setting only limits how many download at the same time; each finished one starts the next, so it all drains. A completed download leaves the wishlist. Needs the YouTube library folder set on Settings → Downloads.", "available": True,
|
||||
"config_fields": [
|
||||
|
|
|
|||
|
|
@ -40,6 +40,7 @@ from core.automation.handlers.video_auto_wishlist_airing import auto_video_add_a
|
|||
from core.automation.handlers.video_scan_watchlist_people import auto_video_scan_watchlist_people
|
||||
from core.automation.handlers.video_scan_watchlist_channels import auto_video_scan_watchlist_channels
|
||||
from core.automation.handlers.video_process_youtube_wishlist import auto_video_process_youtube_wishlist
|
||||
from core.automation.handlers.video_scan_watchlist_playlists import auto_video_scan_watchlist_playlists
|
||||
from core.automation.handlers.video_scan_library import (
|
||||
auto_video_scan_library, auto_video_scan_server, auto_video_update_database,
|
||||
)
|
||||
|
|
@ -242,6 +243,12 @@ def register_all(deps: AutomationDeps) -> None:
|
|||
'video_scan_watchlist_channels',
|
||||
lambda config: auto_video_scan_watchlist_channels(config, deps),
|
||||
)
|
||||
# Watchlist-playlists scan: mirror followed YouTube playlists into the wishlist
|
||||
# (whole list + new additions; playlist-as-show).
|
||||
engine.register_action_handler(
|
||||
'video_scan_watchlist_playlists',
|
||||
lambda config: auto_video_scan_watchlist_playlists(config, deps),
|
||||
)
|
||||
# Drain side: enqueue wished YouTube videos into the download queue (yt-dlp worker).
|
||||
engine.register_action_handler(
|
||||
'video_process_youtube_wishlist',
|
||||
|
|
|
|||
176
core/automation/handlers/video_scan_watchlist_playlists.py
Normal file
176
core/automation/handlers/video_scan_watchlist_playlists.py
Normal file
|
|
@ -0,0 +1,176 @@
|
|||
"""Automation handler: ``video_scan_watchlist_playlists`` action.
|
||||
|
||||
Sibling of the watchlist-CHANNELS scan, with a deliberately different rule. A channel is a
|
||||
creator posting new stuff over time, so that scan is forward-looking (new uploads + a
|
||||
last-N net). A **playlist** is a curated, finite set someone assembled — the reason you
|
||||
follow it is "give me this whole list and keep it complete." So this scan MIRRORS the
|
||||
playlist: it wishlists every long-form video in it you don't already have, plus anything
|
||||
later added. No follow-date baseline, no last-N net.
|
||||
|
||||
Organisation: the playlist becomes its own "show" (playlist-as-show) — its videos are
|
||||
wishlisted under the playlist's title, so the download worker files them as
|
||||
``Playlist Name / Season YEAR / Playlist Name - DATE - Title`` (matches the ytdl-sub
|
||||
``tv_show_name``-on-a-playlist convention). All other plumbing is shared with channels: the
|
||||
same wishlist rows, the same "Download YouTube Wishlist" drain, quality + org template.
|
||||
|
||||
Edge: a video in both a followed channel AND a followed playlist is wishlisted once (dedup
|
||||
by video id); whichever scan touched it last sets its show-name. Accepted.
|
||||
|
||||
Pure selection with all I/O injected; shared automation side; owns its own progress.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import date
|
||||
from typing import Any, Callable, Dict, Iterable, List, Optional
|
||||
|
||||
from core.automation.deps import AutomationDeps
|
||||
from core.automation.handlers.video_scan_watchlist_channels import _day, long_form_uploads
|
||||
|
||||
# how many playlist entries to read per scan (yt-dlp flat). Big enough for almost any real
|
||||
# playlist; genuinely huge ones truncate (logged) rather than hammering YouTube each scan.
|
||||
_PLAYLIST_FETCH_LIMIT = 1000
|
||||
|
||||
|
||||
def select_playlist_video_gaps(
|
||||
videos: List[Dict[str, Any]],
|
||||
*,
|
||||
wishlisted_ids: Iterable = (),
|
||||
downloaded_ids: Iterable = (),
|
||||
dismissed_ids: Iterable = (),
|
||||
today: str,
|
||||
min_seconds: int = 60,
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""The pure core: every long-form video in a playlist not already wishlisted /
|
||||
downloaded / dismissed (mirror the whole list). Future-dated (unaired) entries are
|
||||
skipped. No baseline, no last-N — a curated playlist is wanted in full. No I/O."""
|
||||
excluded = set(wishlisted_ids or ()) | set(downloaded_ids or ()) | set(dismissed_ids or ())
|
||||
out: List[Dict[str, Any]] = []
|
||||
for v in long_form_uploads(videos, min_seconds):
|
||||
vid = v["youtube_id"]
|
||||
if vid in excluded:
|
||||
continue
|
||||
d = _day(v.get("published_at"))
|
||||
if d and today and d > today:
|
||||
continue # unaired premiere — not out yet
|
||||
out.append(v)
|
||||
return out
|
||||
|
||||
|
||||
# ── production seams ──────────────────────────────────────────────────────────
|
||||
def _default_fetch_playlists() -> List[Dict[str, Any]]:
|
||||
from api.video import get_video_db
|
||||
return get_video_db().list_watchlist_playlists()
|
||||
|
||||
|
||||
def _default_fetch_videos(playlist_id: Any) -> List[Dict[str, Any]]:
|
||||
"""A playlist's videos (flat, with durations) with upload dates merged from the cache.
|
||||
Playlists have no per-channel RSS, so dates come from whatever's cached (filled over
|
||||
time by the channel/InnerTube enrichers); undated entries organise cleanly."""
|
||||
from api.video import get_video_db
|
||||
from core.video import youtube as yt
|
||||
db = get_video_db()
|
||||
vids = yt.playlist_videos(str(playlist_id), limit=_PLAYLIST_FETCH_LIMIT) or []
|
||||
ids = [v.get("youtube_id") for v in vids if v.get("youtube_id")]
|
||||
dates = db.get_video_dates(ids) or {}
|
||||
for v in vids:
|
||||
if not v.get("published_at") and dates.get(v.get("youtube_id")):
|
||||
v["published_at"] = dates[v["youtube_id"]]
|
||||
return vids
|
||||
|
||||
|
||||
def _default_wishlisted_ids(playlist_id: Any) -> List[Any]:
|
||||
# parent_source_id holds the playlist id for playlist-sourced wishlist videos.
|
||||
from api.video import get_video_db
|
||||
return get_video_db().wishlisted_video_ids_for_channel(playlist_id)
|
||||
|
||||
|
||||
def _default_downloaded_ids(playlist_id: Any) -> List[Any]:
|
||||
return [] # populated once the download-history ledger carries youtube grabs
|
||||
|
||||
|
||||
def _default_dismissed_ids(playlist_id: Any) -> List[Any]:
|
||||
return [] # no youtube "dismissed" store yet (see channels-scan note)
|
||||
|
||||
|
||||
def _default_add_videos(playlist: Dict[str, Any], videos: List[Dict[str, Any]]) -> int:
|
||||
"""Wishlist under the PLAYLIST as the show (title = playlist name → playlist-as-show)."""
|
||||
from api.video import get_video_db
|
||||
from core.video.sources import resolve_video_server
|
||||
return get_video_db().add_videos_to_wishlist(playlist, videos, server_source=resolve_video_server())
|
||||
|
||||
|
||||
def auto_video_scan_watchlist_playlists(
|
||||
config: Dict[str, Any],
|
||||
deps: AutomationDeps,
|
||||
*,
|
||||
fetch_playlists: Optional[Callable[[], List[Dict[str, Any]]]] = None,
|
||||
fetch_videos: Optional[Callable[[Any], List[Dict[str, Any]]]] = None,
|
||||
wishlisted_ids: Optional[Callable[[Any], Iterable]] = None,
|
||||
downloaded_ids: Optional[Callable[[Any], Iterable]] = None,
|
||||
dismissed_ids: Optional[Callable[[Any], Iterable]] = None,
|
||||
add_videos: Optional[Callable[[Dict[str, Any], List[Dict[str, Any]]], int]] = None,
|
||||
today_fn: Optional[Callable[[], str]] = None,
|
||||
) -> Dict[str, Any]:
|
||||
"""Mirror every followed YouTube playlist into the wishlist (whole list + new additions).
|
||||
|
||||
Returns ``{'status': 'completed', 'playlists': int, 'videos_added': int, ...}``."""
|
||||
fetch_playlists = fetch_playlists or _default_fetch_playlists
|
||||
fetch_videos = fetch_videos or _default_fetch_videos
|
||||
wishlisted_ids = wishlisted_ids or _default_wishlisted_ids
|
||||
downloaded_ids = downloaded_ids or _default_downloaded_ids
|
||||
dismissed_ids = dismissed_ids or _default_dismissed_ids
|
||||
add_videos = add_videos or _default_add_videos
|
||||
today_fn = today_fn or (lambda: date.today().isoformat())
|
||||
automation_id = config.get('_automation_id')
|
||||
min_seconds = max(0, int(config.get('min_seconds', 60) or 0))
|
||||
|
||||
try:
|
||||
today = today_fn()
|
||||
deps.update_progress(automation_id, phase='Reading your watchlist…', progress=5,
|
||||
log_line='Loading the playlists you follow', log_type='info')
|
||||
playlists = fetch_playlists() or []
|
||||
if not playlists:
|
||||
deps.update_progress(automation_id, status='finished', progress=100, phase='Complete',
|
||||
log_line='No playlists on the watchlist to scan', log_type='info')
|
||||
return {'status': 'completed', 'playlists': 0, 'videos_added': 0,
|
||||
'_manages_own_progress': True}
|
||||
|
||||
added = 0
|
||||
total = len(playlists)
|
||||
for i, pl in enumerate(playlists):
|
||||
pid = pl.get('playlist_id')
|
||||
ptitle = pl.get('title') or pid
|
||||
deps.update_progress(automation_id, phase='Scanning playlists…',
|
||||
progress=10 + int(80 * i / max(total, 1)),
|
||||
log_line="Mirroring '%s'" % ptitle, log_type='info')
|
||||
if not pid:
|
||||
continue
|
||||
try:
|
||||
videos = fetch_videos(pid) or []
|
||||
except Exception: # noqa: BLE001 - one flaky playlist shouldn't abort the scan
|
||||
deps.update_progress(automation_id, log_line="Couldn't reach '%s' — skipping" % ptitle,
|
||||
log_type='warning')
|
||||
continue
|
||||
|
||||
gaps = select_playlist_video_gaps(
|
||||
videos, wishlisted_ids=wishlisted_ids(pid), downloaded_ids=downloaded_ids(pid),
|
||||
dismissed_ids=dismissed_ids(pid), today=today, min_seconds=min_seconds)
|
||||
if gaps:
|
||||
n = int(add_videos({'youtube_id': pid, 'title': ptitle,
|
||||
'avatar_url': pl.get('poster_url')}, gaps) or 0)
|
||||
added += n
|
||||
if n:
|
||||
deps.update_progress(
|
||||
automation_id, log_type='success',
|
||||
log_line="Wishlisted %d new video(s) from '%s'" % (n, ptitle))
|
||||
|
||||
done = ('Wishlisted %d new video(s) across %d playlist(s)' % (added, total)) if added \
|
||||
else ('Playlists are up to date — nothing new across %d playlist(s)' % total)
|
||||
deps.update_progress(automation_id, status='finished', progress=100, phase='Complete',
|
||||
log_line=done, log_type='success')
|
||||
return {'status': 'completed', 'playlists': total, 'videos_added': added,
|
||||
'_manages_own_progress': True}
|
||||
except Exception as e: # noqa: BLE001
|
||||
deps.update_progress(automation_id, status='error', phase='Error', log_line=str(e), log_type='error')
|
||||
return {'status': 'error', 'error': str(e), '_manages_own_progress': True}
|
||||
|
|
@ -259,6 +259,15 @@ SYSTEM_AUTOMATIONS = [
|
|||
'initial_delay': 1200,
|
||||
'owned_by': 'video',
|
||||
},
|
||||
# Playlists: every 6h, mirror the whole list (playlist-as-show). No-ops if you follow none.
|
||||
{
|
||||
'name': 'Auto-Scan Watchlist Playlists',
|
||||
'trigger_type': 'schedule',
|
||||
'trigger_config': {'interval': 6, 'unit': 'hours'},
|
||||
'action_type': 'video_scan_watchlist_playlists',
|
||||
'initial_delay': 1320,
|
||||
'owned_by': 'video',
|
||||
},
|
||||
# Drain side: download wished YouTube videos hourly (queues the whole wishlist, runs a
|
||||
# few at a time). Skips quietly until a YouTube library folder is set.
|
||||
{
|
||||
|
|
|
|||
|
|
@ -62,6 +62,7 @@ EXPECTED_ACTION_NAMES = frozenset({
|
|||
'video_add_airing_episodes',
|
||||
'video_scan_watchlist_people',
|
||||
'video_scan_watchlist_channels',
|
||||
'video_scan_watchlist_playlists',
|
||||
'video_process_youtube_wishlist',
|
||||
'video_clean_search_history',
|
||||
'video_clean_completed_downloads',
|
||||
|
|
|
|||
120
tests/test_video_scan_watchlist_playlists.py
Normal file
120
tests/test_video_scan_watchlist_playlists.py
Normal file
|
|
@ -0,0 +1,120 @@
|
|||
"""Watchlist-playlists scan: MIRROR a followed YouTube playlist — wishlist every long-form
|
||||
video in it you don't have (plus new additions), filed playlist-as-show. Differs from the
|
||||
channel scan (no forward-looking baseline, no last-N net). Pure selection, I/O injected.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from core.automation.handlers.video_scan_watchlist_playlists import (
|
||||
auto_video_scan_watchlist_playlists,
|
||||
select_playlist_video_gaps,
|
||||
)
|
||||
|
||||
|
||||
class _Deps:
|
||||
def __init__(self):
|
||||
self.progress = []
|
||||
|
||||
def update_progress(self, automation_id, **kw):
|
||||
self.progress.append(kw)
|
||||
|
||||
|
||||
def _vid(vid, *, date=None, dur=600):
|
||||
return {"youtube_id": vid, "title": "Video " + vid, "published_at": date,
|
||||
"duration_seconds": dur, "thumbnail_url": "/t/" + vid + ".jpg"}
|
||||
|
||||
|
||||
# ── pure: mirror the whole list (no baseline / no net) ────────────────────────
|
||||
def test_mirrors_every_unowned_long_form_video():
|
||||
# old AND new alike — a curated playlist is wanted in full
|
||||
vids = [_vid("a", date="2024-06-01"), _vid("b", date="2018-01-01"),
|
||||
_vid("c", date="2010-05-05")]
|
||||
gaps = select_playlist_video_gaps(vids, today="2026-06-25")
|
||||
assert [g["youtube_id"] for g in gaps] == ["a", "b", "c"] # all of them, even ancient
|
||||
|
||||
|
||||
def test_excludes_already_wishlisted_downloaded_dismissed():
|
||||
vids = [_vid("a"), _vid("b"), _vid("c")]
|
||||
gaps = select_playlist_video_gaps(
|
||||
vids, wishlisted_ids=["a"], downloaded_ids=["b"], dismissed_ids=["c"], today="2026-06-25")
|
||||
assert gaps == []
|
||||
|
||||
|
||||
def test_excludes_shorts_and_future_premieres():
|
||||
vids = [_vid("short", dur=20), _vid("real", dur=600, date="2024-01-01"),
|
||||
_vid("premiere", dur=600, date="2099-01-01")]
|
||||
gaps = select_playlist_video_gaps(vids, today="2026-06-25")
|
||||
assert [g["youtube_id"] for g in gaps] == ["real"]
|
||||
|
||||
|
||||
# ── handler ───────────────────────────────────────────────────────────────────
|
||||
def _run(playlists, videos_by_pl, *, wished=None, today="2026-06-25"):
|
||||
adds = []
|
||||
|
||||
def add_videos(playlist, videos):
|
||||
adds.append((playlist["youtube_id"], playlist["title"], [v["youtube_id"] for v in videos]))
|
||||
return len(videos)
|
||||
|
||||
deps = _Deps()
|
||||
res = auto_video_scan_watchlist_playlists(
|
||||
{"_automation_id": "a"}, deps,
|
||||
fetch_playlists=lambda: playlists,
|
||||
fetch_videos=lambda pid: videos_by_pl.get(pid, []),
|
||||
wishlisted_ids=lambda pid: (wished or {}).get(pid, []),
|
||||
downloaded_ids=lambda pid: [], dismissed_ids=lambda pid: [],
|
||||
add_videos=add_videos, today_fn=lambda: today)
|
||||
return res, adds, deps
|
||||
|
||||
|
||||
def test_wishlists_under_the_playlist_as_the_show():
|
||||
playlists = [{"playlist_id": "PL1", "title": "Best Sci-Fi", "poster_url": "/p.jpg"}]
|
||||
vids = [_vid("a", date="2024-01-01"), _vid("b", date="2023-01-01")]
|
||||
res, adds, _ = _run(playlists, {"PL1": vids})
|
||||
assert res["status"] == "completed" and res["playlists"] == 1 and res["videos_added"] == 2
|
||||
# the PLAYLIST id + title travel to add_videos → playlist-as-show organisation
|
||||
assert adds == [("PL1", "Best Sci-Fi", ["a", "b"])]
|
||||
|
||||
|
||||
def test_rerun_only_adds_new_additions():
|
||||
playlists = [{"playlist_id": "PL1", "title": "Mix"}]
|
||||
vids = [_vid("new", date="2026-06-20"), _vid("old1"), _vid("old2")]
|
||||
res, adds, _ = _run(playlists, {"PL1": vids}, wished={"PL1": ["old1", "old2"]})
|
||||
assert adds == [("PL1", "Mix", ["new"])] and res["videos_added"] == 1
|
||||
|
||||
|
||||
def test_multiple_playlists_independent():
|
||||
playlists = [{"playlist_id": "PL1", "title": "A"}, {"playlist_id": "PL2", "title": "B"}]
|
||||
videos = {"PL1": [_vid("a1")], "PL2": [_vid("b1")]}
|
||||
res, adds, _ = _run(playlists, videos)
|
||||
assert res["playlists"] == 2 and res["videos_added"] == 2
|
||||
|
||||
|
||||
def test_one_unreachable_playlist_does_not_abort():
|
||||
playlists = [{"playlist_id": "PL1", "title": "Breaks"}, {"playlist_id": "PL2", "title": "Works"}]
|
||||
|
||||
def fetch_videos(pid):
|
||||
if pid == "PL1":
|
||||
raise RuntimeError("yt-dlp blew up")
|
||||
return [_vid("ok")]
|
||||
|
||||
adds = []
|
||||
res = auto_video_scan_watchlist_playlists(
|
||||
{"_automation_id": "a"}, _Deps(),
|
||||
fetch_playlists=lambda: playlists, fetch_videos=fetch_videos,
|
||||
wishlisted_ids=lambda pid: [], downloaded_ids=lambda pid: [],
|
||||
dismissed_ids=lambda pid: [], add_videos=lambda pl, v: len(v), today_fn=lambda: "2026-06-25")
|
||||
assert res["status"] == "completed" and res["videos_added"] == 1
|
||||
|
||||
|
||||
def test_empty_watchlist_is_a_clean_noop():
|
||||
res, adds, _ = _run([], {})
|
||||
assert res["status"] == "completed" and res["playlists"] == 0 and adds == []
|
||||
|
||||
|
||||
def test_top_level_error_is_caught():
|
||||
def boom():
|
||||
raise RuntimeError("watchlist read failed")
|
||||
deps = _Deps()
|
||||
res = auto_video_scan_watchlist_playlists({"_automation_id": "a"}, deps, fetch_playlists=boom)
|
||||
assert res["status"] == "error" and "watchlist read failed" in res["error"]
|
||||
assert any(p.get("status") == "error" for p in deps.progress)
|
||||
|
|
@ -1777,7 +1777,7 @@ const _autoIcons = {
|
|||
video_scan_library: '\uD83C\uDFAC', video_scan_server: '\uD83D\uDD04', video_update_database: '\uD83D\uDDC4\uFE0F',
|
||||
video_add_airing_episodes: '\uD83D\uDCFA', video_deep_scan_movies: '\uD83C\uDFAC', video_deep_scan_tv: '\uD83D\uDCFA',
|
||||
video_scan_watchlist_people: '\uD83C\uDFAD', video_scan_watchlist_channels: '\uD83D\uDCE1',
|
||||
video_process_youtube_wishlist: '\u2B07\uFE0F',
|
||||
video_scan_watchlist_playlists: '\uD83C\uDFB5', video_process_youtube_wishlist: '\u2B07\uFE0F',
|
||||
};
|
||||
|
||||
// --- Inspiration Templates ---
|
||||
|
|
@ -3374,6 +3374,7 @@ function _autoFormatAction(type) {
|
|||
video_deep_scan_movies: 'Deep Scan Movie Library', video_deep_scan_tv: 'Deep Scan TV Library',
|
||||
video_scan_watchlist_people: 'Scan Watchlist People',
|
||||
video_scan_watchlist_channels: 'Scan Watchlist Channels',
|
||||
video_scan_watchlist_playlists: 'Scan Watchlist Playlists',
|
||||
video_process_youtube_wishlist: 'Download YouTube Wishlist',
|
||||
};
|
||||
return labels[type] || type || 'Unknown';
|
||||
|
|
|
|||
Loading…
Reference in a new issue