From f380e8ac27369b5f54860667fe56b36502d0828c Mon Sep 17 00:00:00 2001 From: BoulderBadgeDad Date: Fri, 26 Jun 2026 09:34:58 -0700 Subject: [PATCH] youtube scans: dedup against downloaded videos so they don't re-download MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit a completed youtube download is removed from the wishlist (correct), but the channel/playlist scans' downloaded_ids seam was stubbed to [] — so the next scan, not seeing it wishlisted and not knowing it was downloaded, re-added it (still in the channel's recent uploads / last-N net) → re-download loop for the last-N videos every scan. fix: db.downloaded_youtube_video_ids() pulls completed youtube grabs from the permanent download history; both scans' _default_downloaded_ids now use it. so a downloaded video stays gone. tested (db method + the scans already exclude downloaded_ids in their pure logic). --- .../handlers/video_scan_watchlist_channels.py | 7 ++++--- .../handlers/video_scan_watchlist_playlists.py | 5 ++++- database/video_database.py | 13 +++++++++++++ tests/test_video_process_youtube_wishlist.py | 11 +++++++++++ 4 files changed, 32 insertions(+), 4 deletions(-) diff --git a/core/automation/handlers/video_scan_watchlist_channels.py b/core/automation/handlers/video_scan_watchlist_channels.py index 8e623cd3..bdba6801 100644 --- a/core/automation/handlers/video_scan_watchlist_channels.py +++ b/core/automation/handlers/video_scan_watchlist_channels.py @@ -143,9 +143,10 @@ def _default_dismissed_ids(channel_id: Any) -> List[Any]: def _default_downloaded_ids(channel_id: Any) -> List[Any]: - # The permanent download-history ledger has no YouTube grabs until the fulfillment - # engine runs; empty for now, so the scan is correct the day those rows appear. - return [] + # Already-downloaded YouTube videos (global; a video id is unique). A completed download + # leaves the wishlist, so without this the scan would re-add + re-download it. + from api.video import get_video_db + return get_video_db().downloaded_youtube_video_ids() def _default_add_videos(channel: Dict[str, Any], videos: List[Dict[str, Any]]) -> int: diff --git a/core/automation/handlers/video_scan_watchlist_playlists.py b/core/automation/handlers/video_scan_watchlist_playlists.py index c6cbbbac..b3bb64c6 100644 --- a/core/automation/handlers/video_scan_watchlist_playlists.py +++ b/core/automation/handlers/video_scan_watchlist_playlists.py @@ -86,7 +86,10 @@ def _default_wishlisted_ids(playlist_id: Any) -> List[Any]: def _default_downloaded_ids(playlist_id: Any) -> List[Any]: - return [] # populated once the download-history ledger carries youtube grabs + # Already-downloaded YouTube videos (global). A completed download leaves the wishlist, + # so without this the scan would re-add + re-download it. + from api.video import get_video_db + return get_video_db().downloaded_youtube_video_ids() def _default_dismissed_ids(playlist_id: Any) -> List[Any]: diff --git a/database/video_database.py b/database/video_database.py index 98b49373..c6826247 100644 --- a/database/video_database.py +++ b/database/video_database.py @@ -1459,6 +1459,19 @@ class VideoDatabase: finally: conn.close() + def downloaded_youtube_video_ids(self) -> list: + """YouTube video ids already successfully downloaded (from the permanent history). + The watchlist scans exclude these — a completed video is removed from the wishlist + on download, so without this it would be re-added (it's still in the channel's recent + uploads / last-N net) and re-downloaded on every scan.""" + conn = self._get_connection() + try: + return [r["media_id"] for r in conn.execute( + "SELECT DISTINCT media_id FROM video_download_history " + "WHERE source='youtube' AND outcome='completed' AND media_id IS NOT NULL")] + finally: + conn.close() + def media_tmdb_id(self, kind: str, media_id) -> tuple: """(tmdb_id, imdb_id) for a library movie/show row — used to resolve sidecar / subtitle metadata for an owned re-grab (whose media_id is the library id, not a diff --git a/tests/test_video_process_youtube_wishlist.py b/tests/test_video_process_youtube_wishlist.py index e361e658..e54f790d 100644 --- a/tests/test_video_process_youtube_wishlist.py +++ b/tests/test_video_process_youtube_wishlist.py @@ -176,6 +176,17 @@ def test_youtube_wishlist_to_download_shape(db): assert top["video_title"] == "Second" and top["published_at"] == "2024-05-01" +def test_downloaded_youtube_video_ids_only_completed_youtube(db): + # the dedup the scans use so a downloaded video (removed from the wishlist) isn't re-added + db.record_download_history({"id": 1, "kind": "youtube", "source": "youtube", + "media_id": "v1", "status": "completed", "dest_path": "/a.mp4"}) + db.record_download_history({"id": 2, "kind": "youtube", "source": "youtube", + "media_id": "v2", "status": "failed"}) # failed → not counted + db.record_download_history({"id": 3, "kind": "movie", "source": "soulseek", + "media_id": "99", "status": "completed"}) # not youtube + assert set(db.downloaded_youtube_video_ids()) == {"v1"} + + def test_count_and_claim_queue(db): a = db.add_video_download({"kind": "youtube", "source": "youtube", "media_id": "v1", "title": "A", "status": "queued"})