youtube scans: dedup against downloaded videos so they don't re-download
a completed youtube download is removed from the wishlist (correct), but the channel/playlist scans' downloaded_ids seam was stubbed to [] — so the next scan, not seeing it wishlisted and not knowing it was downloaded, re-added it (still in the channel's recent uploads / last-N net) → re-download loop for the last-N videos every scan. fix: db.downloaded_youtube_video_ids() pulls completed youtube grabs from the permanent download history; both scans' _default_downloaded_ids now use it. so a downloaded video stays gone. tested (db method + the scans already exclude downloaded_ids in their pure logic).
This commit is contained in:
parent
bacc528ba1
commit
f380e8ac27
4 changed files with 32 additions and 4 deletions
|
|
@ -143,9 +143,10 @@ def _default_dismissed_ids(channel_id: Any) -> List[Any]:
|
||||||
|
|
||||||
|
|
||||||
def _default_downloaded_ids(channel_id: Any) -> List[Any]:
|
def _default_downloaded_ids(channel_id: Any) -> List[Any]:
|
||||||
# The permanent download-history ledger has no YouTube grabs until the fulfillment
|
# Already-downloaded YouTube videos (global; a video id is unique). A completed download
|
||||||
# engine runs; empty for now, so the scan is correct the day those rows appear.
|
# leaves the wishlist, so without this the scan would re-add + re-download it.
|
||||||
return []
|
from api.video import get_video_db
|
||||||
|
return get_video_db().downloaded_youtube_video_ids()
|
||||||
|
|
||||||
|
|
||||||
def _default_add_videos(channel: Dict[str, Any], videos: List[Dict[str, Any]]) -> int:
|
def _default_add_videos(channel: Dict[str, Any], videos: List[Dict[str, Any]]) -> int:
|
||||||
|
|
|
||||||
|
|
@ -86,7 +86,10 @@ def _default_wishlisted_ids(playlist_id: Any) -> List[Any]:
|
||||||
|
|
||||||
|
|
||||||
def _default_downloaded_ids(playlist_id: Any) -> List[Any]:
|
def _default_downloaded_ids(playlist_id: Any) -> List[Any]:
|
||||||
return [] # populated once the download-history ledger carries youtube grabs
|
# Already-downloaded YouTube videos (global). A completed download leaves the wishlist,
|
||||||
|
# so without this the scan would re-add + re-download it.
|
||||||
|
from api.video import get_video_db
|
||||||
|
return get_video_db().downloaded_youtube_video_ids()
|
||||||
|
|
||||||
|
|
||||||
def _default_dismissed_ids(playlist_id: Any) -> List[Any]:
|
def _default_dismissed_ids(playlist_id: Any) -> List[Any]:
|
||||||
|
|
|
||||||
|
|
@ -1459,6 +1459,19 @@ class VideoDatabase:
|
||||||
finally:
|
finally:
|
||||||
conn.close()
|
conn.close()
|
||||||
|
|
||||||
|
def downloaded_youtube_video_ids(self) -> list:
|
||||||
|
"""YouTube video ids already successfully downloaded (from the permanent history).
|
||||||
|
The watchlist scans exclude these — a completed video is removed from the wishlist
|
||||||
|
on download, so without this it would be re-added (it's still in the channel's recent
|
||||||
|
uploads / last-N net) and re-downloaded on every scan."""
|
||||||
|
conn = self._get_connection()
|
||||||
|
try:
|
||||||
|
return [r["media_id"] for r in conn.execute(
|
||||||
|
"SELECT DISTINCT media_id FROM video_download_history "
|
||||||
|
"WHERE source='youtube' AND outcome='completed' AND media_id IS NOT NULL")]
|
||||||
|
finally:
|
||||||
|
conn.close()
|
||||||
|
|
||||||
def media_tmdb_id(self, kind: str, media_id) -> tuple:
|
def media_tmdb_id(self, kind: str, media_id) -> tuple:
|
||||||
"""(tmdb_id, imdb_id) for a library movie/show row — used to resolve sidecar /
|
"""(tmdb_id, imdb_id) for a library movie/show row — used to resolve sidecar /
|
||||||
subtitle metadata for an owned re-grab (whose media_id is the library id, not a
|
subtitle metadata for an owned re-grab (whose media_id is the library id, not a
|
||||||
|
|
|
||||||
|
|
@ -176,6 +176,17 @@ def test_youtube_wishlist_to_download_shape(db):
|
||||||
assert top["video_title"] == "Second" and top["published_at"] == "2024-05-01"
|
assert top["video_title"] == "Second" and top["published_at"] == "2024-05-01"
|
||||||
|
|
||||||
|
|
||||||
|
def test_downloaded_youtube_video_ids_only_completed_youtube(db):
|
||||||
|
# the dedup the scans use so a downloaded video (removed from the wishlist) isn't re-added
|
||||||
|
db.record_download_history({"id": 1, "kind": "youtube", "source": "youtube",
|
||||||
|
"media_id": "v1", "status": "completed", "dest_path": "/a.mp4"})
|
||||||
|
db.record_download_history({"id": 2, "kind": "youtube", "source": "youtube",
|
||||||
|
"media_id": "v2", "status": "failed"}) # failed → not counted
|
||||||
|
db.record_download_history({"id": 3, "kind": "movie", "source": "soulseek",
|
||||||
|
"media_id": "99", "status": "completed"}) # not youtube
|
||||||
|
assert set(db.downloaded_youtube_video_ids()) == {"v1"}
|
||||||
|
|
||||||
|
|
||||||
def test_count_and_claim_queue(db):
|
def test_count_and_claim_queue(db):
|
||||||
a = db.add_video_download({"kind": "youtube", "source": "youtube", "media_id": "v1",
|
a = db.add_video_download({"kind": "youtube", "source": "youtube", "media_id": "v1",
|
||||||
"title": "A", "status": "queued"})
|
"title": "A", "status": "queued"})
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue