From 605de7469712a5825f416a844a674e217800b030 Mon Sep 17 00:00:00 2001 From: Jesse Bannon Date: Sun, 17 Jul 2022 22:02:50 -0700 Subject: [PATCH] [FEATURE] `download_individually` for youtube playlists (#102) --- src/ytdl_sub/downloaders/youtube/playlist.py | 54 ++++++++++++--- .../test_playlist_as_kodi_music_videos.py | 67 +++++++++---------- 2 files changed, 76 insertions(+), 45 deletions(-) diff --git a/src/ytdl_sub/downloaders/youtube/playlist.py b/src/ytdl_sub/downloaders/youtube/playlist.py index 68f537a4..7dd3041f 100644 --- a/src/ytdl_sub/downloaders/youtube/playlist.py +++ b/src/ytdl_sub/downloaders/youtube/playlist.py @@ -1,10 +1,12 @@ from typing import Dict -from typing import List +from typing import Generator +from typing import Optional from ytdl_sub.downloaders.youtube.abc import YoutubeDownloader from ytdl_sub.downloaders.youtube.abc import YoutubeDownloaderOptions from ytdl_sub.entries.youtube import YoutubePlaylistVideo from ytdl_sub.validators.url_validator import YoutubePlaylistUrlValidator +from ytdl_sub.validators.validators import BoolValidator class YoutubePlaylistDownloaderOptions(YoutubeDownloaderOptions): @@ -21,15 +23,21 @@ class YoutubePlaylistDownloaderOptions(YoutubeDownloaderOptions): # required download_strategy: "playlist" playlist_url: "https://www.youtube.com/playlist?list=UCsvn_Po0SmunchJYtttWpOxMg" + # optional + download_individually: True """ _required_keys = {"playlist_url"} + _optional_keys = {"download_individually"} def __init__(self, name, value): super().__init__(name, value) self._playlist_url = self._validate_key( "playlist_url", YoutubePlaylistUrlValidator ).playlist_url + self._download_individually = self._validate_key_if_present( + "download_individually", BoolValidator, default=True + ) @property def playlist_url(self) -> str: @@ -39,6 +47,15 @@ class YoutubePlaylistDownloaderOptions(YoutubeDownloaderOptions): """ return self._playlist_url + @property + def download_individually(self) -> Optional[bool]: + """ + Optional. Downloads files from the playlist individually instead of in bulk. Setting to True + is safer when downloading large amounts of videos in case an error occurs. Downloading by + bulk (by setting to False) can increase speeds. Defaults to True. + """ + return self._download_individually.value + class YoutubePlaylistDownloader( YoutubeDownloader[YoutubePlaylistDownloaderOptions, YoutubePlaylistVideo] @@ -65,19 +82,36 @@ class YoutubePlaylistDownloader( # pylint: enable=line-too-long - def download(self) -> List[YoutubePlaylistVideo]: + def download(self) -> Generator[YoutubePlaylistVideo, None, None]: """ Downloads all videos in a Youtube playlist """ - playlist_videos: List[YoutubePlaylistVideo] = [] + ytdl_options_overrides = {} + + # If downloading individually, dry-run the entire channel download first, this will get the + # videos that will be downloaded. Afterwards, download each video one-by-one + if self.download_options.download_individually: + ytdl_options_overrides["skip_download"] = True + + entry_dicts = self.extract_info_via_info_json( + ytdl_options_overrides=ytdl_options_overrides, url=self.download_options.playlist_url + ) + + # If downloading individually, remove the skip_download to actually download the video + if self.download_options.download_individually: + del ytdl_options_overrides["skip_download"] - entry_dicts = self.extract_info_via_info_json(url=self.download_options.playlist_url) for entry_dict in entry_dicts: if entry_dict.get("extractor") == "youtube": - playlist_videos.append( - YoutubePlaylistVideo( - entry_dict=entry_dict, working_directory=self.working_directory - ) - ) - return playlist_videos + # Only do the individual download if it is not dry-run and downloading individually + if not self.is_dry_run and self.download_options.download_individually: + ytdl_options_overrides["playlist_items"] = str(entry_dict.get("playlist_index")) + _ = self.extract_info( + ytdl_options_overrides=ytdl_options_overrides, + url=self.download_options.playlist_url, + ) + + yield YoutubePlaylistVideo( + entry_dict=entry_dict, working_directory=self.working_directory + ) diff --git a/tests/e2e/youtube/test_playlist_as_kodi_music_videos.py b/tests/e2e/youtube/test_playlist_as_kodi_music_videos.py index a8e0e797..a3e3471c 100644 --- a/tests/e2e/youtube/test_playlist_as_kodi_music_videos.py +++ b/tests/e2e/youtube/test_playlist_as_kodi_music_videos.py @@ -8,12 +8,11 @@ from e2e.expected_download import ExpectedDownloads from e2e.expected_transaction_log import assert_transaction_log_matches import ytdl_sub.downloaders.downloader -from ytdl_sub.config.preset import Preset from ytdl_sub.subscriptions.subscription import Subscription @pytest.fixture -def subscription_dict(output_directory): +def playlist_preset_dict(output_directory): return { "preset": "yt_music_video_playlist", "youtube": {"playlist_url": "https://youtube.com/playlist?list=PL5BC0FC26BECA5A35"}, @@ -31,20 +30,6 @@ def subscription_dict(output_directory): # PLAYLIST FIXTURES -@pytest.fixture -def playlist_subscription(music_video_config, subscription_dict): - playlist_preset = Preset.from_dict( - config=music_video_config, - preset_name="music_video_playlist_test", - preset_dict=subscription_dict, - ) - - return Subscription.from_preset( - preset=playlist_preset, - config=music_video_config, - ) - - @pytest.fixture def expected_playlist_download(): # turn off black formatter here for readability @@ -74,11 +59,11 @@ def expected_playlist_download(): #################################################################################################### # SINGLE VIDEO FIXTURES @pytest.fixture -def single_video_subscription_dict(subscription_dict): - del subscription_dict["youtube"] +def single_video_preset_dict(playlist_preset_dict): + del playlist_preset_dict["youtube"] return mergedeep.merge( - subscription_dict, + playlist_preset_dict, { "preset": "yt_music_video", "youtube": {"video_url": "https://youtube.com/watch?v=HKTNxEqsN3Q"}, @@ -86,20 +71,6 @@ def single_video_subscription_dict(subscription_dict): ) -@pytest.fixture -def single_video_subscription(music_video_config, single_video_subscription_dict): - single_video_preset = Preset.from_dict( - config=music_video_config, - preset_name="music_video_single_video_test", - preset_dict=single_video_subscription_dict, - ) - - return Subscription.from_preset( - preset=single_video_preset, - config=music_video_config, - ) - - @pytest.fixture def expected_single_video_download(): # turn off black formatter here for readability @@ -121,9 +92,24 @@ class TestPlaylistAsKodiMusicVideo: """ @pytest.mark.parametrize("dry_run", [True, False]) + @pytest.mark.parametrize("download_individually", [True, False]) def test_playlist_download( - self, playlist_subscription, expected_playlist_download, output_directory, dry_run + self, + music_video_config, + playlist_preset_dict, + expected_playlist_download, + output_directory, + dry_run, + download_individually, ): + playlist_preset_dict["youtube"]["download_individually"] = download_individually + + playlist_subscription = Subscription.from_dict( + config=music_video_config, + preset_name="music_video_playlist_test", + preset_dict=playlist_preset_dict, + ) + transaction_log = playlist_subscription.download() assert_transaction_log_matches( output_directory=output_directory, @@ -144,8 +130,19 @@ class TestPlaylistAsKodiMusicVideo: @pytest.mark.parametrize("dry_run", [True, False]) def test_single_video_download( - self, single_video_subscription, expected_single_video_download, output_directory, dry_run + self, + music_video_config, + single_video_preset_dict, + expected_single_video_download, + output_directory, + dry_run, ): + single_video_subscription = Subscription.from_dict( + config=music_video_config, + preset_name="music_video_single_video_test", + preset_dict=single_video_preset_dict, + ) + transaction_log = single_video_subscription.download() assert_transaction_log_matches( output_directory=output_directory,