From 8e35c1674be1b4cb0fa9addb227e89e56d6bc859 Mon Sep 17 00:00:00 2001 From: Jesse Bannon Date: Sun, 17 Jul 2022 18:17:00 -0700 Subject: [PATCH] [FEATURE] `download_individually` for Youtube channel (#101) --- src/ytdl_sub/downloaders/downloader.py | 5 +- src/ytdl_sub/downloaders/youtube/channel.py | 61 +++++-- src/ytdl_sub/subscriptions/subscription.py | 4 + .../youtube/test_channel_as_kodi_tv_show.py | 162 ++++++++---------- 4 files changed, 125 insertions(+), 107 deletions(-) diff --git a/src/ytdl_sub/downloaders/downloader.py b/src/ytdl_sub/downloaders/downloader.py index a2743121..add6339b 100644 --- a/src/ytdl_sub/downloaders/downloader.py +++ b/src/ytdl_sub/downloaders/downloader.py @@ -6,6 +6,7 @@ from contextlib import contextmanager from pathlib import Path from typing import Dict from typing import Generic +from typing import Iterable from typing import List from typing import Optional from typing import Tuple @@ -199,7 +200,9 @@ class Downloader(DownloadArchiver, Generic[DownloaderOptionsT, DownloaderEntryT] return self._get_entry_dicts_from_info_json_files() @abc.abstractmethod - def download(self) -> List[DownloaderEntryT] | List[Tuple[DownloaderEntryT, FileMetadata]]: + def download( + self, + ) -> Iterable[DownloaderEntryT] | Iterable[Tuple[DownloaderEntryT, FileMetadata]]: """The function to perform the download of all media entries""" def post_download(self, overrides: Overrides): diff --git a/src/ytdl_sub/downloaders/youtube/channel.py b/src/ytdl_sub/downloaders/youtube/channel.py index a5aef803..1157e6a9 100644 --- a/src/ytdl_sub/downloaders/youtube/channel.py +++ b/src/ytdl_sub/downloaders/youtube/channel.py @@ -1,5 +1,6 @@ from pathlib import Path from typing import Dict +from typing import Generator from typing import List from typing import Optional @@ -14,6 +15,7 @@ from ytdl_sub.utils.thumbnail import convert_url_thumbnail from ytdl_sub.validators.date_range_validator import DateRangeValidator from ytdl_sub.validators.string_formatter_validators import OverridesStringFormatterValidator from ytdl_sub.validators.url_validator import YoutubeChannelUrlValidator +from ytdl_sub.validators.validators import BoolValidator from ytdl_sub.ytdl_additions.enhanced_download_archive import EnhancedDownloadArchive logger = Logger.get() @@ -36,12 +38,19 @@ class YoutubeChannelDownloaderOptions(YoutubeDownloaderOptions, DateRangeValidat # optional channel_avatar_path: "poster.jpg" channel_banner_path: "fanart.jpg" + download_individually: True before: "now" after: "today-2weeks" """ _required_keys = {"channel_url"} - _optional_keys = {"before", "after", "channel_avatar_path", "channel_banner_path"} + _optional_keys = { + "before", + "after", + "channel_avatar_path", + "channel_banner_path", + "download_individually", + } def __init__(self, name, value): YoutubeDownloaderOptions.__init__(self, name, value) @@ -55,6 +64,9 @@ class YoutubeChannelDownloaderOptions(YoutubeDownloaderOptions, DateRangeValidat self._channel_banner_path = self._validate_key_if_present( "channel_banner_path", OverridesStringFormatterValidator ) + self._download_individually = self._validate_key_if_present( + "download_individually", BoolValidator, default=True + ) @property def channel_url(self) -> str: @@ -79,6 +91,15 @@ class YoutubeChannelDownloaderOptions(YoutubeDownloaderOptions, DateRangeValidat """ return self._channel_banner_path + @property + def download_individually(self) -> Optional[bool]: + """ + Optional. Downloads files from the channel individually instead of in bulk. Setting to True + is safer when downloading large amounts of videos in case an error occurs. Downloading by + bulk (by setting to False) can increase speeds. Defaults to True. + """ + return self._download_individually.value + class YoutubeChannelDownloader(YoutubeDownloader[YoutubeChannelDownloaderOptions, YoutubeVideo]): downloader_options_type = YoutubeChannelDownloaderOptions @@ -120,13 +141,25 @@ class YoutubeChannelDownloader(YoutubeDownloader[YoutubeChannelDownloaderOptions ) self.channel: Optional[YoutubeChannel] = None - def download(self) -> List[YoutubeVideo]: + def _get_channel_from_entry_dicts(self, entry_dicts: List[Dict]) -> YoutubeChannel: + channel_entry_dict = [ + entry_dict for entry_dict in entry_dicts if entry_dict.get("extractor") == "youtube:tab" + ][0] + return YoutubeChannel( + entry_dict=channel_entry_dict, working_directory=self.working_directory + ) + + def download(self) -> Generator[YoutubeVideo, None, None]: """ Downloads all videos from a channel """ - channel_videos: List[YoutubeVideo] = [] ytdl_options_overrides = {} + # If downloading individually, dry-run the entire channel download first, this will get the + # videos that will be downloaded. Afterwards, download each video one-by-one + if self.download_options.download_individually: + ytdl_options_overrides["skip_download"] = True + # If a date range is specified when download a YT channel, add it into the ytdl options source_date_range = self.download_options.get_date_range() if source_date_range: @@ -135,18 +168,22 @@ class YoutubeChannelDownloader(YoutubeDownloader[YoutubeChannelDownloaderOptions entry_dicts = self.extract_info_via_info_json( ytdl_options_overrides=ytdl_options_overrides, url=self.download_options.channel_url ) + self.channel = self._get_channel_from_entry_dicts(entry_dicts=entry_dicts) + + # If downloading individually, remove the skip_download to actually download the video + if self.download_options.download_individually: + del ytdl_options_overrides["skip_download"] for entry_dict in entry_dicts: if entry_dict.get("extractor") == "youtube": - channel_videos.append( - YoutubeVideo(entry_dict=entry_dict, working_directory=self.working_directory) - ) - if entry_dict.get("extractor") == "youtube:tab": - self.channel = YoutubeChannel( - entry_dict=entry_dict, working_directory=self.working_directory - ) - - return channel_videos + # Only do the individual download if it is not dry-run and downloading individually + if not self.is_dry_run and self.download_options.download_individually: + ytdl_options_overrides["playlist_items"] = str(entry_dict.get("playlist_index")) + _ = self.extract_info( + ytdl_options_overrides=ytdl_options_overrides, + url=self.download_options.channel_url, + ) + yield YoutubeVideo(entry_dict=entry_dict, working_directory=self.working_directory) def _download_thumbnail( self, diff --git a/src/ytdl_sub/subscriptions/subscription.py b/src/ytdl_sub/subscriptions/subscription.py index 8dab4fdf..eb7476c9 100644 --- a/src/ytdl_sub/subscriptions/subscription.py +++ b/src/ytdl_sub/subscriptions/subscription.py @@ -292,6 +292,10 @@ class Subscription: entry=entry, entry_metadata=entry_metadata ) + # Re-save the download archive after each entry is moved to the output directory + if self.maintain_download_archive: + self._enhanced_download_archive.save_download_mappings() + downloader.post_download(overrides=self.overrides) for plugin in plugins: plugin.post_process_subscription() diff --git a/tests/e2e/youtube/test_channel_as_kodi_tv_show.py b/tests/e2e/youtube/test_channel_as_kodi_tv_show.py index 61d160c3..9f2f2bdc 100644 --- a/tests/e2e/youtube/test_channel_as_kodi_tv_show.py +++ b/tests/e2e/youtube/test_channel_as_kodi_tv_show.py @@ -1,4 +1,6 @@ +import copy from pathlib import Path +from typing import Dict import mergedeep import pytest @@ -8,28 +10,16 @@ from e2e.expected_download import ExpectedDownloads from e2e.expected_transaction_log import assert_transaction_log_matches import ytdl_sub.downloaders.downloader -from ytdl_sub.config.config_file import ConfigFile -from ytdl_sub.config.preset import Preset from ytdl_sub.subscriptions.subscription import Subscription -@pytest.fixture -def config_path(): - return "examples/kodi_tv_shows_config.yaml" - - @pytest.fixture def subscription_name(): return "pz" @pytest.fixture -def config(config_path): - return ConfigFile.from_file_path(config_path=config_path) - - -@pytest.fixture -def subscription_dict(output_directory): +def channel_preset_dict(output_directory): return { "preset": "yt_channel_as_tv", "youtube": {"channel_url": "https://youtube.com/channel/UCcRSMoQqXc_JrBZRHDFGbqA"}, @@ -45,24 +35,20 @@ def subscription_dict(output_directory): } +@pytest.fixture +def channel_subscription_generator(channel_as_tv_show_config, subscription_name): + def _channel_subscription_generator(preset_dict: Dict): + return Subscription.from_dict( + config=channel_as_tv_show_config, preset_name=subscription_name, preset_dict=preset_dict + ) + + return _channel_subscription_generator + + #################################################################################################### # FULL CHANNEL FIXTURES -@pytest.fixture -def full_channel_subscription(config, subscription_name, subscription_dict): - full_channel_preset = Preset.from_dict( - config=config, - preset_name=subscription_name, - preset_dict=subscription_dict, - ) - - return Subscription.from_preset( - preset=full_channel_preset, - config=config, - ) - - @pytest.fixture def expected_full_channel_download(): # turn off black formatter here for readability @@ -133,11 +119,12 @@ def expected_full_channel_download(): #################################################################################################### # RECENT CHANNEL FIXTURES @pytest.fixture -def recent_channel_subscription_dict(subscription_dict): +def recent_channel_preset_dict(channel_preset_dict): # TODO: remove this hack by using a different channel - del subscription_dict["ytdl_options"]["break_on_reject"] + channel_preset_dict = copy.deepcopy(channel_preset_dict) + del channel_preset_dict["ytdl_options"]["break_on_reject"] return mergedeep.merge( - subscription_dict, + channel_preset_dict, { "preset": "yt_channel_as_tv__recent", "youtube": {"after": "20150101"}, @@ -145,20 +132,6 @@ def recent_channel_subscription_dict(subscription_dict): ) -@pytest.fixture -def recent_channel_subscription(config, subscription_name, recent_channel_subscription_dict): - recent_channel_preset = Preset.from_dict( - config=config, - preset_name=subscription_name, - preset_dict=recent_channel_subscription_dict, - ) - - return Subscription.from_preset( - preset=recent_channel_preset, - config=config, - ) - - @pytest.fixture def expected_recent_channel_download(): # turn off black formatter here for readability @@ -188,33 +161,6 @@ def expected_recent_channel_download(): #################################################################################################### # RECENT CHANNEL FIXTURES -- NO VIDS IN RANGE -@pytest.fixture -def recent_channel_no_vids_in_range_subscription_dict(subscription_dict): - # TODO: remove this hack by using a different channel - del subscription_dict["ytdl_options"]["break_on_reject"] - return mergedeep.merge( - subscription_dict, - { - "preset": "yt_channel_as_tv__recent", - "youtube": {"after": "20880101"}, - }, - ) - - -@pytest.fixture -def recent_channel_no_vids_in_range_subscription( - config, subscription_name, recent_channel_no_vids_in_range_subscription_dict -): - recent_channel_preset = Preset.from_dict( - config=config, - preset_name=subscription_name, - preset_dict=recent_channel_no_vids_in_range_subscription_dict, - ) - - return Subscription.from_preset( - preset=recent_channel_preset, - config=config, - ) @pytest.fixture @@ -238,9 +184,10 @@ def expected_recent_channel_no_vids_in_range_download(): #################################################################################################### # ROLLING RECENT CHANNEL FIXTURES @pytest.fixture -def rolling_recent_channel_subscription_dict(recent_channel_subscription_dict): +def rolling_recent_channel_preset_dict(recent_channel_preset_dict): + recent_channel_preset_dict = copy.deepcopy(recent_channel_preset_dict) return mergedeep.merge( - recent_channel_subscription_dict, + recent_channel_preset_dict, { "preset": "yt_channel_as_tv__only_recent", "output_options": {"keep_files_after": "20181101"}, @@ -248,22 +195,6 @@ def rolling_recent_channel_subscription_dict(recent_channel_subscription_dict): ) -@pytest.fixture -def rolling_recent_channel_subscription( - config, subscription_name, rolling_recent_channel_subscription_dict -): - rolling_recent_channel_preset = Preset.from_dict( - config=config, - preset_name=subscription_name, - preset_dict=rolling_recent_channel_subscription_dict, - ) - - return Subscription.from_preset( - preset=rolling_recent_channel_preset, - config=config, - ) - - @pytest.fixture def expected_rolling_recent_channel_download(): # turn off black formatter here for readability @@ -294,9 +225,19 @@ class TestChannelAsKodiTvShow: """ @pytest.mark.parametrize("dry_run", [True, False]) + @pytest.mark.parametrize("download_individually", [True, False]) def test_full_channel_download( - self, full_channel_subscription, expected_full_channel_download, output_directory, dry_run + self, + channel_subscription_generator, + channel_preset_dict, + expected_full_channel_download, + output_directory, + dry_run, + download_individually, ): + channel_preset_dict["youtube"]["download_individually"] = download_individually + + full_channel_subscription = channel_subscription_generator(preset_dict=channel_preset_dict) transaction_log = full_channel_subscription.download(dry_run=dry_run) assert_transaction_log_matches( output_directory=output_directory, @@ -307,13 +248,22 @@ class TestChannelAsKodiTvShow: expected_full_channel_download.assert_files_exist(relative_directory=output_directory) @pytest.mark.parametrize("dry_run", [True, False]) + @pytest.mark.parametrize("download_individually", [True, False]) def test_recent_channel_download( self, - recent_channel_subscription, + channel_subscription_generator, + recent_channel_preset_dict, expected_recent_channel_download, output_directory, dry_run, + download_individually, ): + recent_channel_preset_dict["youtube"]["download_individually"] = download_individually + + recent_channel_subscription = channel_subscription_generator( + preset_dict=recent_channel_preset_dict + ) + transaction_log = recent_channel_subscription.download(dry_run=dry_run) assert_transaction_log_matches( output_directory=output_directory, @@ -341,13 +291,22 @@ class TestChannelAsKodiTvShow: ) @pytest.mark.parametrize("dry_run", [True, False]) + @pytest.mark.parametrize("download_individually", [True, False]) def test_recent_channel_download__no_vids_in_range( self, - recent_channel_no_vids_in_range_subscription, + channel_subscription_generator, + recent_channel_preset_dict, expected_recent_channel_no_vids_in_range_download, output_directory, dry_run, + download_individually, ): + recent_channel_preset_dict["youtube"]["download_individually"] = download_individually + recent_channel_preset_dict["youtube"]["after"] = "21000101" + + recent_channel_no_vids_in_range_subscription = channel_subscription_generator( + preset_dict=recent_channel_preset_dict + ) # Run twice, ensure nothing changes between runs for _ in range(2): transaction_log = recent_channel_no_vids_in_range_subscription.download(dry_run=dry_run) @@ -362,15 +321,30 @@ class TestChannelAsKodiTvShow: ) @pytest.mark.parametrize("dry_run", [True, False]) + @pytest.mark.parametrize("download_individually", [True, False]) def test_rolling_recent_channel_download( self, - recent_channel_subscription, - rolling_recent_channel_subscription, + channel_subscription_generator, + recent_channel_preset_dict, + rolling_recent_channel_preset_dict, expected_recent_channel_download, expected_rolling_recent_channel_download, output_directory, dry_run, + download_individually, ): + recent_channel_preset_dict["youtube"]["download_individually"] = download_individually + rolling_recent_channel_preset_dict["youtube"][ + "download_individually" + ] = download_individually + + recent_channel_subscription = channel_subscription_generator( + preset_dict=recent_channel_preset_dict + ) + rolling_recent_channel_subscription = channel_subscription_generator( + preset_dict=rolling_recent_channel_preset_dict + ) + # First, download recent vids. Always download since we want to test dry-run # on the rolling recent portion. with assert_debug_log(