[FEATURE] download_individually for Youtube channel (#101)

src/ytdl_sub/downloaders/downloader.py

@@ -6,6 +6,7 @@ from contextlib import contextmanager
 from pathlib import Path
 from typing import Dict
 from typing import Generic
+from typing import Iterable
 from typing import List
 from typing import Optional
 from typing import Tuple
@@ -199,7 +200,9 @@ class Downloader(DownloadArchiver, Generic[DownloaderOptionsT, DownloaderEntryT]
         return self._get_entry_dicts_from_info_json_files()
 
     @abc.abstractmethod
-    def download(self) -> List[DownloaderEntryT] | List[Tuple[DownloaderEntryT, FileMetadata]]:
+    def download(
+        self,
+    ) -> Iterable[DownloaderEntryT] | Iterable[Tuple[DownloaderEntryT, FileMetadata]]:
         """The function to perform the download of all media entries"""
 
     def post_download(self, overrides: Overrides):

src/ytdl_sub/downloaders/youtube/channel.py

@@ -1,5 +1,6 @@
 from pathlib import Path
 from typing import Dict
+from typing import Generator
 from typing import List
 from typing import Optional
 
@@ -14,6 +15,7 @@ from ytdl_sub.utils.thumbnail import convert_url_thumbnail
 from ytdl_sub.validators.date_range_validator import DateRangeValidator
 from ytdl_sub.validators.string_formatter_validators import OverridesStringFormatterValidator
 from ytdl_sub.validators.url_validator import YoutubeChannelUrlValidator
+from ytdl_sub.validators.validators import BoolValidator
 from ytdl_sub.ytdl_additions.enhanced_download_archive import EnhancedDownloadArchive
 
 logger = Logger.get()
@@ -36,12 +38,19 @@ class YoutubeChannelDownloaderOptions(YoutubeDownloaderOptions, DateRangeValidat
             # optional
             channel_avatar_path: "poster.jpg"
             channel_banner_path: "fanart.jpg"
+            download_individually: True
             before: "now"
             after: "today-2weeks"
     """
 
     _required_keys = {"channel_url"}
-    _optional_keys = {"before", "after", "channel_avatar_path", "channel_banner_path"}
+    _optional_keys = {
+        "before",
+        "after",
+        "channel_avatar_path",
+        "channel_banner_path",
+        "download_individually",
+    }
 
     def __init__(self, name, value):
         YoutubeDownloaderOptions.__init__(self, name, value)
@@ -55,6 +64,9 @@ class YoutubeChannelDownloaderOptions(YoutubeDownloaderOptions, DateRangeValidat
         self._channel_banner_path = self._validate_key_if_present(
             "channel_banner_path", OverridesStringFormatterValidator
         )
+        self._download_individually = self._validate_key_if_present(
+            "download_individually", BoolValidator, default=True
+        )
 
     @property
     def channel_url(self) -> str:
@@ -79,6 +91,15 @@ class YoutubeChannelDownloaderOptions(YoutubeDownloaderOptions, DateRangeValidat
         """
         return self._channel_banner_path
 
+    @property
+    def download_individually(self) -> Optional[bool]:
+        """
+        Optional. Downloads files from the channel individually instead of in bulk. Setting to True
+        is safer when downloading large amounts of videos in case an error occurs. Downloading by
+        bulk (by setting to False) can increase speeds. Defaults to True.
+        """
+        return self._download_individually.value
+
 
 class YoutubeChannelDownloader(YoutubeDownloader[YoutubeChannelDownloaderOptions, YoutubeVideo]):
     downloader_options_type = YoutubeChannelDownloaderOptions
@@ -120,13 +141,25 @@ class YoutubeChannelDownloader(YoutubeDownloader[YoutubeChannelDownloaderOptions
         )
         self.channel: Optional[YoutubeChannel] = None
 
-    def download(self) -> List[YoutubeVideo]:
+    def _get_channel_from_entry_dicts(self, entry_dicts: List[Dict]) -> YoutubeChannel:
+        channel_entry_dict = [
+            entry_dict for entry_dict in entry_dicts if entry_dict.get("extractor") == "youtube:tab"
+        ][0]
+        return YoutubeChannel(
+            entry_dict=channel_entry_dict, working_directory=self.working_directory
+        )
+
+    def download(self) -> Generator[YoutubeVideo, None, None]:
         """
         Downloads all videos from a channel
         """
-        channel_videos: List[YoutubeVideo] = []
         ytdl_options_overrides = {}
 
+        # If downloading individually, dry-run the entire channel download first, this will get the
+        # videos that will be downloaded. Afterwards, download each video one-by-one
+        if self.download_options.download_individually:
+            ytdl_options_overrides["skip_download"] = True
+
         # If a date range is specified when download a YT channel, add it into the ytdl options
         source_date_range = self.download_options.get_date_range()
         if source_date_range:
@@ -135,18 +168,22 @@ class YoutubeChannelDownloader(YoutubeDownloader[YoutubeChannelDownloaderOptions
         entry_dicts = self.extract_info_via_info_json(
             ytdl_options_overrides=ytdl_options_overrides, url=self.download_options.channel_url
         )
+        self.channel = self._get_channel_from_entry_dicts(entry_dicts=entry_dicts)
+
+        # If downloading individually, remove the skip_download to actually download the video
+        if self.download_options.download_individually:
+            del ytdl_options_overrides["skip_download"]
 
         for entry_dict in entry_dicts:
             if entry_dict.get("extractor") == "youtube":
-                channel_videos.append(
+                # Only do the individual download if it is not dry-run and downloading individually
-                    YoutubeVideo(entry_dict=entry_dict, working_directory=self.working_directory)
+                if not self.is_dry_run and self.download_options.download_individually:
-                )
+                    ytdl_options_overrides["playlist_items"] = str(entry_dict.get("playlist_index"))
-            if entry_dict.get("extractor") == "youtube:tab":
+                    _ = self.extract_info(
-                self.channel = YoutubeChannel(
+                        ytdl_options_overrides=ytdl_options_overrides,
-                    entry_dict=entry_dict, working_directory=self.working_directory
+                        url=self.download_options.channel_url,
-                )
+                    )
-
+                yield YoutubeVideo(entry_dict=entry_dict, working_directory=self.working_directory)
-        return channel_videos
 
     def _download_thumbnail(
         self,

src/ytdl_sub/subscriptions/subscription.py

@@ -292,6 +292,10 @@ class Subscription:
                     entry=entry, entry_metadata=entry_metadata
                 )
 
+                # Re-save the download archive after each entry is moved to the output directory
+                if self.maintain_download_archive:
+                    self._enhanced_download_archive.save_download_mappings()
+
             downloader.post_download(overrides=self.overrides)
             for plugin in plugins:
                 plugin.post_process_subscription()

tests/e2e/youtube/test_channel_as_kodi_tv_show.py

@@ -1,4 +1,6 @@
+import copy
 from pathlib import Path
+from typing import Dict
 
 import mergedeep
 import pytest
@@ -8,28 +10,16 @@ from e2e.expected_download import ExpectedDownloads
 from e2e.expected_transaction_log import assert_transaction_log_matches
 
 import ytdl_sub.downloaders.downloader
-from ytdl_sub.config.config_file import ConfigFile
-from ytdl_sub.config.preset import Preset
 from ytdl_sub.subscriptions.subscription import Subscription
 
 
-@pytest.fixture
-def config_path():
-    return "examples/kodi_tv_shows_config.yaml"
-
-
 @pytest.fixture
 def subscription_name():
     return "pz"
 
 
 @pytest.fixture
-def config(config_path):
+def channel_preset_dict(output_directory):
-    return ConfigFile.from_file_path(config_path=config_path)
-
-
-@pytest.fixture
-def subscription_dict(output_directory):
     return {
         "preset": "yt_channel_as_tv",
         "youtube": {"channel_url": "https://youtube.com/channel/UCcRSMoQqXc_JrBZRHDFGbqA"},
@@ -45,24 +35,20 @@ def subscription_dict(output_directory):
     }
 
 
+@pytest.fixture
+def channel_subscription_generator(channel_as_tv_show_config, subscription_name):
+    def _channel_subscription_generator(preset_dict: Dict):
+        return Subscription.from_dict(
+            config=channel_as_tv_show_config, preset_name=subscription_name, preset_dict=preset_dict
+        )
+
+    return _channel_subscription_generator
+
+
 ####################################################################################################
 # FULL CHANNEL FIXTURES
 
 
-@pytest.fixture
-def full_channel_subscription(config, subscription_name, subscription_dict):
-    full_channel_preset = Preset.from_dict(
-        config=config,
-        preset_name=subscription_name,
-        preset_dict=subscription_dict,
-    )
-
-    return Subscription.from_preset(
-        preset=full_channel_preset,
-        config=config,
-    )
-
-
 @pytest.fixture
 def expected_full_channel_download():
     # turn off black formatter here for readability
@@ -133,11 +119,12 @@ def expected_full_channel_download():
 ####################################################################################################
 # RECENT CHANNEL FIXTURES
 @pytest.fixture
-def recent_channel_subscription_dict(subscription_dict):
+def recent_channel_preset_dict(channel_preset_dict):
     # TODO: remove this hack by using a different channel
-    del subscription_dict["ytdl_options"]["break_on_reject"]
+    channel_preset_dict = copy.deepcopy(channel_preset_dict)
+    del channel_preset_dict["ytdl_options"]["break_on_reject"]
     return mergedeep.merge(
-        subscription_dict,
+        channel_preset_dict,
         {
             "preset": "yt_channel_as_tv__recent",
             "youtube": {"after": "20150101"},
@@ -145,20 +132,6 @@ def recent_channel_subscription_dict(subscription_dict):
     )
 
 
-@pytest.fixture
-def recent_channel_subscription(config, subscription_name, recent_channel_subscription_dict):
-    recent_channel_preset = Preset.from_dict(
-        config=config,
-        preset_name=subscription_name,
-        preset_dict=recent_channel_subscription_dict,
-    )
-
-    return Subscription.from_preset(
-        preset=recent_channel_preset,
-        config=config,
-    )
-
-
 @pytest.fixture
 def expected_recent_channel_download():
     # turn off black formatter here for readability
@@ -188,33 +161,6 @@ def expected_recent_channel_download():
 
 ####################################################################################################
 # RECENT CHANNEL FIXTURES -- NO VIDS IN RANGE
-@pytest.fixture
-def recent_channel_no_vids_in_range_subscription_dict(subscription_dict):
-    # TODO: remove this hack by using a different channel
-    del subscription_dict["ytdl_options"]["break_on_reject"]
-    return mergedeep.merge(
-        subscription_dict,
-        {
-            "preset": "yt_channel_as_tv__recent",
-            "youtube": {"after": "20880101"},
-        },
-    )
-
-
-@pytest.fixture
-def recent_channel_no_vids_in_range_subscription(
-    config, subscription_name, recent_channel_no_vids_in_range_subscription_dict
-):
-    recent_channel_preset = Preset.from_dict(
-        config=config,
-        preset_name=subscription_name,
-        preset_dict=recent_channel_no_vids_in_range_subscription_dict,
-    )
-
-    return Subscription.from_preset(
-        preset=recent_channel_preset,
-        config=config,
-    )
 
 
 @pytest.fixture
@@ -238,9 +184,10 @@ def expected_recent_channel_no_vids_in_range_download():
 ####################################################################################################
 # ROLLING RECENT CHANNEL FIXTURES
 @pytest.fixture
-def rolling_recent_channel_subscription_dict(recent_channel_subscription_dict):
+def rolling_recent_channel_preset_dict(recent_channel_preset_dict):
+    recent_channel_preset_dict = copy.deepcopy(recent_channel_preset_dict)
     return mergedeep.merge(
-        recent_channel_subscription_dict,
+        recent_channel_preset_dict,
         {
             "preset": "yt_channel_as_tv__only_recent",
             "output_options": {"keep_files_after": "20181101"},
@@ -248,22 +195,6 @@ def rolling_recent_channel_subscription_dict(recent_channel_subscription_dict):
     )
 
 
-@pytest.fixture
-def rolling_recent_channel_subscription(
-    config, subscription_name, rolling_recent_channel_subscription_dict
-):
-    rolling_recent_channel_preset = Preset.from_dict(
-        config=config,
-        preset_name=subscription_name,
-        preset_dict=rolling_recent_channel_subscription_dict,
-    )
-
-    return Subscription.from_preset(
-        preset=rolling_recent_channel_preset,
-        config=config,
-    )
-
-
 @pytest.fixture
 def expected_rolling_recent_channel_download():
     # turn off black formatter here for readability
@@ -294,9 +225,19 @@ class TestChannelAsKodiTvShow:
     """
 
     @pytest.mark.parametrize("dry_run", [True, False])
+    @pytest.mark.parametrize("download_individually", [True, False])
     def test_full_channel_download(
-        self, full_channel_subscription, expected_full_channel_download, output_directory, dry_run
+        self,
+        channel_subscription_generator,
+        channel_preset_dict,
+        expected_full_channel_download,
+        output_directory,
+        dry_run,
+        download_individually,
     ):
+        channel_preset_dict["youtube"]["download_individually"] = download_individually
+
+        full_channel_subscription = channel_subscription_generator(preset_dict=channel_preset_dict)
         transaction_log = full_channel_subscription.download(dry_run=dry_run)
         assert_transaction_log_matches(
             output_directory=output_directory,
@@ -307,13 +248,22 @@ class TestChannelAsKodiTvShow:
             expected_full_channel_download.assert_files_exist(relative_directory=output_directory)
 
     @pytest.mark.parametrize("dry_run", [True, False])
+    @pytest.mark.parametrize("download_individually", [True, False])
     def test_recent_channel_download(
         self,
-        recent_channel_subscription,
+        channel_subscription_generator,
+        recent_channel_preset_dict,
         expected_recent_channel_download,
         output_directory,
         dry_run,
+        download_individually,
     ):
+        recent_channel_preset_dict["youtube"]["download_individually"] = download_individually
+
+        recent_channel_subscription = channel_subscription_generator(
+            preset_dict=recent_channel_preset_dict
+        )
+
         transaction_log = recent_channel_subscription.download(dry_run=dry_run)
         assert_transaction_log_matches(
             output_directory=output_directory,
@@ -341,13 +291,22 @@ class TestChannelAsKodiTvShow:
                 )
 
     @pytest.mark.parametrize("dry_run", [True, False])
+    @pytest.mark.parametrize("download_individually", [True, False])
     def test_recent_channel_download__no_vids_in_range(
         self,
-        recent_channel_no_vids_in_range_subscription,
+        channel_subscription_generator,
+        recent_channel_preset_dict,
         expected_recent_channel_no_vids_in_range_download,
         output_directory,
         dry_run,
+        download_individually,
     ):
+        recent_channel_preset_dict["youtube"]["download_individually"] = download_individually
+        recent_channel_preset_dict["youtube"]["after"] = "21000101"
+
+        recent_channel_no_vids_in_range_subscription = channel_subscription_generator(
+            preset_dict=recent_channel_preset_dict
+        )
         # Run twice, ensure nothing changes between runs
         for _ in range(2):
             transaction_log = recent_channel_no_vids_in_range_subscription.download(dry_run=dry_run)
@@ -362,15 +321,30 @@ class TestChannelAsKodiTvShow:
                 )
 
     @pytest.mark.parametrize("dry_run", [True, False])
+    @pytest.mark.parametrize("download_individually", [True, False])
     def test_rolling_recent_channel_download(
         self,
-        recent_channel_subscription,
+        channel_subscription_generator,
-        rolling_recent_channel_subscription,
+        recent_channel_preset_dict,
+        rolling_recent_channel_preset_dict,
         expected_recent_channel_download,
         expected_rolling_recent_channel_download,
         output_directory,
         dry_run,
+        download_individually,
     ):
+        recent_channel_preset_dict["youtube"]["download_individually"] = download_individually
+        rolling_recent_channel_preset_dict["youtube"][
+            "download_individually"
+        ] = download_individually
+
+        recent_channel_subscription = channel_subscription_generator(
+            preset_dict=recent_channel_preset_dict
+        )
+        rolling_recent_channel_subscription = channel_subscription_generator(
+            preset_dict=rolling_recent_channel_preset_dict
+        )
+
         # First, download recent vids. Always download since we want to test dry-run
         # on the rolling recent portion.
         with assert_debug_log(