ytdl_options defaults for each download strategy (#72)

* `ytdl_options` defaults for each download strategy

* fix docker, fix defaults getting added, add debugger test

* test playlist as well

* fix with hack

* fix full channel test

* existing

docker/Dockerfile

@@ -14,8 +14,8 @@ RUN apk update --no-cache && \
       python3 \
       py3-pip && \
     apk add --repository=http://dl-3.alpinelinux.org/alpine/edge/main/ \
-      python3=3.10.4-r0 \
+      python3=~3.10 \
-      py3-setuptools=59.4.0-r0 && \
+      py3-setuptools && \
     apk add --repository=http://dl-3.alpinelinux.org/alpine/edge/community/ \
       py3-pip && \
     mkdir -p /config && \

docker/root/defaults/config.yaml

@@ -13,9 +13,6 @@ presets:
     youtube:
       download_strategy: "video"
 
-    ytdl_options:
-      ignoreerrors: True
-
     output_options:
       output_directory: "{music_video_directory}"
       file_name: "{music_video_name}.{ext}"
@@ -45,6 +42,3 @@ presets:
 
     output_options:
       maintain_download_archive: True
-
-    ytdl_options:
-      break_on_existing: True

docs/config.rst

@@ -51,6 +51,8 @@ _______
   :inherited-members:
   :exclude-members: get_date_range
 
+.. autofunction:: ytdl_sub.downloaders.youtube_downloader.YoutubeChannelDownloader.ytdl_option_defaults()
+
 -------------------------------------------------------------------------------
 
 .. _YouTube Playlist:
@@ -62,6 +64,8 @@ ________
   :member-order: bysource
   :inherited-members:
 
+.. autofunction:: ytdl_sub.downloaders.youtube_downloader.YoutubePlaylistDownloader.ytdl_option_defaults()
+
 -------------------------------------------------------------------------------
 
 video
@@ -71,6 +75,8 @@ _____
   :member-order: bysource
   :inherited-members:
 
+.. autofunction:: ytdl_sub.downloaders.youtube_downloader.YoutubeVideoDownloader.ytdl_option_defaults()
+
 -------------------------------------------------------------------------------
 
 soundcloud
@@ -86,6 +92,8 @@ __________________
   :member-order: bysource
   :inherited-members:
 
+.. autofunction:: ytdl_sub.downloaders.soundcloud_downloader.SoundcloudAlbumsAndSinglesDownloader.ytdl_option_defaults()
+
 -------------------------------------------------------------------------------
 
 output_options

examples/kodi_music_videos_config.yaml

@@ -23,11 +23,13 @@ presets:
     youtube:
       download_strategy: "video"
 
-    # For advanced YTDL users only.
+    # For advanced YTDL users only; any YTDL parameter can be set here.
-    # It is wise to leave ignoreerrors=True to avoid errors for things
+    # To download age-restricted videos, you will need to set your cookie
-    # like age-restricted videos in case you have not set your cookie.
+    # file here as a ytdl parameter. For more info, see
-    ytdl_options:
+    # https://ytdl-sub.readthedocs.io/en/latest/faq.html#download-age-restricted-youtube-videos
-      ignoreerrors: True
+    #
+    # ytdl_options:
+    #   cookiefile: "path/to/cookie_file.txt
 
     # For each video downloaded, set the file and thumbnail name here.
     # We set both with {music_video_name}, which is a variable we define in
@@ -70,8 +72,3 @@ presets:
     output_options:
       maintain_download_archive: True
 
-    # Setting break_on_existing=True is also a good thing anytime you are using
-    # a download archive, because it will tell the downloader to stop trying
-    # to download videos that have already been downloaded.
-    ytdl_options:
-      break_on_existing: True

examples/kodi_tv_shows_config.yaml

@@ -22,10 +22,8 @@ configuration:
 presets:
 
   ###############################################################################
-  # LEVEL 1 - FULL ARCHIVE
+  # FULL ARCHIVE
-  #
+  # This preset will download every single video in a YouTube channel.
-  # We will call this preset `yt_channel_as_tv`, and it will download every single video in
-  # a YouTube channel.
   yt_channel_as_tv:
     # YouTube channels are our source/download strategy
     # Use the channel avatar and banner images for Kodi
@@ -34,11 +32,13 @@ presets:
       channel_avatar_path: "poster.jpg"
       channel_banner_path: "fanart.jpg"
 
-    # For advanced YTDL users only.
+    # For advanced YTDL users only; any YTDL parameter can be set here.
-    # It is wise to leave ignoreerrors=True to avoid errors for things
+    # To download age-restricted videos, you will need to set your cookie
-    # like age-restricted videos in case you have not set your cookie.
+    # file here as a ytdl parameter. For more info, see
-    ytdl_options:
+    # https://ytdl-sub.readthedocs.io/en/latest/faq.html#download-age-restricted-youtube-videos
-      ignoreerrors: True
+    #
+    # ytdl_options:
+    #   cookiefile: "path/to/cookie_file.txt
 
     # For each video downloaded, set the file and thumbnail name here.
     # We set both with {episode_name}, which is a variable we define in
@@ -85,11 +85,9 @@ presets:
       episode_name: "Season {upload_year}/s{upload_year}.e{upload_month_padded}{upload_day_padded} - {title_sanitized}"
 
   ###############################################################################
-  # LEVEL 2 - RECENT ARCHIVE
+  # RECENT ARCHIVE
-  #
+  # This preset shows how you can download just the last 14 days-worth of videos
-  # It is not always ideal to go full data-hoarder on a channel's videos.
+  # on each download invocation.
-  # This example shows how you can only download the last 14 days-worth
-  # of videos on each download invocation.
   yt_channel_as_tv__recent:
     # `preset` can be set to any other preset in the config, and will inherit all its defined fields.
     # This helps reduce copy-paste in the config.yaml
@@ -100,30 +98,10 @@ presets:
     youtube:
       after: "today-2weeks"
 
-    # This is getting into YTDL voodoo. By default, YTDL will try to
-    # download all channel videos beginning with the most recent one.
-    # By setting break_on_reject, we will break this full-download on
-    # the first video that gets rejected. Since we have youtube.after
-    # defined, all videos after today-14days will be rejected. Therefore,
-    # the first video that is out of range that it tries to download, it
-    # will stop there, and save a significant amount of time.
-    #
-    # Similar to break_on_reject, but instead, breaks if a video has
-    # already been downloaded. If you were to perform a download on this
-    # subscription one-after-the-other, the second invocation would stop
-    # after looking at the first (most recent) video since it would exist
-    # in the download archive.
-    ytdl_options:
-      break_on_reject: True
-      break_on_existing: True
-
   ###############################################################################
-  # LEVEL 3 - ROLLING ARCHIVE
+  # ROLLING ARCHIVE
-
+  # This example shows how to only keep the last 14-days worth of videos, and
-  # If you put the recent archive example in a cron job, then in a year or so
+  # delete the rest.
-  # you will basically be datahoarding that channel unless you manually delete
-  # old videos. This example shows how to only keep the last 14-days worth of videos,
-  # and delete the rest.
   yt_channel_as_tv__only_recent:
     # Reuse `yt_channel_as_tv__recent` to only download the last 2 weeks' of videos
     preset: "yt_channel_as_tv__recent"

examples/soundcloud_discography_config.yaml

@@ -26,11 +26,11 @@ presets:
       download_strategy: "albums_and_singles"
       skip_premiere_tracks: True
 
-    # For advanced YTDL users only.
+    # For advanced YTDL users only; any YTDL parameter can be set here.
-    # This sets the download format to grab the best mp3 available.
+    # You can set the type of format you prefer YTDL to download here.
-    # Other formats should work but have not been tested.
+    # By default, soundcloud will download the best mp3.
-    ytdl_options:
+    # ytdl_options:
-      format: 'bestaudio[ext=mp3]'
+    #   format: 'bestaudio[ext=mp3]'
 
     # For each song downloaded, set the file and thumbnail name here.
     # The output directory stores all artists in a shared {music_directory}.

src/ytdl_sub/downloaders/downloader.py

@@ -21,6 +21,8 @@ from ytdl_sub.entries.entry import Entry
 from ytdl_sub.utils.logger import Logger
 from ytdl_sub.validators.strict_dict_validator import StrictDictValidator
 
+logger = Logger.get(name="downloader")
+
 
 class DownloaderValidator(StrictDictValidator, ABC):
     """
@@ -49,8 +51,13 @@ class Downloader(Generic[DownloaderOptionsT, DownloaderEntryT], ABC):
 
     @classmethod
     def ytdl_option_defaults(cls) -> Dict:
-        """Downloader defaults that can be overwritten from user input"""
+        """
-        return {}
+        .. code-block:: yaml
+
+           ytdl_options:
+             ignoreerrors: True  # ignore errors like hidden videos, age restriction, etc
+        """
+        return {"ignoreerrors": True}
 
     @classmethod
     def _configure_ytdl_options(
@@ -101,7 +108,7 @@ class Downloader(Generic[DownloaderOptionsT, DownloaderEntryT], ABC):
         """
         self.working_directory = working_directory
         self.download_options = download_options
-        self.ytdl_options = Downloader._configure_ytdl_options(
+        self.ytdl_options = self._configure_ytdl_options(
             ytdl_options=ytdl_options,
             working_directory=self.working_directory,
             download_archive_file_name=download_archive_file_name,
@@ -180,8 +187,10 @@ class Downloader(Generic[DownloaderOptionsT, DownloaderEntryT], ABC):
 
         try:
             _ = self.extract_info(ytdl_options_overrides=ytdl_options_overrides, **kwargs)
-        except (RejectedVideoReached, ExistingVideoReached):
+        except RejectedVideoReached:
-            pass
+            logger.debug("RejectedVideoReached, stopping additional downloads")
+        except ExistingVideoReached:
+            logger.debug("ExistingVideoReached, stopping additional downloads")
 
         return self._get_entry_dicts_from_info_json_files()
 

src/ytdl_sub/downloaders/soundcloud_downloader.py

@@ -53,13 +53,6 @@ class SoundcloudDownloader(
 
     downloader_entry_type = SoundcloudTrack
 
-    @classmethod
-    def ytdl_option_defaults(cls) -> Dict:
-        """Returns default format to be best mp3"""
-        return {
-            "format": "bestaudio[ext=mp3]",
-        }
-
     @classmethod
     def artist_albums_url(cls, artist_url: str) -> str:
         """
@@ -124,6 +117,24 @@ class SoundcloudAlbumsAndSinglesDownloader(
 ):
     downloader_options_type = SoundcloudAlbumsAndSinglesDownloadOptions
 
+    @classmethod
+    def ytdl_option_defaults(cls) -> Dict:
+        """
+        Default `ytdl_options`_ for ``albums_and_singles``
+
+        .. code-block:: yaml
+
+           ytdl_options:
+             ignoreerrors: True  # ignore errors like hidden videos, age restriction, etc
+             format: "bestaudio[ext=mp3]"  # download format the best possible mp3
+        """
+        return dict(
+            super().ytdl_option_defaults(),
+            **{
+                "format": "bestaudio[ext=mp3]",
+            },
+        )
+
     def _get_albums(self, entry_dicts: List[Dict]) -> List[SoundcloudAlbum]:
         """
         Parameters

src/ytdl_sub/downloaders/youtube_downloader.py

@@ -95,6 +95,21 @@ class YoutubeVideoDownloader(YoutubeDownloader[YoutubeVideoDownloaderOptions, Yo
     downloader_options_type = YoutubeVideoDownloaderOptions
     downloader_entry_type = YoutubeVideo
 
+    @classmethod
+    def ytdl_option_defaults(cls) -> Dict:
+        """
+        Default `ytdl_options`_ for ``video``
+
+        .. code-block:: yaml
+
+           ytdl_options:
+             ignoreerrors: True  # ignore errors like hidden videos, age restriction, etc
+        """
+        return dict(
+            super().ytdl_option_defaults(),
+            **{"break_on_existing": True},
+        )
+
     def download(self) -> List[YoutubeVideo]:
         """Download a single Youtube video"""
         entry_dict = self.extract_info(url=self.download_options.video_url)
@@ -144,6 +159,25 @@ class YoutubePlaylistDownloader(
     downloader_options_type = YoutubePlaylistDownloaderOptions
     downloader_entry_type = YoutubePlaylistVideo
 
+    # pylint: disable=line-too-long
+    @classmethod
+    def ytdl_option_defaults(cls) -> Dict:
+        """
+        Default `ytdl_options`_ for ``playlist``
+
+        .. code-block:: yaml
+
+           ytdl_options:
+             ignoreerrors: True  # ignore errors like hidden videos, age restriction, etc
+             break_on_existing: True  # stop downloads (newest to oldest) if a video is already downloaded
+        """
+        return dict(
+            super().ytdl_option_defaults(),
+            **{"break_on_existing": True},
+        )
+
+    # pylint: enable=line-too-long
+
     def download(self) -> List[YoutubePlaylistVideo]:
         """
         Downloads all videos in a Youtube playlist
@@ -231,6 +265,29 @@ class YoutubeChannelDownloader(YoutubeDownloader[YoutubeChannelDownloaderOptions
     downloader_options_type = YoutubeChannelDownloaderOptions
     downloader_entry_type = YoutubeVideo
 
+    # pylint: disable=line-too-long
+    @classmethod
+    def ytdl_option_defaults(cls) -> Dict:
+        """
+        Default `ytdl_options`_ for ``channel``
+
+        .. code-block:: yaml
+
+           ytdl_options:
+             ignoreerrors: True  # ignore errors like hidden videos, age restriction, etc
+             break_on_existing: True  # stop downloads (newest to oldest) if a video is already downloaded
+             break_on_reject: True  # stops downloads if the video's upload date is out of the specified 'before'/'after' range
+        """
+        return dict(
+            super().ytdl_option_defaults(),
+            **{
+                "break_on_existing": True,
+                "break_on_reject": True,
+            },
+        )
+
+    # pylint: enable=line-too-long
+
     def __init__(
         self,
         working_directory: str,

tests/conftest.py

@@ -0,0 +1,28 @@
+import contextlib
+import logging
+from typing import Callable
+from unittest.mock import MagicMock
+from unittest.mock import patch
+
+from ytdl_sub.utils.logger import Logger
+
+
+@contextlib.contextmanager
+def assert_debug_log(logger: logging.Logger, expected_message: str):
+    """
+    Patches any function, but calls the original function.
+    Intended to see if the particular function is called.
+    """
+    debug_logger = Logger.get()
+
+    def _wrapped_debug(*args, **kwargs):
+        debug_logger.info(*args, **kwargs)
+
+    with patch.object(logger, "debug", wraps=_wrapped_debug) as patched_debug:
+        yield
+
+    for call_args in patched_debug.call_args_list:
+        if expected_message in call_args.args[0]:
+            return
+
+    assert False, f"{expected_message} was not found in a logger.debug call"

tests/e2e/youtube/test_channel_as_kodi_tv_show.py

@@ -2,8 +2,10 @@ from pathlib import Path
 
 import mergedeep
 import pytest
+from conftest import assert_debug_log
 from e2e.expected_download import ExpectedDownload
 
+import ytdl_sub.downloaders.downloader
 from ytdl_sub.config.config_file import ConfigFile
 from ytdl_sub.config.preset import Preset
 from ytdl_sub.subscriptions.subscription import Subscription
@@ -35,6 +37,7 @@ def subscription_dict(output_directory, subscription_name):
         "ytdl_options": {
             "format": "worst[ext=mp4]",
             "max_views": 100000,  # do not download the popular PJ concert
+            "break_on_reject": False,  # do not break from max views
         },
         "overrides": {"tv_show_name": "Project / Zombie"},
     }
@@ -129,6 +132,8 @@ def expected_full_channel_download():
 # RECENT CHANNEL FIXTURES
 @pytest.fixture
 def recent_channel_subscription_dict(subscription_dict):
+    # TODO: remove this hack by using a different channel
+    del subscription_dict["ytdl_options"]["break_on_reject"]
     return mergedeep.merge(
         subscription_dict,
         {
@@ -250,9 +255,12 @@ class TestChannelAsKodiTvShow:
         expected_recent_channel_download.assert_files_exist(relative_directory=output_directory)
 
         # try downloading again, ensure nothing more was downloaded
-        # TODO: add patch around the output of download to see what entry dicts were returned
+        with assert_debug_log(
-        recent_channel_subscription.download()
+            logger=ytdl_sub.downloaders.downloader.logger,
-        expected_recent_channel_download.assert_files_exist(relative_directory=output_directory)
+            expected_message="ExistingVideoReached, stopping additional downloads",
+        ):
+            recent_channel_subscription.download()
+            expected_recent_channel_download.assert_files_exist(relative_directory=output_directory)
 
     def test_rolling_recent_channel_download(
         self,
@@ -263,18 +271,31 @@ class TestChannelAsKodiTvShow:
         output_directory,
     ):
         # First, download recent vids
-        recent_channel_subscription.download()
+        with assert_debug_log(
-        expected_recent_channel_download.assert_files_exist(relative_directory=output_directory)
+            logger=ytdl_sub.downloaders.downloader.logger,
+            expected_message="RejectedVideoReached, stopping additional downloads",
+        ):
+            recent_channel_subscription.download()
+            expected_recent_channel_download.assert_files_exist(relative_directory=output_directory)
 
         # Then, download the rolling recent vids subscription. This should remove one of the
         # two videos
-        rolling_recent_channel_subscription.download()
+        with assert_debug_log(
-        expected_rolling_recent_channel_download.assert_files_exist(
+            logger=ytdl_sub.downloaders.downloader.logger,
-            relative_directory=output_directory
+            expected_message="ExistingVideoReached, stopping additional downloads",
-        )
+        ):
+            rolling_recent_channel_subscription.download()
+            expected_rolling_recent_channel_download.assert_files_exist(
+                relative_directory=output_directory
+            )
 
-        # Invoke the rolling download again, ensure nothing has changed
+        # Invoke the rolling download again, ensure downloading stopped early from it already
-        rolling_recent_channel_subscription.download()
+        # existing
-        expected_rolling_recent_channel_download.assert_files_exist(
+        with assert_debug_log(
-            relative_directory=output_directory
+            logger=ytdl_sub.downloaders.downloader.logger,
-        )
+            expected_message="ExistingVideoReached, stopping additional downloads",
+        ):
+            rolling_recent_channel_subscription.download()
+            expected_rolling_recent_channel_download.assert_files_exist(
+                relative_directory=output_directory
+            )

tests/e2e/youtube/test_playlist_as_kodi_music_videos.py

@@ -2,8 +2,10 @@ from pathlib import Path
 
 import mergedeep
 import pytest
+from conftest import assert_debug_log
 from e2e.expected_download import ExpectedDownload
 
+import ytdl_sub.downloaders.downloader
 from ytdl_sub.config.config_file import ConfigFile
 from ytdl_sub.config.preset import Preset
 from ytdl_sub.subscriptions.subscription import Subscription
@@ -138,6 +140,14 @@ class TestPlaylistAsKodiMusicVideo:
         playlist_subscription.download()
         expected_playlist_download.assert_files_exist(relative_directory=output_directory)
 
+        # After the playlist is downloaded, ensure another invocation will hit ExistingVideoReached
+        with assert_debug_log(
+            logger=ytdl_sub.downloaders.downloader.logger,
+            expected_message="ExistingVideoReached, stopping additional downloads",
+        ):
+            playlist_subscription.download()
+            expected_playlist_download.assert_files_exist(relative_directory=output_directory)
+
     def test_single_video_download(
         self, single_video_subscription, expected_single_video_download, output_directory
     ):