diff --git a/docker/Dockerfile b/docker/Dockerfile index 1e5ac152..8b5d7d9b 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -14,8 +14,8 @@ RUN apk update --no-cache && \ python3 \ py3-pip && \ apk add --repository=http://dl-3.alpinelinux.org/alpine/edge/main/ \ - python3=3.10.4-r0 \ - py3-setuptools=59.4.0-r0 && \ + python3=~3.10 \ + py3-setuptools && \ apk add --repository=http://dl-3.alpinelinux.org/alpine/edge/community/ \ py3-pip && \ mkdir -p /config && \ diff --git a/docker/root/defaults/config.yaml b/docker/root/defaults/config.yaml index e7b4c6b0..b95b4fc5 100644 --- a/docker/root/defaults/config.yaml +++ b/docker/root/defaults/config.yaml @@ -13,9 +13,6 @@ presets: youtube: download_strategy: "video" - ytdl_options: - ignoreerrors: True - output_options: output_directory: "{music_video_directory}" file_name: "{music_video_name}.{ext}" @@ -45,6 +42,3 @@ presets: output_options: maintain_download_archive: True - - ytdl_options: - break_on_existing: True diff --git a/docs/config.rst b/docs/config.rst index 169ee871..c102a584 100644 --- a/docs/config.rst +++ b/docs/config.rst @@ -51,6 +51,8 @@ _______ :inherited-members: :exclude-members: get_date_range +.. autofunction:: ytdl_sub.downloaders.youtube_downloader.YoutubeChannelDownloader.ytdl_option_defaults() + ------------------------------------------------------------------------------- .. _YouTube Playlist: @@ -62,6 +64,8 @@ ________ :member-order: bysource :inherited-members: +.. autofunction:: ytdl_sub.downloaders.youtube_downloader.YoutubePlaylistDownloader.ytdl_option_defaults() + ------------------------------------------------------------------------------- video @@ -71,6 +75,8 @@ _____ :member-order: bysource :inherited-members: +.. autofunction:: ytdl_sub.downloaders.youtube_downloader.YoutubeVideoDownloader.ytdl_option_defaults() + ------------------------------------------------------------------------------- soundcloud @@ -86,6 +92,8 @@ __________________ :member-order: bysource :inherited-members: +.. autofunction:: ytdl_sub.downloaders.soundcloud_downloader.SoundcloudAlbumsAndSinglesDownloader.ytdl_option_defaults() + ------------------------------------------------------------------------------- output_options diff --git a/examples/kodi_music_videos_config.yaml b/examples/kodi_music_videos_config.yaml index 589d3290..1f16ca55 100644 --- a/examples/kodi_music_videos_config.yaml +++ b/examples/kodi_music_videos_config.yaml @@ -23,11 +23,13 @@ presets: youtube: download_strategy: "video" - # For advanced YTDL users only. - # It is wise to leave ignoreerrors=True to avoid errors for things - # like age-restricted videos in case you have not set your cookie. - ytdl_options: - ignoreerrors: True + # For advanced YTDL users only; any YTDL parameter can be set here. + # To download age-restricted videos, you will need to set your cookie + # file here as a ytdl parameter. For more info, see + # https://ytdl-sub.readthedocs.io/en/latest/faq.html#download-age-restricted-youtube-videos + # + # ytdl_options: + # cookiefile: "path/to/cookie_file.txt # For each video downloaded, set the file and thumbnail name here. # We set both with {music_video_name}, which is a variable we define in @@ -70,8 +72,3 @@ presets: output_options: maintain_download_archive: True - # Setting break_on_existing=True is also a good thing anytime you are using - # a download archive, because it will tell the downloader to stop trying - # to download videos that have already been downloaded. - ytdl_options: - break_on_existing: True diff --git a/examples/kodi_tv_shows_config.yaml b/examples/kodi_tv_shows_config.yaml index 6bcb4db8..1fa4de79 100644 --- a/examples/kodi_tv_shows_config.yaml +++ b/examples/kodi_tv_shows_config.yaml @@ -22,10 +22,8 @@ configuration: presets: ############################################################################### - # LEVEL 1 - FULL ARCHIVE - # - # We will call this preset `yt_channel_as_tv`, and it will download every single video in - # a YouTube channel. + # FULL ARCHIVE + # This preset will download every single video in a YouTube channel. yt_channel_as_tv: # YouTube channels are our source/download strategy # Use the channel avatar and banner images for Kodi @@ -34,11 +32,13 @@ presets: channel_avatar_path: "poster.jpg" channel_banner_path: "fanart.jpg" - # For advanced YTDL users only. - # It is wise to leave ignoreerrors=True to avoid errors for things - # like age-restricted videos in case you have not set your cookie. - ytdl_options: - ignoreerrors: True + # For advanced YTDL users only; any YTDL parameter can be set here. + # To download age-restricted videos, you will need to set your cookie + # file here as a ytdl parameter. For more info, see + # https://ytdl-sub.readthedocs.io/en/latest/faq.html#download-age-restricted-youtube-videos + # + # ytdl_options: + # cookiefile: "path/to/cookie_file.txt # For each video downloaded, set the file and thumbnail name here. # We set both with {episode_name}, which is a variable we define in @@ -85,11 +85,9 @@ presets: episode_name: "Season {upload_year}/s{upload_year}.e{upload_month_padded}{upload_day_padded} - {title_sanitized}" ############################################################################### - # LEVEL 2 - RECENT ARCHIVE - # - # It is not always ideal to go full data-hoarder on a channel's videos. - # This example shows how you can only download the last 14 days-worth - # of videos on each download invocation. + # RECENT ARCHIVE + # This preset shows how you can download just the last 14 days-worth of videos + # on each download invocation. yt_channel_as_tv__recent: # `preset` can be set to any other preset in the config, and will inherit all its defined fields. # This helps reduce copy-paste in the config.yaml @@ -100,30 +98,10 @@ presets: youtube: after: "today-2weeks" - # This is getting into YTDL voodoo. By default, YTDL will try to - # download all channel videos beginning with the most recent one. - # By setting break_on_reject, we will break this full-download on - # the first video that gets rejected. Since we have youtube.after - # defined, all videos after today-14days will be rejected. Therefore, - # the first video that is out of range that it tries to download, it - # will stop there, and save a significant amount of time. - # - # Similar to break_on_reject, but instead, breaks if a video has - # already been downloaded. If you were to perform a download on this - # subscription one-after-the-other, the second invocation would stop - # after looking at the first (most recent) video since it would exist - # in the download archive. - ytdl_options: - break_on_reject: True - break_on_existing: True - ############################################################################### - # LEVEL 3 - ROLLING ARCHIVE - - # If you put the recent archive example in a cron job, then in a year or so - # you will basically be datahoarding that channel unless you manually delete - # old videos. This example shows how to only keep the last 14-days worth of videos, - # and delete the rest. + # ROLLING ARCHIVE + # This example shows how to only keep the last 14-days worth of videos, and + # delete the rest. yt_channel_as_tv__only_recent: # Reuse `yt_channel_as_tv__recent` to only download the last 2 weeks' of videos preset: "yt_channel_as_tv__recent" diff --git a/examples/soundcloud_discography_config.yaml b/examples/soundcloud_discography_config.yaml index 1bc061fb..13304eb4 100644 --- a/examples/soundcloud_discography_config.yaml +++ b/examples/soundcloud_discography_config.yaml @@ -26,11 +26,11 @@ presets: download_strategy: "albums_and_singles" skip_premiere_tracks: True - # For advanced YTDL users only. - # This sets the download format to grab the best mp3 available. - # Other formats should work but have not been tested. - ytdl_options: - format: 'bestaudio[ext=mp3]' + # For advanced YTDL users only; any YTDL parameter can be set here. + # You can set the type of format you prefer YTDL to download here. + # By default, soundcloud will download the best mp3. + # ytdl_options: + # format: 'bestaudio[ext=mp3]' # For each song downloaded, set the file and thumbnail name here. # The output directory stores all artists in a shared {music_directory}. diff --git a/src/ytdl_sub/downloaders/downloader.py b/src/ytdl_sub/downloaders/downloader.py index 06dddd08..5bacf31a 100644 --- a/src/ytdl_sub/downloaders/downloader.py +++ b/src/ytdl_sub/downloaders/downloader.py @@ -21,6 +21,8 @@ from ytdl_sub.entries.entry import Entry from ytdl_sub.utils.logger import Logger from ytdl_sub.validators.strict_dict_validator import StrictDictValidator +logger = Logger.get(name="downloader") + class DownloaderValidator(StrictDictValidator, ABC): """ @@ -49,8 +51,13 @@ class Downloader(Generic[DownloaderOptionsT, DownloaderEntryT], ABC): @classmethod def ytdl_option_defaults(cls) -> Dict: - """Downloader defaults that can be overwritten from user input""" - return {} + """ + .. code-block:: yaml + + ytdl_options: + ignoreerrors: True # ignore errors like hidden videos, age restriction, etc + """ + return {"ignoreerrors": True} @classmethod def _configure_ytdl_options( @@ -101,7 +108,7 @@ class Downloader(Generic[DownloaderOptionsT, DownloaderEntryT], ABC): """ self.working_directory = working_directory self.download_options = download_options - self.ytdl_options = Downloader._configure_ytdl_options( + self.ytdl_options = self._configure_ytdl_options( ytdl_options=ytdl_options, working_directory=self.working_directory, download_archive_file_name=download_archive_file_name, @@ -180,8 +187,10 @@ class Downloader(Generic[DownloaderOptionsT, DownloaderEntryT], ABC): try: _ = self.extract_info(ytdl_options_overrides=ytdl_options_overrides, **kwargs) - except (RejectedVideoReached, ExistingVideoReached): - pass + except RejectedVideoReached: + logger.debug("RejectedVideoReached, stopping additional downloads") + except ExistingVideoReached: + logger.debug("ExistingVideoReached, stopping additional downloads") return self._get_entry_dicts_from_info_json_files() diff --git a/src/ytdl_sub/downloaders/soundcloud_downloader.py b/src/ytdl_sub/downloaders/soundcloud_downloader.py index eee50289..c68e3424 100644 --- a/src/ytdl_sub/downloaders/soundcloud_downloader.py +++ b/src/ytdl_sub/downloaders/soundcloud_downloader.py @@ -53,13 +53,6 @@ class SoundcloudDownloader( downloader_entry_type = SoundcloudTrack - @classmethod - def ytdl_option_defaults(cls) -> Dict: - """Returns default format to be best mp3""" - return { - "format": "bestaudio[ext=mp3]", - } - @classmethod def artist_albums_url(cls, artist_url: str) -> str: """ @@ -124,6 +117,24 @@ class SoundcloudAlbumsAndSinglesDownloader( ): downloader_options_type = SoundcloudAlbumsAndSinglesDownloadOptions + @classmethod + def ytdl_option_defaults(cls) -> Dict: + """ + Default `ytdl_options`_ for ``albums_and_singles`` + + .. code-block:: yaml + + ytdl_options: + ignoreerrors: True # ignore errors like hidden videos, age restriction, etc + format: "bestaudio[ext=mp3]" # download format the best possible mp3 + """ + return dict( + super().ytdl_option_defaults(), + **{ + "format": "bestaudio[ext=mp3]", + }, + ) + def _get_albums(self, entry_dicts: List[Dict]) -> List[SoundcloudAlbum]: """ Parameters diff --git a/src/ytdl_sub/downloaders/youtube_downloader.py b/src/ytdl_sub/downloaders/youtube_downloader.py index c247cdd3..9690c833 100644 --- a/src/ytdl_sub/downloaders/youtube_downloader.py +++ b/src/ytdl_sub/downloaders/youtube_downloader.py @@ -95,6 +95,21 @@ class YoutubeVideoDownloader(YoutubeDownloader[YoutubeVideoDownloaderOptions, Yo downloader_options_type = YoutubeVideoDownloaderOptions downloader_entry_type = YoutubeVideo + @classmethod + def ytdl_option_defaults(cls) -> Dict: + """ + Default `ytdl_options`_ for ``video`` + + .. code-block:: yaml + + ytdl_options: + ignoreerrors: True # ignore errors like hidden videos, age restriction, etc + """ + return dict( + super().ytdl_option_defaults(), + **{"break_on_existing": True}, + ) + def download(self) -> List[YoutubeVideo]: """Download a single Youtube video""" entry_dict = self.extract_info(url=self.download_options.video_url) @@ -144,6 +159,25 @@ class YoutubePlaylistDownloader( downloader_options_type = YoutubePlaylistDownloaderOptions downloader_entry_type = YoutubePlaylistVideo + # pylint: disable=line-too-long + @classmethod + def ytdl_option_defaults(cls) -> Dict: + """ + Default `ytdl_options`_ for ``playlist`` + + .. code-block:: yaml + + ytdl_options: + ignoreerrors: True # ignore errors like hidden videos, age restriction, etc + break_on_existing: True # stop downloads (newest to oldest) if a video is already downloaded + """ + return dict( + super().ytdl_option_defaults(), + **{"break_on_existing": True}, + ) + + # pylint: enable=line-too-long + def download(self) -> List[YoutubePlaylistVideo]: """ Downloads all videos in a Youtube playlist @@ -231,6 +265,29 @@ class YoutubeChannelDownloader(YoutubeDownloader[YoutubeChannelDownloaderOptions downloader_options_type = YoutubeChannelDownloaderOptions downloader_entry_type = YoutubeVideo + # pylint: disable=line-too-long + @classmethod + def ytdl_option_defaults(cls) -> Dict: + """ + Default `ytdl_options`_ for ``channel`` + + .. code-block:: yaml + + ytdl_options: + ignoreerrors: True # ignore errors like hidden videos, age restriction, etc + break_on_existing: True # stop downloads (newest to oldest) if a video is already downloaded + break_on_reject: True # stops downloads if the video's upload date is out of the specified 'before'/'after' range + """ + return dict( + super().ytdl_option_defaults(), + **{ + "break_on_existing": True, + "break_on_reject": True, + }, + ) + + # pylint: enable=line-too-long + def __init__( self, working_directory: str, diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 00000000..122419e1 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,28 @@ +import contextlib +import logging +from typing import Callable +from unittest.mock import MagicMock +from unittest.mock import patch + +from ytdl_sub.utils.logger import Logger + + +@contextlib.contextmanager +def assert_debug_log(logger: logging.Logger, expected_message: str): + """ + Patches any function, but calls the original function. + Intended to see if the particular function is called. + """ + debug_logger = Logger.get() + + def _wrapped_debug(*args, **kwargs): + debug_logger.info(*args, **kwargs) + + with patch.object(logger, "debug", wraps=_wrapped_debug) as patched_debug: + yield + + for call_args in patched_debug.call_args_list: + if expected_message in call_args.args[0]: + return + + assert False, f"{expected_message} was not found in a logger.debug call" diff --git a/tests/e2e/youtube/test_channel_as_kodi_tv_show.py b/tests/e2e/youtube/test_channel_as_kodi_tv_show.py index 54c6d769..419937fd 100644 --- a/tests/e2e/youtube/test_channel_as_kodi_tv_show.py +++ b/tests/e2e/youtube/test_channel_as_kodi_tv_show.py @@ -2,8 +2,10 @@ from pathlib import Path import mergedeep import pytest +from conftest import assert_debug_log from e2e.expected_download import ExpectedDownload +import ytdl_sub.downloaders.downloader from ytdl_sub.config.config_file import ConfigFile from ytdl_sub.config.preset import Preset from ytdl_sub.subscriptions.subscription import Subscription @@ -35,6 +37,7 @@ def subscription_dict(output_directory, subscription_name): "ytdl_options": { "format": "worst[ext=mp4]", "max_views": 100000, # do not download the popular PJ concert + "break_on_reject": False, # do not break from max views }, "overrides": {"tv_show_name": "Project / Zombie"}, } @@ -129,6 +132,8 @@ def expected_full_channel_download(): # RECENT CHANNEL FIXTURES @pytest.fixture def recent_channel_subscription_dict(subscription_dict): + # TODO: remove this hack by using a different channel + del subscription_dict["ytdl_options"]["break_on_reject"] return mergedeep.merge( subscription_dict, { @@ -250,9 +255,12 @@ class TestChannelAsKodiTvShow: expected_recent_channel_download.assert_files_exist(relative_directory=output_directory) # try downloading again, ensure nothing more was downloaded - # TODO: add patch around the output of download to see what entry dicts were returned - recent_channel_subscription.download() - expected_recent_channel_download.assert_files_exist(relative_directory=output_directory) + with assert_debug_log( + logger=ytdl_sub.downloaders.downloader.logger, + expected_message="ExistingVideoReached, stopping additional downloads", + ): + recent_channel_subscription.download() + expected_recent_channel_download.assert_files_exist(relative_directory=output_directory) def test_rolling_recent_channel_download( self, @@ -263,18 +271,31 @@ class TestChannelAsKodiTvShow: output_directory, ): # First, download recent vids - recent_channel_subscription.download() - expected_recent_channel_download.assert_files_exist(relative_directory=output_directory) + with assert_debug_log( + logger=ytdl_sub.downloaders.downloader.logger, + expected_message="RejectedVideoReached, stopping additional downloads", + ): + recent_channel_subscription.download() + expected_recent_channel_download.assert_files_exist(relative_directory=output_directory) # Then, download the rolling recent vids subscription. This should remove one of the # two videos - rolling_recent_channel_subscription.download() - expected_rolling_recent_channel_download.assert_files_exist( - relative_directory=output_directory - ) + with assert_debug_log( + logger=ytdl_sub.downloaders.downloader.logger, + expected_message="ExistingVideoReached, stopping additional downloads", + ): + rolling_recent_channel_subscription.download() + expected_rolling_recent_channel_download.assert_files_exist( + relative_directory=output_directory + ) - # Invoke the rolling download again, ensure nothing has changed - rolling_recent_channel_subscription.download() - expected_rolling_recent_channel_download.assert_files_exist( - relative_directory=output_directory - ) + # Invoke the rolling download again, ensure downloading stopped early from it already + # existing + with assert_debug_log( + logger=ytdl_sub.downloaders.downloader.logger, + expected_message="ExistingVideoReached, stopping additional downloads", + ): + rolling_recent_channel_subscription.download() + expected_rolling_recent_channel_download.assert_files_exist( + relative_directory=output_directory + ) diff --git a/tests/e2e/youtube/test_playlist_as_kodi_music_videos.py b/tests/e2e/youtube/test_playlist_as_kodi_music_videos.py index e943cd24..a31e39de 100644 --- a/tests/e2e/youtube/test_playlist_as_kodi_music_videos.py +++ b/tests/e2e/youtube/test_playlist_as_kodi_music_videos.py @@ -2,8 +2,10 @@ from pathlib import Path import mergedeep import pytest +from conftest import assert_debug_log from e2e.expected_download import ExpectedDownload +import ytdl_sub.downloaders.downloader from ytdl_sub.config.config_file import ConfigFile from ytdl_sub.config.preset import Preset from ytdl_sub.subscriptions.subscription import Subscription @@ -138,6 +140,14 @@ class TestPlaylistAsKodiMusicVideo: playlist_subscription.download() expected_playlist_download.assert_files_exist(relative_directory=output_directory) + # After the playlist is downloaded, ensure another invocation will hit ExistingVideoReached + with assert_debug_log( + logger=ytdl_sub.downloaders.downloader.logger, + expected_message="ExistingVideoReached, stopping additional downloads", + ): + playlist_subscription.download() + expected_playlist_download.assert_files_exist(relative_directory=output_directory) + def test_single_video_download( self, single_video_subscription, expected_single_video_download, output_directory ):