From 73be77c500988ac38e688399c505b1ba63faa874 Mon Sep 17 00:00:00 2001 From: Jesse Bannon Date: Sun, 28 Aug 2022 11:09:11 -0700 Subject: [PATCH] [FEATURE] `date_range` plugin, supports Override variables (#199) --- docs/config.rst | 8 + src/ytdl_sub/config/preset_class_mappings.py | 2 + src/ytdl_sub/config/preset_options.py | 14 -- src/ytdl_sub/downloaders/youtube/channel.py | 30 ++- src/ytdl_sub/plugins/date_range.py | 67 ++++++ src/ytdl_sub/subscriptions/subscription.py | 7 +- .../subscription_ytdl_options.py | 16 +- src/ytdl_sub/utils/datetime.py | 56 +++++ .../validators/date_range_validator.py | 38 ---- src/ytdl_sub/validators/string_datetime.py | 22 +- .../validators/string_formatter_validators.py | 1 - tests/e2e/plugins/test_date_range.py | 201 +++++++++++++++++ .../date_range/no_downloads.json} | 2 +- .../date_range}/test_channel_recent.json | 2 +- .../test_channel_rolling_recent.json | 2 +- .../date_range/no_downloads.txt} | 2 +- .../date_range}/test_channel_recent.txt | 2 +- .../test_channel_rolling_recent.txt | 2 +- tests/e2e/youtube/test_channel.py | 207 +----------------- 19 files changed, 397 insertions(+), 284 deletions(-) create mode 100644 src/ytdl_sub/plugins/date_range.py create mode 100644 src/ytdl_sub/utils/datetime.py delete mode 100644 src/ytdl_sub/validators/date_range_validator.py create mode 100644 tests/e2e/plugins/test_date_range.py rename tests/e2e/resources/expected_downloads_summaries/{youtube/test_channel_no_additional_downloads.json => plugins/date_range/no_downloads.json} (66%) rename tests/e2e/resources/expected_downloads_summaries/{youtube => plugins/date_range}/test_channel_recent.json (92%) rename tests/e2e/resources/expected_downloads_summaries/{youtube => plugins/date_range}/test_channel_rolling_recent.json (88%) rename tests/e2e/resources/transaction_log_summaries/{youtube/test_channel_no_additional_downloads.txt => plugins/date_range/no_downloads.txt} (80%) rename tests/e2e/resources/transaction_log_summaries/{youtube => plugins/date_range}/test_channel_recent.txt (98%) rename tests/e2e/resources/transaction_log_summaries/{youtube => plugins/date_range}/test_channel_rolling_recent.txt (92%) diff --git a/docs/config.rst b/docs/config.rst index 77290024..eba07f73 100644 --- a/docs/config.rst +++ b/docs/config.rst @@ -185,6 +185,14 @@ chapters ------------------------------------------------------------------------------- +date_range +'''''''''' +.. autoclass:: ytdl_sub.plugins.date_range.DateRangeOptions() + :members: + :member-order: bysource + +------------------------------------------------------------------------------- + music_tags '''''''''' .. autoclass:: ytdl_sub.plugins.music_tags.MusicTagsOptions() diff --git a/src/ytdl_sub/config/preset_class_mappings.py b/src/ytdl_sub/config/preset_class_mappings.py index 9554340f..c42b0bf3 100644 --- a/src/ytdl_sub/config/preset_class_mappings.py +++ b/src/ytdl_sub/config/preset_class_mappings.py @@ -11,6 +11,7 @@ from ytdl_sub.downloaders.youtube.split_video import YoutubeSplitVideoDownloader from ytdl_sub.downloaders.youtube.video import YoutubeVideoDownloader from ytdl_sub.plugins.audio_extract import AudioExtractPlugin from ytdl_sub.plugins.chapters import ChaptersPlugin +from ytdl_sub.plugins.date_range import DateRangePlugin from ytdl_sub.plugins.music_tags import MusicTagsPlugin from ytdl_sub.plugins.nfo_tags import NfoTagsPlugin from ytdl_sub.plugins.output_directory_nfo_tags import OutputDirectoryNfoTagsPlugin @@ -111,6 +112,7 @@ class PluginMapping: _MAPPING: Dict[str, Type[Plugin]] = { "audio_extract": AudioExtractPlugin, + "date_range": DateRangePlugin, "music_tags": MusicTagsPlugin, "video_tags": VideoTagsPlugin, "nfo_tags": NfoTagsPlugin, diff --git a/src/ytdl_sub/config/preset_options.py b/src/ytdl_sub/config/preset_options.py index b3bcc53e..7ae03041 100644 --- a/src/ytdl_sub/config/preset_options.py +++ b/src/ytdl_sub/config/preset_options.py @@ -1,7 +1,6 @@ from typing import Dict from typing import Optional -from yt_dlp.utils import DateRange from yt_dlp.utils import sanitize_filename from ytdl_sub.entries.entry import Entry @@ -226,16 +225,3 @@ class OutputOptions(StrictDictValidator): files after ``19000101``, which implies all files. """ return self._keep_files_after - - def get_upload_date_range_to_keep(self) -> Optional[DateRange]: - """ - Returns - ------- - Date range if the 'before' or 'after' is defined. None otherwise. - """ - if self.keep_files_before or self.keep_files_after: - return DateRange( - start=self.keep_files_after.datetime_str if self.keep_files_after else None, - end=self.keep_files_before.datetime_str if self.keep_files_before else None, - ) - return None diff --git a/src/ytdl_sub/downloaders/youtube/channel.py b/src/ytdl_sub/downloaders/youtube/channel.py index f3f93e9e..e248f02e 100644 --- a/src/ytdl_sub/downloaders/youtube/channel.py +++ b/src/ytdl_sub/downloaders/youtube/channel.py @@ -12,14 +12,15 @@ from ytdl_sub.downloaders.youtube.abc import YoutubeDownloaderOptions from ytdl_sub.downloaders.ytdl_options_builder import YTDLOptionsBuilder from ytdl_sub.entries.youtube import YoutubeChannel from ytdl_sub.entries.youtube import YoutubeVideo +from ytdl_sub.utils.datetime import to_date_range_hack from ytdl_sub.utils.thumbnail import convert_url_thumbnail -from ytdl_sub.validators.date_range_validator import DateRangeValidator +from ytdl_sub.validators.string_datetime import StringDatetimeValidator from ytdl_sub.validators.string_formatter_validators import OverridesStringFormatterValidator from ytdl_sub.validators.url_validator import YoutubeChannelUrlValidator from ytdl_sub.ytdl_additions.enhanced_download_archive import EnhancedDownloadArchive -class YoutubeChannelDownloaderOptions(YoutubeDownloaderOptions, DateRangeValidator): +class YoutubeChannelDownloaderOptions(YoutubeDownloaderOptions): """ Downloads all videos from a youtube channel. @@ -49,8 +50,7 @@ class YoutubeChannelDownloaderOptions(YoutubeDownloaderOptions, DateRangeValidat } def __init__(self, name, value): - YoutubeDownloaderOptions.__init__(self, name, value) - DateRangeValidator.__init__(self, name, value) + super().__init__(name, value) self._channel_url = self._validate_key( "channel_url", YoutubeChannelUrlValidator ).channel_url @@ -60,6 +60,8 @@ class YoutubeChannelDownloaderOptions(YoutubeDownloaderOptions, DateRangeValidat self._channel_banner_path = self._validate_key_if_present( "channel_banner_path", OverridesStringFormatterValidator ) + self._before = self._validate_key_if_present("before", StringDatetimeValidator) + self._after = self._validate_key_if_present("after", StringDatetimeValidator) @property def channel_url(self) -> str: @@ -84,6 +86,22 @@ class YoutubeChannelDownloaderOptions(YoutubeDownloaderOptions, DateRangeValidat """ return self._channel_banner_path + @property + def before(self) -> Optional[StringDatetimeValidator]: + """ + DEPRECATED: use the `date_range` plugin instead. Will be removed in version 0.5.0 + Optional. Only download videos before this datetime. + """ + return self._before + + @property + def after(self) -> Optional[StringDatetimeValidator]: + """ + DEPRECATED: use the `date_range` plugin instead. Will be removed in version 0.5.0 + Optional. Only download videos after this datetime. + """ + return self._after + class YoutubeChannelDownloader(YoutubeDownloader[YoutubeChannelDownloaderOptions, YoutubeVideo]): downloader_options_type = YoutubeChannelDownloaderOptions @@ -138,7 +156,9 @@ class YoutubeChannelDownloader(YoutubeDownloader[YoutubeChannelDownloaderOptions ytdl_options_overrides = {} # If a date range is specified when download a YT channel, add it into the ytdl options - source_date_range = self.download_options.get_date_range() + source_date_range = to_date_range_hack( + before=self.download_options.before, after=self.download_options.after + ) if source_date_range: ytdl_options_overrides["daterange"] = source_date_range diff --git a/src/ytdl_sub/plugins/date_range.py b/src/ytdl_sub/plugins/date_range.py new file mode 100644 index 00000000..15b592e0 --- /dev/null +++ b/src/ytdl_sub/plugins/date_range.py @@ -0,0 +1,67 @@ +from typing import Dict +from typing import Optional + +from ytdl_sub.downloaders.ytdl_options_builder import YTDLOptionsBuilder +from ytdl_sub.plugins.plugin import Plugin +from ytdl_sub.plugins.plugin import PluginOptions +from ytdl_sub.utils.datetime import to_date_range +from ytdl_sub.validators.string_datetime import StringDatetimeValidator + + +class DateRangeOptions(PluginOptions): + """ + Only download files uploaded within the specified date range. + + Usage: + + .. code-block:: yaml + + presets: + my_example_preset: + date_range: + before: "now" + after: "today-2weeks" + """ + + _optional_keys = {"before", "after"} + + def __init__(self, name, value): + super().__init__(name, value) + self._before = self._validate_key_if_present("before", StringDatetimeValidator) + self._after = self._validate_key_if_present("after", StringDatetimeValidator) + + @property + def before(self) -> Optional[StringDatetimeValidator]: + """ + Optional. Only download videos before this datetime. + """ + return self._before + + @property + def after(self) -> Optional[StringDatetimeValidator]: + """ + Optional. Only download videos after this datetime. + """ + return self._after + + +class DateRangePlugin(Plugin[DateRangeOptions]): + plugin_options_type = DateRangeOptions + + def ytdl_options(self) -> Optional[Dict]: + """ + Returns + ------- + YTDL options for extracting audio + """ + ytdl_options_builder = YTDLOptionsBuilder() + + source_date_range = to_date_range( + before=self.plugin_options.before, + after=self.plugin_options.after, + overrides=self.overrides, + ) + if source_date_range: + ytdl_options_builder.add({"daterange": source_date_range}) + + return ytdl_options_builder.to_dict() diff --git a/src/ytdl_sub/subscriptions/subscription.py b/src/ytdl_sub/subscriptions/subscription.py index 5683cab1..251d4caf 100644 --- a/src/ytdl_sub/subscriptions/subscription.py +++ b/src/ytdl_sub/subscriptions/subscription.py @@ -19,6 +19,7 @@ from ytdl_sub.downloaders.downloader import DownloaderValidator from ytdl_sub.entries.entry import Entry from ytdl_sub.plugins.plugin import Plugin from ytdl_sub.subscriptions.subscription_ytdl_options import SubscriptionYTDLOptions +from ytdl_sub.utils.datetime import to_date_range from ytdl_sub.utils.exceptions import ValidationException from ytdl_sub.utils.file_handler import FileHandlerTransactionLog from ytdl_sub.utils.file_handler import FileMetadata @@ -230,7 +231,11 @@ class Subscription: # If output options maintains stale file deletion, perform the delete here prior to saving # the download archive if self.maintain_download_archive: - date_range_to_keep = self.output_options.get_upload_date_range_to_keep() + date_range_to_keep = to_date_range( + before=self.output_options.keep_files_before, + after=self.output_options.keep_files_after, + overrides=self.overrides, + ) if date_range_to_keep: self._enhanced_download_archive.remove_stale_files(date_range=date_range_to_keep) diff --git a/src/ytdl_sub/subscriptions/subscription_ytdl_options.py b/src/ytdl_sub/subscriptions/subscription_ytdl_options.py index 64833e22..b2f49919 100644 --- a/src/ytdl_sub/subscriptions/subscription_ytdl_options.py +++ b/src/ytdl_sub/subscriptions/subscription_ytdl_options.py @@ -10,6 +10,7 @@ from ytdl_sub.downloaders.downloader import Downloader from ytdl_sub.downloaders.ytdl_options_builder import YTDLOptionsBuilder from ytdl_sub.plugins.audio_extract import AudioExtractPlugin from ytdl_sub.plugins.chapters import ChaptersPlugin +from ytdl_sub.plugins.date_range import DateRangePlugin from ytdl_sub.plugins.plugin import Plugin from ytdl_sub.plugins.subtitles import SubtitlesPlugin from ytdl_sub.ytdl_additions.enhanced_download_archive import EnhancedDownloadArchive @@ -111,6 +112,13 @@ class SubscriptionYTDLOptions: return chapters_plugin.ytdl_options() + @property + def _date_range_options(self) -> Dict: + if not (date_range_plugin := self._get_plugin(DateRangePlugin)): + return {} + + return date_range_plugin.ytdl_options() + @property def _user_ytdl_options(self) -> Dict: return self._preset.ytdl_options.dict @@ -125,18 +133,20 @@ class SubscriptionYTDLOptions: ytdl_options_builder = YTDLOptionsBuilder().add(self._global_options) if self._dry_run: ytdl_options_builder.add( + self._date_range_options, self._subtitle_options, self._chapter_options, - self._user_ytdl_options, - self._dry_run_options, + self._user_ytdl_options, # user ytdl options... + self._dry_run_options, # then dry-run ) else: ytdl_options_builder.add( self._output_options, + self._date_range_options, self._subtitle_options, self._chapter_options, self._audio_extract_options, - self._user_ytdl_options, + self._user_ytdl_options, # user ytdl options last ) return ytdl_options_builder diff --git a/src/ytdl_sub/utils/datetime.py b/src/ytdl_sub/utils/datetime.py new file mode 100644 index 00000000..3231461a --- /dev/null +++ b/src/ytdl_sub/utils/datetime.py @@ -0,0 +1,56 @@ +from typing import Optional + +from yt_dlp import DateRange + +from ytdl_sub.config.preset_options import Overrides +from ytdl_sub.validators.string_datetime import StringDatetimeValidator + + +def to_date_range_hack( + before: Optional[StringDatetimeValidator], after: Optional[StringDatetimeValidator] +) -> Optional[DateRange]: + """ + Workaround for channel before/after support. + + Returns + ------- + Date range if the 'before' or 'after' is defined. None otherwise. + """ + start: Optional[str] = None + end: Optional[str] = None + + if after: + start = after.apply_formatter(variable_dict={}) + + if before: + end = before.apply_formatter(variable_dict={}) + + if start or end: + return DateRange(start=start, end=end) + + return None + + +def to_date_range( + before: Optional[StringDatetimeValidator], + after: Optional[StringDatetimeValidator], + overrides: Overrides, +) -> Optional[DateRange]: + """ + Returns + ------- + Date range if the 'before' or 'after' is defined. None otherwise. + """ + start: Optional[str] = None + end: Optional[str] = None + + if after: + start = overrides.apply_formatter(formatter=after) + + if before: + end = overrides.apply_formatter(formatter=before) + + if start or end: + return DateRange(start=start, end=end) + + return None diff --git a/src/ytdl_sub/validators/date_range_validator.py b/src/ytdl_sub/validators/date_range_validator.py deleted file mode 100644 index aa0268bd..00000000 --- a/src/ytdl_sub/validators/date_range_validator.py +++ /dev/null @@ -1,38 +0,0 @@ -from typing import Optional - -from yt_dlp import DateRange - -from ytdl_sub.validators.strict_dict_validator import StrictDictValidator -from ytdl_sub.validators.string_datetime import StringDatetimeValidator - - -class DateRangeValidator(StrictDictValidator): - _optional_keys = {"before", "after"} - - def __init__(self, name, value): - super().__init__(name, value) - self._before = self._validate_key_if_present("before", StringDatetimeValidator) - self._after = self._validate_key_if_present("after", StringDatetimeValidator) - - @property - def before(self) -> Optional[StringDatetimeValidator]: - """Optional. Only download videos before this datetime.""" - return self._before - - @property - def after(self) -> Optional[StringDatetimeValidator]: - """Optional. Only download videos after this datetime.""" - return self._after - - def get_date_range(self) -> Optional[DateRange]: - """ - Returns - ------- - Date range if the 'before' or 'after' is defined. None otherwise. - """ - if self._before or self._after: - return DateRange( - start=self._after.datetime_str if self._after else None, - end=self._before.datetime_str if self._before else None, - ) - return None diff --git a/src/ytdl_sub/validators/string_datetime.py b/src/ytdl_sub/validators/string_datetime.py index 5e85c581..69f6bb95 100644 --- a/src/ytdl_sub/validators/string_datetime.py +++ b/src/ytdl_sub/validators/string_datetime.py @@ -1,9 +1,11 @@ +from typing import Dict + from yt_dlp.utils import datetime_from_str -from ytdl_sub.validators.validators import Validator +from ytdl_sub.validators.string_formatter_validators import OverridesStringFormatterValidator -class StringDatetimeValidator(Validator): +class StringDatetimeValidator(OverridesStringFormatterValidator): """ String that contains a yt-dlp datetime. From their docs: @@ -12,21 +14,15 @@ class StringDatetimeValidator(Validator): A string in the format YYYYMMDD or (now|today|yesterday|date)[+-][0-9](microsecond|second|minute|hour|day|week|month|year)(s) - Valid examples are ``now-2weeks`` or ``20200101``. + Valid examples are ``now-2weeks`` or ``20200101``. Can use override variables in this. """ - _expected_value_type = str _expected_value_type_name = "datetime string" - def __init__(self, name, value): - super().__init__(name, value) - + def apply_formatter(self, variable_dict: Dict[str, str]) -> str: + output = super().apply_formatter(variable_dict) try: _ = datetime_from_str(self._value) except Exception as exc: - raise self._validation_exception(str(exc)) - - @property - def datetime_str(self) -> str: - """Returns the datetime as a string""" - return self._value + raise self._validation_exception(f"Invalid datetime string: {str(exc)}") + return output diff --git a/src/ytdl_sub/validators/string_formatter_validators.py b/src/ytdl_sub/validators/string_formatter_validators.py index 9a590e7f..97c61a84 100644 --- a/src/ytdl_sub/validators/string_formatter_validators.py +++ b/src/ytdl_sub/validators/string_formatter_validators.py @@ -151,7 +151,6 @@ class StringFormatterValidator(Validator): value=formatter.format_string.format(**OrderedDict(variable_dict)), ) - @final def apply_formatter(self, variable_dict: Dict[str, str]) -> str: """ Calls `format` on the format string using the variable_dict as input kwargs diff --git a/tests/e2e/plugins/test_date_range.py b/tests/e2e/plugins/test_date_range.py new file mode 100644 index 00000000..d4080c4c --- /dev/null +++ b/tests/e2e/plugins/test_date_range.py @@ -0,0 +1,201 @@ +import copy + +import mergedeep +import pytest +from conftest import assert_debug_log +from e2e.expected_download import assert_expected_downloads +from e2e.expected_transaction_log import assert_transaction_log_matches + +import ytdl_sub.downloaders.downloader +from ytdl_sub.subscriptions.subscription import Subscription + + +@pytest.fixture +def recent_preset_dict(output_directory): + return { + "preset": "yt_channel_as_tv", + "youtube": {"channel_url": "https://youtube.com/channel/UCcRSMoQqXc_JrBZRHDFGbqA"}, + "date_range": {"after": "20150101"}, + # override the output directory with our fixture-generated dir + "output_options": {"output_directory": output_directory}, + # download the worst format so it is fast + "ytdl_options": { + "format": "worst[ext=mp4]", + "max_views": 100000, # do not download the popular PJ concert + }, + "subtitles": { + "subtitles_name": "{episode_name}.{lang}.{subtitles_ext}", + "allow_auto_generated_subtitles": True, + }, + "overrides": {"tv_show_name": "Project / Zombie"}, + } + + +@pytest.fixture +def rolling_recent_channel_preset_dict(recent_preset_dict): + preset_dict = copy.deepcopy(recent_preset_dict) + return mergedeep.merge( + preset_dict, + { + "output_options": {"keep_files_after": "20181101"}, + }, + ) + + +class TestDateRange: + @pytest.mark.parametrize("dry_run", [True, False]) + def test_recent_channel_download( + self, + recent_preset_dict, + channel_as_tv_show_config, + output_directory, + dry_run, + ): + recent_channel_subscription = Subscription.from_dict( + config=channel_as_tv_show_config, + preset_name="recent", + preset_dict=recent_preset_dict, + ) + + transaction_log = recent_channel_subscription.download(dry_run=dry_run) + assert_transaction_log_matches( + output_directory=output_directory, + transaction_log=transaction_log, + transaction_log_summary_file_name="plugins/date_range/test_channel_recent.txt", + ) + assert_expected_downloads( + output_directory=output_directory, + dry_run=dry_run, + expected_download_summary_file_name="plugins/date_range/test_channel_recent.json", + ) + if not dry_run: + # try downloading again, ensure nothing more was downloaded + with assert_debug_log( + logger=ytdl_sub.downloaders.downloader.download_logger, + expected_message="ExistingVideoReached, stopping additional downloads", + ): + transaction_log = recent_channel_subscription.download() + assert_transaction_log_matches( + output_directory=output_directory, + transaction_log=transaction_log, + transaction_log_summary_file_name=("plugins/date_range/no_downloads.txt"), + ) + assert_expected_downloads( + output_directory=output_directory, + dry_run=dry_run, + expected_download_summary_file_name="plugins/date_range/test_channel_recent.json", + ) + + @pytest.mark.parametrize("dry_run", [True, False]) + def test_recent_channel_download__no_vids_in_range( + self, + channel_as_tv_show_config, + recent_preset_dict, + output_directory, + dry_run, + ): + recent_preset_dict["date_range"]["after"] = "21000101" + + recent_channel_no_vids_in_range_subscription = Subscription.from_dict( + config=channel_as_tv_show_config, + preset_name="recent", + preset_dict=recent_preset_dict, + ) + + # Run twice, ensure nothing changes between runsyoutube + for _ in range(2): + transaction_log = recent_channel_no_vids_in_range_subscription.download(dry_run=dry_run) + assert_transaction_log_matches( + output_directory=output_directory, + transaction_log=transaction_log, + transaction_log_summary_file_name="plugins/date_range/no_downloads.txt", + ) + assert_expected_downloads( + output_directory=output_directory, + dry_run=dry_run, + expected_download_summary_file_name="plugins/date_range/no_downloads.json", + ) + + @pytest.mark.parametrize("dry_run", [True, False]) + def test_rolling_recent_channel_download( + self, + channel_as_tv_show_config, + recent_preset_dict, + rolling_recent_channel_preset_dict, + output_directory, + dry_run, + ): + recent_channel_subscription = Subscription.from_dict( + config=channel_as_tv_show_config, + preset_name="recent", + preset_dict=recent_preset_dict, + ) + rolling_recent_channel_subscription = Subscription.from_dict( + config=channel_as_tv_show_config, + preset_name="recent", + preset_dict=rolling_recent_channel_preset_dict, + ) + + # First, download recent vids. Always download since we want to test dry-run + # on the rolling recent portion. + with assert_debug_log( + logger=ytdl_sub.downloaders.downloader.download_logger, + expected_message="RejectedVideoReached, stopping additional downloads", + ): + transaction_log = recent_channel_subscription.download(dry_run=False) + + assert_transaction_log_matches( + output_directory=output_directory, + transaction_log=transaction_log, + transaction_log_summary_file_name="plugins/date_range/test_channel_recent.txt", + ) + assert_expected_downloads( + output_directory=output_directory, + dry_run=False, + expected_download_summary_file_name="plugins/date_range/test_channel_recent.json", + ) + + # Then, download the rolling recent vids subscription. This should remove one of the + # two videos + with assert_debug_log( + logger=ytdl_sub.downloaders.downloader.download_logger, + expected_message="ExistingVideoReached, stopping additional downloads", + ): + transaction_log = rolling_recent_channel_subscription.download(dry_run=dry_run) + + expected_downloads_summary = ( + "plugins/date_range/test_channel_recent.json" + if dry_run + else "plugins/date_range/test_channel_rolling_recent.json" + ) + + assert_transaction_log_matches( + output_directory=output_directory, + transaction_log=transaction_log, + transaction_log_summary_file_name="plugins/date_range/test_channel_rolling_recent.txt", + ) + assert_expected_downloads( + output_directory=output_directory, + dry_run=False, + expected_download_summary_file_name=expected_downloads_summary, + ) + + # Invoke the rolling download again, ensure downloading stopped early from it already + # existing + if not dry_run: + with assert_debug_log( + logger=ytdl_sub.downloaders.downloader.download_logger, + expected_message="ExistingVideoReached, stopping additional downloads", + ): + transaction_log = rolling_recent_channel_subscription.download() + + assert_transaction_log_matches( + output_directory=output_directory, + transaction_log=transaction_log, + transaction_log_summary_file_name="plugins/date_range/no_downloads.txt", + ) + assert_expected_downloads( + output_directory=output_directory, + dry_run=False, + expected_download_summary_file_name=expected_downloads_summary, + ) diff --git a/tests/e2e/resources/expected_downloads_summaries/youtube/test_channel_no_additional_downloads.json b/tests/e2e/resources/expected_downloads_summaries/plugins/date_range/no_downloads.json similarity index 66% rename from tests/e2e/resources/expected_downloads_summaries/youtube/test_channel_no_additional_downloads.json rename to tests/e2e/resources/expected_downloads_summaries/plugins/date_range/no_downloads.json index 34b1ae5e..2f5b0143 100644 --- a/tests/e2e/resources/expected_downloads_summaries/youtube/test_channel_no_additional_downloads.json +++ b/tests/e2e/resources/expected_downloads_summaries/plugins/date_range/no_downloads.json @@ -1,5 +1,5 @@ { - ".ytdl-sub-pz-download-archive.json": "99914b932bd37a50b983c5e7c90ae93b", + ".ytdl-sub-recent-download-archive.json": "99914b932bd37a50b983c5e7c90ae93b", "fanart.jpg": "129c6639b47299bc48062f0365e670ee", "poster.jpg": "5de28eea5a921a041452ab3ce1041f73", "tvshow.nfo": "85cef9db54e9afb7af8e4912b5262d3f" diff --git a/tests/e2e/resources/expected_downloads_summaries/youtube/test_channel_recent.json b/tests/e2e/resources/expected_downloads_summaries/plugins/date_range/test_channel_recent.json similarity index 92% rename from tests/e2e/resources/expected_downloads_summaries/youtube/test_channel_recent.json rename to tests/e2e/resources/expected_downloads_summaries/plugins/date_range/test_channel_recent.json index f2f77d53..8890ede2 100644 --- a/tests/e2e/resources/expected_downloads_summaries/youtube/test_channel_recent.json +++ b/tests/e2e/resources/expected_downloads_summaries/plugins/date_range/test_channel_recent.json @@ -1,5 +1,5 @@ { - ".ytdl-sub-pz-download-archive.json": "756b60d7a6c47d8163e3283404493a8d", + ".ytdl-sub-recent-download-archive.json": "756b60d7a6c47d8163e3283404493a8d", "Season 2018/s2018.e1029 - Jesse's Minecraft Server ｜ Teaser Trailer-thumb.jpg": "705ca4e0d99b37e9ecdf6bfe4b90c59b", "Season 2018/s2018.e1029 - Jesse's Minecraft Server ｜ Teaser Trailer.mp4": "82f6ee7253e1dbb83ae7215af08ffacc", "Season 2018/s2018.e1029 - Jesse's Minecraft Server ｜ Teaser Trailer.nfo": "368d68db0cbe9eb4f43ece0517445e82", diff --git a/tests/e2e/resources/expected_downloads_summaries/youtube/test_channel_rolling_recent.json b/tests/e2e/resources/expected_downloads_summaries/plugins/date_range/test_channel_rolling_recent.json similarity index 88% rename from tests/e2e/resources/expected_downloads_summaries/youtube/test_channel_rolling_recent.json rename to tests/e2e/resources/expected_downloads_summaries/plugins/date_range/test_channel_rolling_recent.json index b8772653..9e2d28bf 100644 --- a/tests/e2e/resources/expected_downloads_summaries/youtube/test_channel_rolling_recent.json +++ b/tests/e2e/resources/expected_downloads_summaries/plugins/date_range/test_channel_rolling_recent.json @@ -1,5 +1,5 @@ { - ".ytdl-sub-pz-download-archive.json": "68e164a35d1541ce761a6d7fef19ee08", + ".ytdl-sub-recent-download-archive.json": "68e164a35d1541ce761a6d7fef19ee08", "Season 2018/s2018.e1102 - Jesse's Minecraft Server ｜ IP mc.jesse.id-thumb.jpg": "28d852ede73b879b9ebf9a061cfc7d46", "Season 2018/s2018.e1102 - Jesse's Minecraft Server ｜ IP mc.jesse.id.en.srt": "3d2c4e7f65d2ca5e96da38ce7eecfc4e", "Season 2018/s2018.e1102 - Jesse's Minecraft Server ｜ IP mc.jesse.id.mp4": "e733b4cc385b953b08c8eb0f47e03c1e", diff --git a/tests/e2e/resources/transaction_log_summaries/youtube/test_channel_no_additional_downloads.txt b/tests/e2e/resources/transaction_log_summaries/plugins/date_range/no_downloads.txt similarity index 80% rename from tests/e2e/resources/transaction_log_summaries/youtube/test_channel_no_additional_downloads.txt rename to tests/e2e/resources/transaction_log_summaries/plugins/date_range/no_downloads.txt index 7275663d..8b9b508b 100644 --- a/tests/e2e/resources/transaction_log_summaries/youtube/test_channel_no_additional_downloads.txt +++ b/tests/e2e/resources/transaction_log_summaries/plugins/date_range/no_downloads.txt @@ -1,6 +1,6 @@ Files created in '{output_directory}' ---------------------------------------- -.ytdl-sub-pz-download-archive.json +.ytdl-sub-recent-download-archive.json fanart.jpg poster.jpg tvshow.nfo diff --git a/tests/e2e/resources/transaction_log_summaries/youtube/test_channel_recent.txt b/tests/e2e/resources/transaction_log_summaries/plugins/date_range/test_channel_recent.txt similarity index 98% rename from tests/e2e/resources/transaction_log_summaries/youtube/test_channel_recent.txt rename to tests/e2e/resources/transaction_log_summaries/plugins/date_range/test_channel_recent.txt index e1cc52b0..248f1b9c 100644 --- a/tests/e2e/resources/transaction_log_summaries/youtube/test_channel_recent.txt +++ b/tests/e2e/resources/transaction_log_summaries/plugins/date_range/test_channel_recent.txt @@ -1,6 +1,6 @@ Files created in '{output_directory}' ---------------------------------------- -.ytdl-sub-pz-download-archive.json +.ytdl-sub-recent-download-archive.json Season 2018/s2018.e1029 - Jesse's Minecraft Server ｜ Teaser Trailer-thumb.jpg Season 2018/s2018.e1029 - Jesse's Minecraft Server ｜ Teaser Trailer.mp4 Season 2018/s2018.e1029 - Jesse's Minecraft Server ｜ Teaser Trailer.nfo diff --git a/tests/e2e/resources/transaction_log_summaries/youtube/test_channel_rolling_recent.txt b/tests/e2e/resources/transaction_log_summaries/plugins/date_range/test_channel_rolling_recent.txt similarity index 92% rename from tests/e2e/resources/transaction_log_summaries/youtube/test_channel_rolling_recent.txt rename to tests/e2e/resources/transaction_log_summaries/plugins/date_range/test_channel_rolling_recent.txt index fefdde78..2f549058 100644 --- a/tests/e2e/resources/transaction_log_summaries/youtube/test_channel_rolling_recent.txt +++ b/tests/e2e/resources/transaction_log_summaries/plugins/date_range/test_channel_rolling_recent.txt @@ -1,6 +1,6 @@ Files created in '{output_directory}' ---------------------------------------- -.ytdl-sub-pz-download-archive.json +.ytdl-sub-recent-download-archive.json fanart.jpg poster.jpg tvshow.nfo diff --git a/tests/e2e/youtube/test_channel.py b/tests/e2e/youtube/test_channel.py index c7ff1219..5eab30fc 100644 --- a/tests/e2e/youtube/test_channel.py +++ b/tests/e2e/youtube/test_channel.py @@ -1,21 +1,10 @@ -import copy -from typing import Dict - -import mergedeep import pytest -from conftest import assert_debug_log from e2e.expected_download import assert_expected_downloads from e2e.expected_transaction_log import assert_transaction_log_matches -import ytdl_sub.downloaders.downloader from ytdl_sub.subscriptions.subscription import Subscription -@pytest.fixture -def subscription_name(): - return "pz" - - @pytest.fixture def channel_preset_dict(output_directory): return { @@ -37,46 +26,6 @@ def channel_preset_dict(output_directory): } -@pytest.fixture -def channel_subscription_generator(channel_as_tv_show_config, subscription_name): - def _channel_subscription_generator(preset_dict: Dict): - return Subscription.from_dict( - config=channel_as_tv_show_config, preset_name=subscription_name, preset_dict=preset_dict - ) - - return _channel_subscription_generator - - -#################################################################################################### -# RECENT CHANNEL FIXTURES -@pytest.fixture -def recent_channel_preset_dict(channel_preset_dict): - # TODO: remove this hack by using a different channel - channel_preset_dict = copy.deepcopy(channel_preset_dict) - del channel_preset_dict["ytdl_options"]["break_on_reject"] - return mergedeep.merge( - channel_preset_dict, - { - "preset": "yt_channel_as_tv__recent", - "youtube": {"after": "20150101"}, - }, - ) - - -#################################################################################################### -# ROLLING RECENT CHANNEL FIXTURES -@pytest.fixture -def rolling_recent_channel_preset_dict(recent_channel_preset_dict): - recent_channel_preset_dict = copy.deepcopy(recent_channel_preset_dict) - return mergedeep.merge( - recent_channel_preset_dict, - { - "preset": "yt_channel_as_tv__only_recent", - "output_options": {"keep_files_after": "20181101"}, - }, - ) - - class TestChannelAsKodiTvShow: """ Downloads my old minecraft youtube channel. Ensure the above files exist and have the @@ -86,12 +35,14 @@ class TestChannelAsKodiTvShow: @pytest.mark.parametrize("dry_run", [True, False]) def test_full_channel_download( self, - channel_subscription_generator, + channel_as_tv_show_config, channel_preset_dict, output_directory, dry_run, ): - full_channel_subscription = channel_subscription_generator(preset_dict=channel_preset_dict) + full_channel_subscription = Subscription.from_dict( + config=channel_as_tv_show_config, preset_name="pz", preset_dict=channel_preset_dict + ) transaction_log = full_channel_subscription.download(dry_run=dry_run) assert_transaction_log_matches( output_directory=output_directory, @@ -103,153 +54,3 @@ class TestChannelAsKodiTvShow: dry_run=dry_run, expected_download_summary_file_name="youtube/test_channel_full.json", ) - - @pytest.mark.parametrize("dry_run", [True, False]) - def test_recent_channel_download( - self, - channel_subscription_generator, - recent_channel_preset_dict, - output_directory, - dry_run, - ): - recent_channel_subscription = channel_subscription_generator( - preset_dict=recent_channel_preset_dict - ) - - transaction_log = recent_channel_subscription.download(dry_run=dry_run) - assert_transaction_log_matches( - output_directory=output_directory, - transaction_log=transaction_log, - transaction_log_summary_file_name="youtube/test_channel_recent.txt", - ) - assert_expected_downloads( - output_directory=output_directory, - dry_run=dry_run, - expected_download_summary_file_name="youtube/test_channel_recent.json", - ) - if not dry_run: - # try downloading again, ensure nothing more was downloaded - with assert_debug_log( - logger=ytdl_sub.downloaders.downloader.download_logger, - expected_message="ExistingVideoReached, stopping additional downloads", - ): - transaction_log = recent_channel_subscription.download() - assert_transaction_log_matches( - output_directory=output_directory, - transaction_log=transaction_log, - transaction_log_summary_file_name=( - "youtube/test_channel_no_additional_downloads.txt" - ), - ) - assert_expected_downloads( - output_directory=output_directory, - dry_run=dry_run, - expected_download_summary_file_name="youtube/test_channel_recent.json", - ) - - @pytest.mark.parametrize("dry_run", [True, False]) - def test_recent_channel_download__no_vids_in_range( - self, - channel_subscription_generator, - recent_channel_preset_dict, - output_directory, - dry_run, - ): - recent_channel_preset_dict["youtube"]["after"] = "21000101" - - recent_channel_no_vids_in_range_subscription = channel_subscription_generator( - preset_dict=recent_channel_preset_dict - ) - # Run twice, ensure nothing changes between runs - for _ in range(2): - transaction_log = recent_channel_no_vids_in_range_subscription.download(dry_run=dry_run) - assert_transaction_log_matches( - output_directory=output_directory, - transaction_log=transaction_log, - transaction_log_summary_file_name="youtube/test_channel_no_additional_downloads.txt", - ) - assert_expected_downloads( - output_directory=output_directory, - dry_run=dry_run, - expected_download_summary_file_name="youtube/test_channel_no_additional_downloads.json", - ) - - @pytest.mark.parametrize("dry_run", [True, False]) - def test_rolling_recent_channel_download( - self, - channel_subscription_generator, - recent_channel_preset_dict, - rolling_recent_channel_preset_dict, - output_directory, - dry_run, - ): - recent_channel_subscription = channel_subscription_generator( - preset_dict=recent_channel_preset_dict - ) - rolling_recent_channel_subscription = channel_subscription_generator( - preset_dict=rolling_recent_channel_preset_dict - ) - - # First, download recent vids. Always download since we want to test dry-run - # on the rolling recent portion. - with assert_debug_log( - logger=ytdl_sub.downloaders.downloader.download_logger, - expected_message="RejectedVideoReached, stopping additional downloads", - ): - transaction_log = recent_channel_subscription.download(dry_run=False) - - assert_transaction_log_matches( - output_directory=output_directory, - transaction_log=transaction_log, - transaction_log_summary_file_name="youtube/test_channel_recent.txt", - ) - assert_expected_downloads( - output_directory=output_directory, - dry_run=False, - expected_download_summary_file_name="youtube/test_channel_recent.json", - ) - - # Then, download the rolling recent vids subscription. This should remove one of the - # two videos - with assert_debug_log( - logger=ytdl_sub.downloaders.downloader.download_logger, - expected_message="ExistingVideoReached, stopping additional downloads", - ): - transaction_log = rolling_recent_channel_subscription.download(dry_run=dry_run) - - expected_downloads_summary = ( - "youtube/test_channel_recent.json" - if dry_run - else "youtube/test_channel_rolling_recent.json" - ) - - assert_transaction_log_matches( - output_directory=output_directory, - transaction_log=transaction_log, - transaction_log_summary_file_name="youtube/test_channel_rolling_recent.txt", - ) - assert_expected_downloads( - output_directory=output_directory, - dry_run=False, - expected_download_summary_file_name=expected_downloads_summary, - ) - - # Invoke the rolling download again, ensure downloading stopped early from it already - # existing - if not dry_run: - with assert_debug_log( - logger=ytdl_sub.downloaders.downloader.download_logger, - expected_message="ExistingVideoReached, stopping additional downloads", - ): - transaction_log = rolling_recent_channel_subscription.download() - - assert_transaction_log_matches( - output_directory=output_directory, - transaction_log=transaction_log, - transaction_log_summary_file_name="youtube/test_channel_no_additional_downloads.txt", - ) - assert_expected_downloads( - output_directory=output_directory, - dry_run=False, - expected_download_summary_file_name=expected_downloads_summary, - )