From f0b3991e3e318cb7a6ee019bbcd02e00285f427c Mon Sep 17 00:00:00 2001 From: Jesse Bannon Date: Thu, 18 Jan 2024 16:23:02 -0800 Subject: [PATCH] [FEATURE] Toggle `date_range` to break or not (#904) Adds a the new field `breaking` to the `date_range` plugin, to toggle whether an entry breaks subsequent metadata pulls. This is useful to disable if you are grabbing a playlist that may have videos out of order, but still want to apply a date range to it. --- docs/source/config_reference/plugins.rst | 11 +++++--- src/ytdl_sub/config/preset_options.py | 3 --- src/ytdl_sub/downloaders/ytdlp.py | 2 +- src/ytdl_sub/plugins/date_range.py | 25 +++++++++++++++++-- src/ytdl_sub/plugins/match_filters.py | 8 +++--- .../helpers/download_deletion_options.yaml | 1 - .../subscription_ytdl_options.py | 5 ---- tests/e2e/plugins/test_date_range.py | 14 +++++++++-- tests/e2e/youtube/test_channel.py | 1 - 9 files changed, 47 insertions(+), 23 deletions(-) diff --git a/docs/source/config_reference/plugins.rst b/docs/source/config_reference/plugins.rst index d8535352..6aa05162 100644 --- a/docs/source/config_reference/plugins.rst +++ b/docs/source/config_reference/plugins.rst @@ -166,6 +166,14 @@ granularity possible. Only download videos before this datetime. +``breaks`` + +:expected type: Optional[OverridesFormatter] +:description: + Toggle to enable breaking subsequent metadata downloads if an entry's upload date + is out of range. Defaults to True. + + ``enable`` :expected type: Optional[OverridesFormatter] @@ -1079,9 +1087,6 @@ for more details. # Stop downloading additional metadata/videos if it # exists in your download archive break_on_existing: True - # Stop downloading additional metadata/videos if it - # is out of your date range - break_on_reject: True # Path to your YouTube cookies file to download 18+ restricted content cookiefile: "/path/to/cookies/file.txt" # Only download this number of videos/audio diff --git a/src/ytdl_sub/config/preset_options.py b/src/ytdl_sub/config/preset_options.py index 8cc4697c..a6737a64 100644 --- a/src/ytdl_sub/config/preset_options.py +++ b/src/ytdl_sub/config/preset_options.py @@ -32,9 +32,6 @@ class YTDLOptions(LiteralDictValidator): # Stop downloading additional metadata/videos if it # exists in your download archive break_on_existing: True - # Stop downloading additional metadata/videos if it - # is out of your date range - break_on_reject: True # Path to your YouTube cookies file to download 18+ restricted content cookiefile: "/path/to/cookies/file.txt" # Only download this number of videos/audio diff --git a/src/ytdl_sub/downloaders/ytdlp.py b/src/ytdl_sub/downloaders/ytdlp.py index 556ef54f..c0ac01e0 100644 --- a/src/ytdl_sub/downloaders/ytdlp.py +++ b/src/ytdl_sub/downloaders/ytdlp.py @@ -224,7 +224,7 @@ class YTDLP: except RejectedVideoReached: cls.logger.debug( "RejectedVideoReached, stopping additional downloads " - "(Can be disable by setting `ytdl_options.break_on_reject` to False)." + "(Can be disable by setting `date_range.breaking` to False)." ) except ExistingVideoReached: cls.logger.debug( diff --git a/src/ytdl_sub/plugins/date_range.py b/src/ytdl_sub/plugins/date_range.py index ff788bff..e01ce1ff 100644 --- a/src/ytdl_sub/plugins/date_range.py +++ b/src/ytdl_sub/plugins/date_range.py @@ -5,7 +5,9 @@ from typing import Tuple from ytdl_sub.config.plugin.plugin import Plugin from ytdl_sub.config.validators.options import ToggleableOptionsDictValidator from ytdl_sub.utils.datetime import to_date_str +from ytdl_sub.utils.script import ScriptUtils from ytdl_sub.validators.string_datetime import StringDatetimeValidator +from ytdl_sub.validators.string_formatter_validators import OverridesBooleanFormatterValidator class DateRangeOptions(ToggleableOptionsDictValidator): @@ -31,12 +33,15 @@ class DateRangeOptions(ToggleableOptionsDictValidator): after: "today-2weeks" """ - _optional_keys = {"enable", "before", "after"} + _optional_keys = {"enable", "before", "after", "breaks"} def __init__(self, name, value): super().__init__(name, value) self._before = self._validate_key_if_present("before", StringDatetimeValidator) self._after = self._validate_key_if_present("after", StringDatetimeValidator) + self._breaks = self._validate_key_if_present( + "breaks", OverridesBooleanFormatterValidator, default="True" + ) @property def before(self) -> Optional[StringDatetimeValidator]: @@ -56,6 +61,16 @@ class DateRangeOptions(ToggleableOptionsDictValidator): """ return self._after + @property + def breaks(self) -> OverridesBooleanFormatterValidator: + """ + :expected type: Optional[OverridesFormatter] + :description: + Toggle to enable breaking subsequent metadata downloads if an entry's upload date + is out of range. Defaults to True. + """ + return self._breaks + class DateRangePlugin(Plugin[DateRangeOptions]): plugin_options_type = DateRangeOptions @@ -79,6 +94,12 @@ class DateRangePlugin(Plugin[DateRangeOptions]): after_str = to_date_str( date_validator=self.plugin_options.after, overrides=self.overrides ) - breaking_match_filters.append(f"upload_date >= {after_str}") + after_filter = f"upload_date >= {after_str}" + if ScriptUtils.bool_formatter_output( + self.overrides.apply_formatter(self.plugin_options.breaks) + ): + breaking_match_filters.append(after_filter) + else: + match_filters.append(after_filter) return match_filters, breaking_match_filters diff --git a/src/ytdl_sub/plugins/match_filters.py b/src/ytdl_sub/plugins/match_filters.py index 9dda20ce..6a777054 100644 --- a/src/ytdl_sub/plugins/match_filters.py +++ b/src/ytdl_sub/plugins/match_filters.py @@ -44,14 +44,12 @@ def combine_filters(filters: List[str], to_combine: List[str]) -> List[str]: if not filters: return copy.deepcopy(to_combine) - if len(to_combine) > 1: - raise ValueError("Match-filters to combine only supports 1 at this time") - output_filters: List[str] = [] - filter_to_combine: str = to_combine[0] for match_filter in filters: - output_filters.append(f"{match_filter} & {filter_to_combine}") + output_filters.append(match_filter) + for filter_combine in to_combine: + output_filters[-1] += f" & {filter_combine}" return output_filters diff --git a/src/ytdl_sub/prebuilt_presets/helpers/download_deletion_options.yaml b/src/ytdl_sub/prebuilt_presets/helpers/download_deletion_options.yaml index cc964d4d..ffb0f596 100644 --- a/src/ytdl_sub/prebuilt_presets/helpers/download_deletion_options.yaml +++ b/src/ytdl_sub/prebuilt_presets/helpers/download_deletion_options.yaml @@ -30,7 +30,6 @@ presets: max_downloads: 20 playlistreverse: True break_on_existing: False - break_on_reject: True "Download in Chunks": preset: diff --git a/src/ytdl_sub/subscriptions/subscription_ytdl_options.py b/src/ytdl_sub/subscriptions/subscription_ytdl_options.py index 4f6292f1..230134db 100644 --- a/src/ytdl_sub/subscriptions/subscription_ytdl_options.py +++ b/src/ytdl_sub/subscriptions/subscription_ytdl_options.py @@ -84,10 +84,6 @@ class SubscriptionYTDLOptions: "extract_flat": "discard", # do not store info.json in mem since its in file } - @property - def _download_only_options(self) -> Dict: - return {"break_on_reject": True} - @property def _output_options(self) -> Dict: ytdl_options = {} @@ -199,7 +195,6 @@ class SubscriptionYTDLOptions: self._plugin_ytdl_options(FormatPlugin), self._plugin_ytdl_options(AudioExtractPlugin), # will override format self._user_ytdl_options, # user ytdl options... - self._download_only_options, # then download_only options ) # Add dry run options last if enabled if self._dry_run: diff --git a/tests/e2e/plugins/test_date_range.py b/tests/e2e/plugins/test_date_range.py index 4196d567..e803fa36 100644 --- a/tests/e2e/plugins/test_date_range.py +++ b/tests/e2e/plugins/test_date_range.py @@ -40,20 +40,30 @@ def rolling_recent_channel_preset_dict(recent_preset_dict): class TestDateRange: @pytest.mark.parametrize("dry_run", [True, False]) + @pytest.mark.parametrize("date_range_breaks", [True, False]) def test_recent_channel_download( self, recent_preset_dict, tv_show_config, output_directory, - dry_run, + dry_run: bool, + date_range_breaks: bool, ): + recent_preset_dict["date_range"]["breaks"] = date_range_breaks recent_channel_subscription = Subscription.from_dict( config=tv_show_config, preset_name="recent", preset_dict=recent_preset_dict, ) - transaction_log = recent_channel_subscription.download(dry_run=dry_run) + with assert_logs( + logger=YTDLP.logger, + expected_message="RejectedVideoReached, stopping additional downloads", + log_level="debug", + expected_occurrences=1 if date_range_breaks else 0, + ): + transaction_log = recent_channel_subscription.download(dry_run=dry_run) + assert_transaction_log_matches( output_directory=output_directory, transaction_log=transaction_log, diff --git a/tests/e2e/youtube/test_channel.py b/tests/e2e/youtube/test_channel.py index 2b36cf9b..4ed9fe20 100644 --- a/tests/e2e/youtube/test_channel.py +++ b/tests/e2e/youtube/test_channel.py @@ -18,7 +18,6 @@ def channel_preset_dict(output_directory): "format": "worst[ext=mp4]", # download the worst format so it is fast "ytdl_options": { "max_views": 100000, # do not download the popular PJ concert - "break_on_reject": False, # do not break from max views }, "subtitles": { "subtitles_name": "{episode_file_path}.{lang}.{subtitles_ext}",