[FEATURE] Toggle date_range to break or not (#904)

Adds a the new field `breaking` to the `date_range` plugin, to toggle whether an entry breaks subsequent metadata pulls. This is useful to disable if you are grabbing a playlist that may have videos out of order, but still want to apply a date range to it.

docs/source/config_reference/plugins.rst

@@ -166,6 +166,14 @@ granularity possible.
   Only download videos before this datetime.
 
 
+``breaks``
+
+:expected type: Optional[OverridesFormatter]
+:description:
+  Toggle to enable breaking subsequent metadata downloads if an entry's upload date
+  is out of range. Defaults to True.
+
+
 ``enable``
 
 :expected type: Optional[OverridesFormatter]
@@ -1079,9 +1087,6 @@ for more details.
              # Stop downloading additional metadata/videos if it
              # exists in your download archive
              break_on_existing: True
-             # Stop downloading additional metadata/videos if it
-             # is out of your date range
-             break_on_reject: True
              # Path to your YouTube cookies file to download 18+ restricted content
              cookiefile: "/path/to/cookies/file.txt"
              # Only download this number of videos/audio

src/ytdl_sub/config/preset_options.py

@@ -32,9 +32,6 @@ class YTDLOptions(LiteralDictValidator):
                  # Stop downloading additional metadata/videos if it
                  # exists in your download archive
                  break_on_existing: True
-                 # Stop downloading additional metadata/videos if it
-                 # is out of your date range
-                 break_on_reject: True
                  # Path to your YouTube cookies file to download 18+ restricted content
                  cookiefile: "/path/to/cookies/file.txt"
                  # Only download this number of videos/audio

src/ytdl_sub/downloaders/ytdlp.py

@@ -224,7 +224,7 @@ class YTDLP:
         except RejectedVideoReached:
             cls.logger.debug(
                 "RejectedVideoReached, stopping additional downloads "
-                "(Can be disable by setting `ytdl_options.break_on_reject` to False)."
+                "(Can be disable by setting `date_range.breaking` to False)."
             )
         except ExistingVideoReached:
             cls.logger.debug(

src/ytdl_sub/plugins/date_range.py

@@ -5,7 +5,9 @@ from typing import Tuple
 from ytdl_sub.config.plugin.plugin import Plugin
 from ytdl_sub.config.validators.options import ToggleableOptionsDictValidator
 from ytdl_sub.utils.datetime import to_date_str
+from ytdl_sub.utils.script import ScriptUtils
 from ytdl_sub.validators.string_datetime import StringDatetimeValidator
+from ytdl_sub.validators.string_formatter_validators import OverridesBooleanFormatterValidator
 
 
 class DateRangeOptions(ToggleableOptionsDictValidator):
@@ -31,12 +33,15 @@ class DateRangeOptions(ToggleableOptionsDictValidator):
          after: "today-2weeks"
     """
 
-    _optional_keys = {"enable", "before", "after"}
+    _optional_keys = {"enable", "before", "after", "breaks"}
 
     def __init__(self, name, value):
         super().__init__(name, value)
         self._before = self._validate_key_if_present("before", StringDatetimeValidator)
         self._after = self._validate_key_if_present("after", StringDatetimeValidator)
+        self._breaks = self._validate_key_if_present(
+            "breaks", OverridesBooleanFormatterValidator, default="True"
+        )
 
     @property
     def before(self) -> Optional[StringDatetimeValidator]:
@@ -56,6 +61,16 @@ class DateRangeOptions(ToggleableOptionsDictValidator):
         """
         return self._after
 
+    @property
+    def breaks(self) -> OverridesBooleanFormatterValidator:
+        """
+        :expected type: Optional[OverridesFormatter]
+        :description:
+          Toggle to enable breaking subsequent metadata downloads if an entry's upload date
+          is out of range. Defaults to True.
+        """
+        return self._breaks
+
 
 class DateRangePlugin(Plugin[DateRangeOptions]):
     plugin_options_type = DateRangeOptions
@@ -79,6 +94,12 @@ class DateRangePlugin(Plugin[DateRangeOptions]):
             after_str = to_date_str(
                 date_validator=self.plugin_options.after, overrides=self.overrides
             )
-            breaking_match_filters.append(f"upload_date >= {after_str}")
+            after_filter = f"upload_date >= {after_str}"
+            if ScriptUtils.bool_formatter_output(
+                self.overrides.apply_formatter(self.plugin_options.breaks)
+            ):
+                breaking_match_filters.append(after_filter)
+            else:
+                match_filters.append(after_filter)
 
         return match_filters, breaking_match_filters

src/ytdl_sub/plugins/match_filters.py

@@ -44,14 +44,12 @@ def combine_filters(filters: List[str], to_combine: List[str]) -> List[str]:
     if not filters:
         return copy.deepcopy(to_combine)
 
-    if len(to_combine) > 1:
-        raise ValueError("Match-filters to combine only supports 1 at this time")
-
     output_filters: List[str] = []
-    filter_to_combine: str = to_combine[0]
 
     for match_filter in filters:
-        output_filters.append(f"{match_filter} & {filter_to_combine}")
+        output_filters.append(match_filter)
+        for filter_combine in to_combine:
+            output_filters[-1] += f" & {filter_combine}"
 
     return output_filters
 

src/ytdl_sub/prebuilt_presets/helpers/download_deletion_options.yaml

@@ -30,7 +30,6 @@ presets:
       max_downloads: 20
       playlistreverse: True
       break_on_existing: False
-      break_on_reject: True
 
   "Download in Chunks":
     preset:

src/ytdl_sub/subscriptions/subscription_ytdl_options.py

@@ -84,10 +84,6 @@ class SubscriptionYTDLOptions:
             "extract_flat": "discard",  # do not store info.json in mem since its in file
         }
 
-    @property
-    def _download_only_options(self) -> Dict:
-        return {"break_on_reject": True}
-
     @property
     def _output_options(self) -> Dict:
         ytdl_options = {}
@@ -199,7 +195,6 @@ class SubscriptionYTDLOptions:
             self._plugin_ytdl_options(FormatPlugin),
             self._plugin_ytdl_options(AudioExtractPlugin),  # will override format
             self._user_ytdl_options,  # user ytdl options...
-            self._download_only_options,  # then download_only options
         )
         # Add dry run options last if enabled
         if self._dry_run:

tests/e2e/plugins/test_date_range.py

@@ -40,20 +40,30 @@ def rolling_recent_channel_preset_dict(recent_preset_dict):
 
 class TestDateRange:
     @pytest.mark.parametrize("dry_run", [True, False])
+    @pytest.mark.parametrize("date_range_breaks", [True, False])
     def test_recent_channel_download(
         self,
         recent_preset_dict,
         tv_show_config,
         output_directory,
-        dry_run,
+        dry_run: bool,
+        date_range_breaks: bool,
     ):
+        recent_preset_dict["date_range"]["breaks"] = date_range_breaks
         recent_channel_subscription = Subscription.from_dict(
             config=tv_show_config,
             preset_name="recent",
             preset_dict=recent_preset_dict,
         )
 
-        transaction_log = recent_channel_subscription.download(dry_run=dry_run)
+        with assert_logs(
+            logger=YTDLP.logger,
+            expected_message="RejectedVideoReached, stopping additional downloads",
+            log_level="debug",
+            expected_occurrences=1 if date_range_breaks else 0,
+        ):
+            transaction_log = recent_channel_subscription.download(dry_run=dry_run)
+
         assert_transaction_log_matches(
             output_directory=output_directory,
             transaction_log=transaction_log,

tests/e2e/youtube/test_channel.py

@@ -18,7 +18,6 @@ def channel_preset_dict(output_directory):
         "format": "worst[ext=mp4]",  # download the worst format so it is fast
         "ytdl_options": {
             "max_views": 100000,  # do not download the popular PJ concert
-            "break_on_reject": False,  # do not break from max views
         },
         "subtitles": {
             "subtitles_name": "{episode_file_path}.{lang}.{subtitles_ext}",