[FEATURE] release_date support for date_range (#1231)

Adds the ability to toggle whether to use `upload_date` or `release_date` in the date_range plugin, which is used to cull older videos.

```
date_range:
  type: "upload_date"
```

If using the Only Recent preset, it will be automatically applied based on whether your season/episode ordering uses release date or upload date.

Closes https://github.com/jmbannon/ytdl-sub/issues/1182

docs/source/config_reference/plugins.rst

@@ -151,6 +151,8 @@ granularity possible.
    date_range:
      before: "now"
      after: "today-2weeks"
+     breaks: True
+     type: "upload_date"
 
 ``after``
 
@@ -183,6 +185,14 @@ granularity possible.
   is enabled or not via Boolean.
 
 
+``type``
+
+:expected type: Optional[OverridesFormatter]
+:description:
+  Which type of date to use. Must be either ``upload_date`` or ``release_date``.
+  Defaults to ``upload_date``.
+
+
 ----------------------------------------------------------------------------------------------------
 
 download

src/ytdl_sub/plugins/date_range.py

@@ -1,5 +1,6 @@
 from typing import List
 from typing import Optional
+from typing import Set
 from typing import Tuple
 
 from ytdl_sub.config.plugin.plugin import Plugin
@@ -7,6 +8,11 @@ from ytdl_sub.config.validators.options import ToggleableOptionsDictValidator
 from ytdl_sub.utils.datetime import to_date_str
 from ytdl_sub.validators.string_datetime import StringDatetimeValidator
 from ytdl_sub.validators.string_formatter_validators import OverridesBooleanFormatterValidator
+from ytdl_sub.validators.string_select_validator import OverridesStringSelectValidator
+
+
+class DateRangeType(OverridesStringSelectValidator):
+    _select_values: Set[str] = {"upload_date", "release_date"}
 
 
 class DateRangeOptions(ToggleableOptionsDictValidator):
@@ -30,9 +36,11 @@ class DateRangeOptions(ToggleableOptionsDictValidator):
        date_range:
          before: "now"
          after: "today-2weeks"
+         breaks: True
+         type: "upload_date"
     """
 
-    _optional_keys = {"enable", "before", "after", "breaks"}
+    _optional_keys = {"enable", "before", "after", "breaks", "type"}
 
     def __init__(self, name, value):
         super().__init__(name, value)
@@ -41,6 +49,7 @@ class DateRangeOptions(ToggleableOptionsDictValidator):
         self._breaks = self._validate_key_if_present(
             "breaks", OverridesBooleanFormatterValidator, default="True"
         )
+        self._type = self._validate_key("type", DateRangeType, default="upload_date")
 
     @property
     def before(self) -> Optional[StringDatetimeValidator]:
@@ -70,6 +79,16 @@ class DateRangeOptions(ToggleableOptionsDictValidator):
         """
         return self._breaks
 
+    @property
+    def type(self) -> DateRangeType:
+        """
+        :expected type: Optional[OverridesFormatter]
+        :description:
+          Which type of date to use. Must be either ``upload_date`` or ``release_date``.
+          Defaults to ``upload_date``.
+        """
+        return self._type
+
 
 class DateRangePlugin(Plugin[DateRangeOptions]):
     plugin_options_type = DateRangeOptions
@@ -83,17 +102,18 @@ class DateRangePlugin(Plugin[DateRangeOptions]):
         match_filters: List[str] = []
         breaking_match_filters: List[str] = []
 
+        date_type: str = self.overrides.apply_formatter(formatter=self.plugin_options.type)
         if self.plugin_options.before:
             before_str = to_date_str(
                 date_validator=self.plugin_options.before, overrides=self.overrides
             )
-            match_filters.append(f"upload_date < {before_str}")
+            match_filters.append(f"{date_type} < {before_str}")
 
         if self.plugin_options.after:
             after_str = to_date_str(
                 date_validator=self.plugin_options.after, overrides=self.overrides
             )
-            after_filter = f"upload_date >= {after_str}"
+            after_filter = f"{date_type} >= {after_str}"
             if self.overrides.evaluate_boolean(self.plugin_options.breaks):
                 breaking_match_filters.append(after_filter)
             else:

src/ytdl_sub/prebuilt_presets/tv_show/episode.yaml

@@ -17,6 +17,9 @@ presets:
     chapters:
       embed_chapters: True
 
+    date_range:
+      type: "{tv_show_date_range_type}"
+
     overrides:
 #   MUST DEFINE:
 #     tv_show_directory
@@ -45,3 +48,7 @@ presets:
       episode_file_path: "{season_directory_name_sanitized}/{episode_file_name_sanitized}"
       thumbnail_file_name: "{episode_file_path}-thumb.jpg"
       episode_year: "{%slice(episode_date_standardized, 0, 4)}"
+
+      # Used to determine which date type to use if `Only Recent` preset
+      # is applied
+      tv_show_date_range_type: "upload_date"

src/ytdl_sub/prebuilt_presets/tv_show/tv_show_by_date.yaml

@@ -117,6 +117,15 @@ presets:
           )
         }
 
+      tv_show_date_range_type: >-
+        {
+          %if(
+            %contains(tv_show_by_date_season_ordering, "release"),
+            "release_date",
+            "upload_date"
+          )
+        }
+
       ##################################### season/episode pairing validation
 
       "%ordering_pair_eq": >-

src/ytdl_sub/prebuilt_presets/tv_show/tv_show_collection.yaml

@@ -71,6 +71,16 @@ presets:
           )
         }
 
+      tv_show_date_range_type: >-
+        {
+          %if(
+            %contains(tv_show_collection_episode_ordering, "release"),
+            "release_date",
+            "upload_date"
+          )
+        }
+        
+
   ### LEGACY PRESETS
 
   season_by_collection__episode_by_year_month_day:

src/ytdl_sub/subscriptions/subscription_download.py

@@ -349,6 +349,34 @@ class SubscriptionDownload(BaseSubscription, ABC):
 
         return self.download_archive.get_file_handler_transaction_log()
 
+    def get_ytdl_options(
+        self, plugins: Optional[List[Plugin]], dry_run: bool
+    ) -> SubscriptionYTDLOptions:
+        """
+        Parameters
+        ----------
+        plugins
+            Optional. If not provided, will reinitialize them
+        dry_run
+            Whether its dry run or not
+
+        Returns
+        -------
+        SubscriptionYTDLOptions
+            Both metadata and download ytdl-options
+        """
+        if plugins is None:
+            plugins = self._initialize_plugins()
+
+        return SubscriptionYTDLOptions(
+            preset=self._preset_options,
+            plugins=plugins,
+            enhanced_download_archive=self.download_archive,
+            overrides=self.overrides,
+            working_directory=self.working_directory,
+            dry_run=dry_run,
+        )
+
     def download(self, dry_run: bool = False) -> FileHandlerTransactionLog:
         """
         Performs the subscription download
@@ -368,14 +396,7 @@ class SubscriptionDownload(BaseSubscription, ABC):
             logging.info("Skipping %s", self.name)
             return FileHandlerTransactionLog()
 
-        subscription_ytdl_options = SubscriptionYTDLOptions(
+        subscription_ytdl_options = self.get_ytdl_options(plugins=plugins, dry_run=dry_run)
-            preset=self._preset_options,
-            plugins=plugins,
-            enhanced_download_archive=self.download_archive,
-            overrides=self.overrides,
-            working_directory=self.working_directory,
-            dry_run=dry_run,
-        )
 
         downloader = MultiUrlDownloader(
             options=self.downloader_options,
@@ -420,15 +441,7 @@ class SubscriptionDownload(BaseSubscription, ABC):
         self.download_archive.reinitialize(dry_run=dry_run)
 
         plugins = self._initialize_plugins()
-
+        subscription_ytdl_options = self.get_ytdl_options(plugins=plugins, dry_run=dry_run)
-        subscription_ytdl_options = SubscriptionYTDLOptions(
-            preset=self._preset_options,
-            plugins=plugins,
-            enhanced_download_archive=self.download_archive,
-            overrides=self.overrides,
-            working_directory=self.working_directory,
-            dry_run=dry_run,
-        )
 
         # Re-add the original downloader class' plugins
         plugins.extend(

src/ytdl_sub/validators/string_select_validator.py

@@ -1,5 +1,6 @@
 from typing import Set
 
+from ytdl_sub.validators.string_formatter_validators import OverridesStringFormatterValidator
 from ytdl_sub.validators.validators import StringValidator
 
 
@@ -17,3 +18,17 @@ class StringSelectValidator(StringValidator):
             raise self._validation_exception(
                 f"Must be one of the following values: {', '.join(self._select_values)}"
             )
+
+
+class OverridesStringSelectValidator(OverridesStringFormatterValidator):
+    _expected_value_type_name = "overrides select"
+
+    _select_values: Set[str] = set()
+
+    def post_process(self, resolved: str) -> str:
+        if resolved not in self._select_values:
+            raise self._validation_exception(
+                f"Must be one of the following values: {', '.join(sorted(self._select_values))}"
+            )
+
+        return resolved

tests/conftest.py

@@ -12,6 +12,7 @@ from typing import Callable
 from typing import Dict
 from typing import List
 from typing import Optional
+from typing import Tuple
 from unittest.mock import patch
 
 import pytest
@@ -272,3 +273,27 @@ def mock_run_from_cli(args: str) -> List[Subscription]:
     args_list = ["ytdl-sub"] + shlex.split(args)
     with patch.object(sys, "argv", args_list):
         return main()
+
+
+def get_match_filters(
+    subscription: Subscription, dry_run: bool, download_filters: bool
+) -> Tuple[List[str], List[str]]:
+    """
+    Util function to get match filters from a subscription.
+
+    Returns
+    -------
+    match_filters, breaking_match_filters
+    """
+    options = subscription.get_ytdl_options(plugins=None, dry_run=dry_run)
+    options_dict = (
+        options.download_builder().to_dict()
+        if download_filters
+        else options.metadata_builder().to_dict()
+    )
+    if "match_filter" not in options_dict:
+        return [], []
+
+    match_filter_str = repr(options_dict["match_filter"])
+    out = eval(match_filter_str.split("(", maxsplit=1)[-1].split(")")[0])
+    return out

tests/integration/conftest.py

@@ -1,6 +1,5 @@
 import contextlib
 import os
-import shutil
 from pathlib import Path
 from typing import Callable
 from typing import Dict

tests/integration/plugins/test_file_convert.py

@@ -4,7 +4,6 @@ import pytest
 from expected_download import assert_expected_downloads
 from expected_transaction_log import assert_transaction_log_matches
 
-from ytdl_sub.entries.entry import ytdl_sub_chapters_from_comments
 from ytdl_sub.subscriptions.subscription import Subscription
 
 

tests/unit/plugins/test_date_range.py

@@ -0,0 +1,108 @@
+import re
+from typing import Any
+from typing import Dict
+
+import pytest
+from conftest import get_match_filters
+
+from ytdl_sub.config.config_file import ConfigFile
+from ytdl_sub.subscriptions.subscription import Subscription
+from ytdl_sub.utils.exceptions import ValidationException
+
+
+@pytest.fixture
+def preset_dict(output_directory) -> Dict[str, Any]:
+    return {
+        "download": "https://your.name.here",
+        "output_options": {"output_directory": output_directory, "file_name": "will_error.mp4"},
+    }
+
+
+class TestDateRange:
+    @pytest.mark.parametrize("date_range_type", ["upload_date", "release_date"])
+    def test_date_range_type(
+        self,
+        default_config: ConfigFile,
+        preset_dict: Dict[str, Any],
+        output_directory: str,
+        date_range_type: str,
+    ):
+        preset_dict["date_range"] = {
+            "before": "20250530",
+            "after": "20250510",
+            "type": date_range_type,
+        }
+        sub = Subscription.from_dict(
+            config=default_config,
+            preset_name="test_date_range",
+            preset_dict=preset_dict,
+        )
+
+        metadata_filter, metadata_breaking_filter = get_match_filters(
+            subscription=sub, dry_run=False, download_filters=False
+        )
+        assert metadata_filter == [
+            f"!is_live & !is_upcoming & !post_live & {date_range_type} < 20250530"
+        ]
+        assert metadata_breaking_filter == [f"{date_range_type} >= 20250510"]
+
+        download_filter, download_breaking_filter = get_match_filters(
+            subscription=sub, dry_run=False, download_filters=True
+        )
+        assert not download_filter
+        assert not download_breaking_filter
+
+    def test_date_range_breaks_false(
+        self,
+        default_config: ConfigFile,
+        preset_dict: Dict[str, Any],
+        output_directory: str,
+    ):
+        preset_dict["date_range"] = {
+            "before": "20250530",
+            "after": "20250510",
+            "breaks": False,
+        }
+        sub = Subscription.from_dict(
+            config=default_config,
+            preset_name="test_date_range",
+            preset_dict=preset_dict,
+        )
+
+        metadata_filter, metadata_breaking_filter = get_match_filters(
+            subscription=sub, dry_run=False, download_filters=False
+        )
+        assert metadata_filter == [
+            f"!is_live & !is_upcoming & !post_live & upload_date < 20250530 & upload_date >= 20250510"
+        ]
+        assert not metadata_breaking_filter
+
+        download_filter, download_breaking_filter = get_match_filters(
+            subscription=sub, dry_run=False, download_filters=True
+        )
+        assert not download_filter
+        assert not download_breaking_filter
+
+    def test_date_range_invalid_type(
+        self,
+        default_config: ConfigFile,
+        preset_dict: Dict[str, Any],
+        output_directory: str,
+    ):
+        preset_dict["date_range"] = {
+            "before": "20250530",
+            "after": "20250510",
+            "type": "no",
+        }
+
+        error_msg = (
+            "Validation error in test_date_range.date_range.type: "
+            "Must be one of the following values: release_date, upload_date"
+        )
+
+        with pytest.raises(ValidationException, match=re.escape(error_msg)):
+            Subscription.from_dict(
+                config=default_config,
+                preset_name="test_date_range",
+                preset_dict=preset_dict,
+            ).download(dry_run=False)