diff --git a/docs/source/config_reference/plugins.rst b/docs/source/config_reference/plugins.rst index 4825f0f7..0eec34ea 100644 --- a/docs/source/config_reference/plugins.rst +++ b/docs/source/config_reference/plugins.rst @@ -151,6 +151,8 @@ granularity possible. date_range: before: "now" after: "today-2weeks" + breaks: True + type: "upload_date" ``after`` @@ -183,6 +185,14 @@ granularity possible. is enabled or not via Boolean. +``type`` + +:expected type: Optional[OverridesFormatter] +:description: + Which type of date to use. Must be either ``upload_date`` or ``release_date``. + Defaults to ``upload_date``. + + ---------------------------------------------------------------------------------------------------- download diff --git a/src/ytdl_sub/plugins/date_range.py b/src/ytdl_sub/plugins/date_range.py index b8b73711..34370a1a 100644 --- a/src/ytdl_sub/plugins/date_range.py +++ b/src/ytdl_sub/plugins/date_range.py @@ -1,5 +1,6 @@ from typing import List from typing import Optional +from typing import Set from typing import Tuple from ytdl_sub.config.plugin.plugin import Plugin @@ -7,6 +8,11 @@ from ytdl_sub.config.validators.options import ToggleableOptionsDictValidator from ytdl_sub.utils.datetime import to_date_str from ytdl_sub.validators.string_datetime import StringDatetimeValidator from ytdl_sub.validators.string_formatter_validators import OverridesBooleanFormatterValidator +from ytdl_sub.validators.string_select_validator import OverridesStringSelectValidator + + +class DateRangeType(OverridesStringSelectValidator): + _select_values: Set[str] = {"upload_date", "release_date"} class DateRangeOptions(ToggleableOptionsDictValidator): @@ -30,9 +36,11 @@ class DateRangeOptions(ToggleableOptionsDictValidator): date_range: before: "now" after: "today-2weeks" + breaks: True + type: "upload_date" """ - _optional_keys = {"enable", "before", "after", "breaks"} + _optional_keys = {"enable", "before", "after", "breaks", "type"} def __init__(self, name, value): super().__init__(name, value) @@ -41,6 +49,7 @@ class DateRangeOptions(ToggleableOptionsDictValidator): self._breaks = self._validate_key_if_present( "breaks", OverridesBooleanFormatterValidator, default="True" ) + self._type = self._validate_key("type", DateRangeType, default="upload_date") @property def before(self) -> Optional[StringDatetimeValidator]: @@ -70,6 +79,16 @@ class DateRangeOptions(ToggleableOptionsDictValidator): """ return self._breaks + @property + def type(self) -> DateRangeType: + """ + :expected type: Optional[OverridesFormatter] + :description: + Which type of date to use. Must be either ``upload_date`` or ``release_date``. + Defaults to ``upload_date``. + """ + return self._type + class DateRangePlugin(Plugin[DateRangeOptions]): plugin_options_type = DateRangeOptions @@ -83,17 +102,18 @@ class DateRangePlugin(Plugin[DateRangeOptions]): match_filters: List[str] = [] breaking_match_filters: List[str] = [] + date_type: str = self.overrides.apply_formatter(formatter=self.plugin_options.type) if self.plugin_options.before: before_str = to_date_str( date_validator=self.plugin_options.before, overrides=self.overrides ) - match_filters.append(f"upload_date < {before_str}") + match_filters.append(f"{date_type} < {before_str}") if self.plugin_options.after: after_str = to_date_str( date_validator=self.plugin_options.after, overrides=self.overrides ) - after_filter = f"upload_date >= {after_str}" + after_filter = f"{date_type} >= {after_str}" if self.overrides.evaluate_boolean(self.plugin_options.breaks): breaking_match_filters.append(after_filter) else: diff --git a/src/ytdl_sub/prebuilt_presets/tv_show/episode.yaml b/src/ytdl_sub/prebuilt_presets/tv_show/episode.yaml index 4e9ccc7d..fe39e5cf 100644 --- a/src/ytdl_sub/prebuilt_presets/tv_show/episode.yaml +++ b/src/ytdl_sub/prebuilt_presets/tv_show/episode.yaml @@ -17,6 +17,9 @@ presets: chapters: embed_chapters: True + date_range: + type: "{tv_show_date_range_type}" + overrides: # MUST DEFINE: # tv_show_directory @@ -45,3 +48,7 @@ presets: episode_file_path: "{season_directory_name_sanitized}/{episode_file_name_sanitized}" thumbnail_file_name: "{episode_file_path}-thumb.jpg" episode_year: "{%slice(episode_date_standardized, 0, 4)}" + + # Used to determine which date type to use if `Only Recent` preset + # is applied + tv_show_date_range_type: "upload_date" diff --git a/src/ytdl_sub/prebuilt_presets/tv_show/tv_show_by_date.yaml b/src/ytdl_sub/prebuilt_presets/tv_show/tv_show_by_date.yaml index db12b9d0..d2056b9d 100644 --- a/src/ytdl_sub/prebuilt_presets/tv_show/tv_show_by_date.yaml +++ b/src/ytdl_sub/prebuilt_presets/tv_show/tv_show_by_date.yaml @@ -117,6 +117,15 @@ presets: ) } + tv_show_date_range_type: >- + { + %if( + %contains(tv_show_by_date_season_ordering, "release"), + "release_date", + "upload_date" + ) + } + ##################################### season/episode pairing validation "%ordering_pair_eq": >- diff --git a/src/ytdl_sub/prebuilt_presets/tv_show/tv_show_collection.yaml b/src/ytdl_sub/prebuilt_presets/tv_show/tv_show_collection.yaml index 71a81dc6..4d77d0a4 100644 --- a/src/ytdl_sub/prebuilt_presets/tv_show/tv_show_collection.yaml +++ b/src/ytdl_sub/prebuilt_presets/tv_show/tv_show_collection.yaml @@ -71,6 +71,16 @@ presets: ) } + tv_show_date_range_type: >- + { + %if( + %contains(tv_show_collection_episode_ordering, "release"), + "release_date", + "upload_date" + ) + } + + ### LEGACY PRESETS season_by_collection__episode_by_year_month_day: diff --git a/src/ytdl_sub/subscriptions/subscription_download.py b/src/ytdl_sub/subscriptions/subscription_download.py index 9acabda4..2fd9c89e 100644 --- a/src/ytdl_sub/subscriptions/subscription_download.py +++ b/src/ytdl_sub/subscriptions/subscription_download.py @@ -349,6 +349,34 @@ class SubscriptionDownload(BaseSubscription, ABC): return self.download_archive.get_file_handler_transaction_log() + def get_ytdl_options( + self, plugins: Optional[List[Plugin]], dry_run: bool + ) -> SubscriptionYTDLOptions: + """ + Parameters + ---------- + plugins + Optional. If not provided, will reinitialize them + dry_run + Whether its dry run or not + + Returns + ------- + SubscriptionYTDLOptions + Both metadata and download ytdl-options + """ + if plugins is None: + plugins = self._initialize_plugins() + + return SubscriptionYTDLOptions( + preset=self._preset_options, + plugins=plugins, + enhanced_download_archive=self.download_archive, + overrides=self.overrides, + working_directory=self.working_directory, + dry_run=dry_run, + ) + def download(self, dry_run: bool = False) -> FileHandlerTransactionLog: """ Performs the subscription download @@ -368,14 +396,7 @@ class SubscriptionDownload(BaseSubscription, ABC): logging.info("Skipping %s", self.name) return FileHandlerTransactionLog() - subscription_ytdl_options = SubscriptionYTDLOptions( - preset=self._preset_options, - plugins=plugins, - enhanced_download_archive=self.download_archive, - overrides=self.overrides, - working_directory=self.working_directory, - dry_run=dry_run, - ) + subscription_ytdl_options = self.get_ytdl_options(plugins=plugins, dry_run=dry_run) downloader = MultiUrlDownloader( options=self.downloader_options, @@ -420,15 +441,7 @@ class SubscriptionDownload(BaseSubscription, ABC): self.download_archive.reinitialize(dry_run=dry_run) plugins = self._initialize_plugins() - - subscription_ytdl_options = SubscriptionYTDLOptions( - preset=self._preset_options, - plugins=plugins, - enhanced_download_archive=self.download_archive, - overrides=self.overrides, - working_directory=self.working_directory, - dry_run=dry_run, - ) + subscription_ytdl_options = self.get_ytdl_options(plugins=plugins, dry_run=dry_run) # Re-add the original downloader class' plugins plugins.extend( diff --git a/src/ytdl_sub/validators/string_select_validator.py b/src/ytdl_sub/validators/string_select_validator.py index 24fe767d..9f28fcd4 100644 --- a/src/ytdl_sub/validators/string_select_validator.py +++ b/src/ytdl_sub/validators/string_select_validator.py @@ -1,5 +1,6 @@ from typing import Set +from ytdl_sub.validators.string_formatter_validators import OverridesStringFormatterValidator from ytdl_sub.validators.validators import StringValidator @@ -17,3 +18,17 @@ class StringSelectValidator(StringValidator): raise self._validation_exception( f"Must be one of the following values: {', '.join(self._select_values)}" ) + + +class OverridesStringSelectValidator(OverridesStringFormatterValidator): + _expected_value_type_name = "overrides select" + + _select_values: Set[str] = set() + + def post_process(self, resolved: str) -> str: + if resolved not in self._select_values: + raise self._validation_exception( + f"Must be one of the following values: {', '.join(sorted(self._select_values))}" + ) + + return resolved diff --git a/tests/conftest.py b/tests/conftest.py index 12ba38a9..aabbba8d 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -12,6 +12,7 @@ from typing import Callable from typing import Dict from typing import List from typing import Optional +from typing import Tuple from unittest.mock import patch import pytest @@ -272,3 +273,27 @@ def mock_run_from_cli(args: str) -> List[Subscription]: args_list = ["ytdl-sub"] + shlex.split(args) with patch.object(sys, "argv", args_list): return main() + + +def get_match_filters( + subscription: Subscription, dry_run: bool, download_filters: bool +) -> Tuple[List[str], List[str]]: + """ + Util function to get match filters from a subscription. + + Returns + ------- + match_filters, breaking_match_filters + """ + options = subscription.get_ytdl_options(plugins=None, dry_run=dry_run) + options_dict = ( + options.download_builder().to_dict() + if download_filters + else options.metadata_builder().to_dict() + ) + if "match_filter" not in options_dict: + return [], [] + + match_filter_str = repr(options_dict["match_filter"]) + out = eval(match_filter_str.split("(", maxsplit=1)[-1].split(")")[0]) + return out diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 621ad68f..fbd26c4c 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -1,6 +1,5 @@ import contextlib import os -import shutil from pathlib import Path from typing import Callable from typing import Dict diff --git a/tests/integration/plugins/test_file_convert.py b/tests/integration/plugins/test_file_convert.py index 740cea5b..06980b22 100644 --- a/tests/integration/plugins/test_file_convert.py +++ b/tests/integration/plugins/test_file_convert.py @@ -4,7 +4,6 @@ import pytest from expected_download import assert_expected_downloads from expected_transaction_log import assert_transaction_log_matches -from ytdl_sub.entries.entry import ytdl_sub_chapters_from_comments from ytdl_sub.subscriptions.subscription import Subscription diff --git a/tests/unit/plugins/__init__.py b/tests/unit/plugins/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/unit/plugins/test_date_range.py b/tests/unit/plugins/test_date_range.py new file mode 100644 index 00000000..a024d8e2 --- /dev/null +++ b/tests/unit/plugins/test_date_range.py @@ -0,0 +1,108 @@ +import re +from typing import Any +from typing import Dict + +import pytest +from conftest import get_match_filters + +from ytdl_sub.config.config_file import ConfigFile +from ytdl_sub.subscriptions.subscription import Subscription +from ytdl_sub.utils.exceptions import ValidationException + + +@pytest.fixture +def preset_dict(output_directory) -> Dict[str, Any]: + return { + "download": "https://your.name.here", + "output_options": {"output_directory": output_directory, "file_name": "will_error.mp4"}, + } + + +class TestDateRange: + @pytest.mark.parametrize("date_range_type", ["upload_date", "release_date"]) + def test_date_range_type( + self, + default_config: ConfigFile, + preset_dict: Dict[str, Any], + output_directory: str, + date_range_type: str, + ): + preset_dict["date_range"] = { + "before": "20250530", + "after": "20250510", + "type": date_range_type, + } + sub = Subscription.from_dict( + config=default_config, + preset_name="test_date_range", + preset_dict=preset_dict, + ) + + metadata_filter, metadata_breaking_filter = get_match_filters( + subscription=sub, dry_run=False, download_filters=False + ) + assert metadata_filter == [ + f"!is_live & !is_upcoming & !post_live & {date_range_type} < 20250530" + ] + assert metadata_breaking_filter == [f"{date_range_type} >= 20250510"] + + download_filter, download_breaking_filter = get_match_filters( + subscription=sub, dry_run=False, download_filters=True + ) + assert not download_filter + assert not download_breaking_filter + + def test_date_range_breaks_false( + self, + default_config: ConfigFile, + preset_dict: Dict[str, Any], + output_directory: str, + ): + preset_dict["date_range"] = { + "before": "20250530", + "after": "20250510", + "breaks": False, + } + sub = Subscription.from_dict( + config=default_config, + preset_name="test_date_range", + preset_dict=preset_dict, + ) + + metadata_filter, metadata_breaking_filter = get_match_filters( + subscription=sub, dry_run=False, download_filters=False + ) + assert metadata_filter == [ + f"!is_live & !is_upcoming & !post_live & upload_date < 20250530 & upload_date >= 20250510" + ] + assert not metadata_breaking_filter + + download_filter, download_breaking_filter = get_match_filters( + subscription=sub, dry_run=False, download_filters=True + ) + assert not download_filter + assert not download_breaking_filter + + def test_date_range_invalid_type( + self, + default_config: ConfigFile, + preset_dict: Dict[str, Any], + output_directory: str, + ): + preset_dict["date_range"] = { + "before": "20250530", + "after": "20250510", + "type": "no", + } + + error_msg = ( + "Validation error in test_date_range.date_range.type: " + "Must be one of the following values: release_date, upload_date" + ) + + with pytest.raises(ValidationException, match=re.escape(error_msg)): + Subscription.from_dict( + config=default_config, + preset_name="test_date_range", + preset_dict=preset_dict, + ).download(dry_run=False)