From ffa318a74ded8d1e7f2600765f9ebaa1cc4ace01 Mon Sep 17 00:00:00 2001 From: Jesse Bannon Date: Fri, 17 Nov 2023 01:22:05 -0800 Subject: [PATCH] updatead and working --- examples/advanced/tv_show_config.yaml | 3 ++- src/ytdl_sub/config/preset_options.py | 16 ++++++++-------- .../prebuilt_presets/helpers/common.yaml | 4 ++-- .../subscriptions/subscription_download.py | 10 +++++++++- .../validators/string_formatter_validators.py | 12 ++++++++++++ .../ytdl_additions/enhanced_download_archive.py | 2 +- tests/e2e/youtube/test_channel.py | 9 +++++++-- 7 files changed, 41 insertions(+), 15 deletions(-) diff --git a/examples/advanced/tv_show_config.yaml b/examples/advanced/tv_show_config.yaml index b2d0c694..9cfa4ca7 100644 --- a/examples/advanced/tv_show_config.yaml +++ b/examples/advanced/tv_show_config.yaml @@ -97,4 +97,5 @@ presets: - "Only Recent" overrides: - date_range: "2months" + only_recent_date_range: "2months" + only_recent_max_files: 30 \ No newline at end of file diff --git a/src/ytdl_sub/config/preset_options.py b/src/ytdl_sub/config/preset_options.py index 19475858..b4738a8c 100644 --- a/src/ytdl_sub/config/preset_options.py +++ b/src/ytdl_sub/config/preset_options.py @@ -16,10 +16,10 @@ from ytdl_sub.validators.file_path_validators import StringFormatterFileNameVali from ytdl_sub.validators.strict_dict_validator import StrictDictValidator from ytdl_sub.validators.string_datetime import StringDatetimeValidator from ytdl_sub.validators.string_formatter_validators import DictFormatterValidator +from ytdl_sub.validators.string_formatter_validators import OverridesIntegerFormatterValidator from ytdl_sub.validators.string_formatter_validators import OverridesStringFormatterValidator from ytdl_sub.validators.string_formatter_validators import StringFormatterValidator from ytdl_sub.validators.validators import BoolValidator -from ytdl_sub.validators.validators import IntValidator from ytdl_sub.validators.validators import LiteralDictValidator from ytdl_sub.validators.validators import Validator @@ -309,13 +309,15 @@ class OutputOptions(StrictDictValidator): self._keep_files_after = self._validate_key_if_present( "keep_files_after", StringDatetimeValidator ) - self._keep_max_files = self._validate_key_if_present("keep_max_files", IntValidator) + self._keep_max_files = self._validate_key_if_present( + "keep_max_files", OverridesIntegerFormatterValidator + ) if ( - self._keep_files_before or self._keep_files_after + self._keep_files_before or self._keep_files_after or self._keep_max_files ) and not self.maintain_download_archive: raise self._validation_exception( - "keep_files requires maintain_download_archive set to True" + "keep_files/keep_max requires maintain_download_archive set to True" ) @property @@ -407,13 +409,11 @@ class OutputOptions(StrictDictValidator): return self._keep_files_after @property - def keep_max_files(self) -> Optional[int]: + def keep_max_files(self) -> Optional[OverridesIntegerFormatterValidator]: """ Optional. Requires ``maintain_download_archive`` set to True. Only keeps N most recently uploaded videos. If set to <= 0, ``keep_max_files`` will not be applied. Can be used in conjunction with ``keep_files_before`` and ``keep_files_after``. """ - if self._keep_max_files and self._keep_max_files.value > 0: - return self._keep_max_files.value - return None + return self._keep_max_files diff --git a/src/ytdl_sub/prebuilt_presets/helpers/common.yaml b/src/ytdl_sub/prebuilt_presets/helpers/common.yaml index adc1c3e5..9278d012 100644 --- a/src/ytdl_sub/prebuilt_presets/helpers/common.yaml +++ b/src/ytdl_sub/prebuilt_presets/helpers/common.yaml @@ -27,10 +27,10 @@ presets: # Only keep files uploaded after date_range output_options: keep_files_after: "today-{only_recent_date_range}" - keep_files_max: "{only_recent_max_files}" + keep_max_files: "{only_recent_max_files}" # Set the default date_range to 2 months overrides: - date_range: "2months" + date_range: "2months" # keep for legacy-reasons only_recent_date_range: "{date_range}" only_recent_max_files: 0 \ No newline at end of file diff --git a/src/ytdl_sub/subscriptions/subscription_download.py b/src/ytdl_sub/subscriptions/subscription_download.py index fd284e59..312fb46f 100644 --- a/src/ytdl_sub/subscriptions/subscription_download.py +++ b/src/ytdl_sub/subscriptions/subscription_download.py @@ -145,9 +145,17 @@ class SubscriptionDownload(BaseSubscription, ABC): after=self.output_options.keep_files_after, overrides=self.overrides, ) + + keep_max_files: Optional[int] = None + if self.output_options.keep_max_files: + # validated it can be cast to int within the validator + keep_max_files = int( + self.overrides.apply_formatter(self.output_options.keep_max_files) + ) + if date_range_to_keep or self.output_options.keep_max_files is not None: self._enhanced_download_archive.remove_stale_files( - date_range=date_range_to_keep, keep_max_files=self.output_options.keep_max_files + date_range=date_range_to_keep, keep_max_files=keep_max_files ) self._enhanced_download_archive.save_download_mappings() diff --git a/src/ytdl_sub/validators/string_formatter_validators.py b/src/ytdl_sub/validators/string_formatter_validators.py index b3a83cc1..d5f2a00f 100644 --- a/src/ytdl_sub/validators/string_formatter_validators.py +++ b/src/ytdl_sub/validators/string_formatter_validators.py @@ -218,6 +218,18 @@ class OverridesStringFormatterValidator(StringFormatterValidator): # pylint: enable=line-too-long +class OverridesIntegerFormatterValidator(StringFormatterValidator): + _expected_value_type_name = "integer" + + def apply_formatter(self, variable_dict: Dict[str, str]) -> str: + output = super().apply_formatter(variable_dict) + try: + int(output) + except Exception as exc: + raise self._validation_exception(f"Expected an integer, but received '{output}'") from exc + return output + + class ListFormatterValidator(ListValidator[StringFormatterValidator]): _inner_list_type = StringFormatterValidator diff --git a/src/ytdl_sub/ytdl_additions/enhanced_download_archive.py b/src/ytdl_sub/ytdl_additions/enhanced_download_archive.py index 9fe14454..584f5541 100644 --- a/src/ytdl_sub/ytdl_additions/enhanced_download_archive.py +++ b/src/ytdl_sub/ytdl_additions/enhanced_download_archive.py @@ -570,7 +570,7 @@ class EnhancedDownloadArchive: for uid, mapping in stale_mappings.items(): self._remove_entry(uid=uid, mapping=mapping) - if keep_max_files is not None: + if keep_max_files is not None and keep_max_files > 0: num_files = 0 for uid, mapping in sorted( self.mapping.entry_mappings.items(), diff --git a/tests/e2e/youtube/test_channel.py b/tests/e2e/youtube/test_channel.py index f8341b12..2b36cf9b 100644 --- a/tests/e2e/youtube/test_channel.py +++ b/tests/e2e/youtube/test_channel.py @@ -12,7 +12,9 @@ from ytdl_sub.subscriptions.subscription import Subscription @pytest.fixture def channel_preset_dict(output_directory): return { - "preset": "TV Show Full Archive", + "preset": [ + "TV Show Full Archive", + ], "format": "worst[ext=mp4]", # download the worst format so it is fast "ytdl_options": { "max_views": 100000, # do not download the popular PJ concert @@ -98,7 +100,10 @@ class TestChannel: output_directory: str, ): subscription_name = "pz" - channel_preset_dict = dict(channel_preset_dict, **{"output_options": {"keep_max_files": 1}}) + channel_preset_dict["preset"].append("Only Recent") + channel_preset_dict["overrides"]["only_recent_date_range"] = "10years" + channel_preset_dict["overrides"]["only_recent_max_files"] = 1 + full_channel_subscription = Subscription.from_dict( config=tv_show_config, preset_name=subscription_name, preset_dict=channel_preset_dict )