diff --git a/docs/source/config_reference/plugins.rst b/docs/source/config_reference/plugins.rst index 69afce80..8e63e291 100644 --- a/docs/source/config_reference/plugins.rst +++ b/docs/source/config_reference/plugins.rst @@ -640,6 +640,8 @@ Defines where to output files and thumbnails after all post-processing has compl maintain_download_archive: True keep_files_before: now keep_files_after: 19000101 + keep_max_files: 1000 + keep_files_date_eval: "{upload_date_standardized}" ``download_archive_name`` @@ -690,6 +692,16 @@ Defines where to output files and thumbnails after all post-processing has compl ``keep_max_files``. +``keep_files_date_eval`` + +:expected type: str +:description: + Uses this standardized date in the form of YYYY-MM-DD to record in the + download archive for a given entry. Subsequently, uses this value to + perform evaluation for keep_files_before/after and keep_max_files. Defaults + to the entry's upload_date_standardized variable. + + ``keep_max_files`` :expected type: Optional[OverridesFormatter] diff --git a/docs/source/config_reference/scripting/entry_variables.rst b/docs/source/config_reference/scripting/entry_variables.rst index 84247234..eea90981 100644 --- a/docs/source/config_reference/scripting/entry_variables.rst +++ b/docs/source/config_reference/scripting/entry_variables.rst @@ -659,3 +659,9 @@ ytdl_sub_input_url_index :type: ``Integer`` :description: The index of the input URL as defined in the subscription, top-most being the 0th index. + +ytdl_sub_keep_files_date_eval +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +:type: ``String`` +:description: + The standardized date variable supplied in ``output_options.keep_files_date_eval``. diff --git a/src/ytdl_sub/config/preset_options.py b/src/ytdl_sub/config/preset_options.py index f1ba8534..a16476ed 100644 --- a/src/ytdl_sub/config/preset_options.py +++ b/src/ytdl_sub/config/preset_options.py @@ -1,15 +1,19 @@ from typing import Any from typing import Dict from typing import Optional +from typing import Set from ytdl_sub.config.defaults import DEFAULT_DOWNLOAD_ARCHIVE_NAME from ytdl_sub.config.overrides import Overrides +from ytdl_sub.config.plugin.plugin_operation import PluginOperation +from ytdl_sub.config.validators.options import OptionsDictValidator +from ytdl_sub.entries.script.variable_definitions import VARIABLES as v from ytdl_sub.validators.file_path_validators import OverridesStringFormatterFilePathValidator from ytdl_sub.validators.file_path_validators import StringFormatterFileNameValidator -from ytdl_sub.validators.strict_dict_validator import StrictDictValidator from ytdl_sub.validators.string_datetime import StringDatetimeValidator from ytdl_sub.validators.string_formatter_validators import OverridesIntegerFormatterValidator from ytdl_sub.validators.string_formatter_validators import OverridesStringFormatterValidator +from ytdl_sub.validators.string_formatter_validators import StandardizedDateValidator from ytdl_sub.validators.string_formatter_validators import StringFormatterValidator from ytdl_sub.validators.string_formatter_validators import ( UnstructuredOverridesDictFormatterValidator, @@ -65,7 +69,7 @@ class YTDLOptions(UnstructuredOverridesDictFormatterValidator): # pylint: disable=line-too-long -class OutputOptions(StrictDictValidator): +class OutputOptions(OptionsDictValidator): """ Defines where to output files and thumbnails after all post-processing has completed. @@ -87,6 +91,8 @@ class OutputOptions(StrictDictValidator): maintain_download_archive: True keep_files_before: now keep_files_after: 19000101 + keep_max_files: 1000 + keep_files_date_eval: "{upload_date_standardized}" """ _required_keys = {"output_directory", "file_name"} @@ -99,6 +105,8 @@ class OutputOptions(StrictDictValidator): "keep_files_before", "keep_files_after", "keep_max_files", + "download_archive_standardized_date", + "keep_files_date_eval", } @classmethod @@ -156,6 +164,11 @@ class OutputOptions(StrictDictValidator): self._keep_max_files = self._validate_key_if_present( "keep_max_files", OverridesIntegerFormatterValidator ) + self._keep_files_date_eval = self._validate_key( + "keep_files_date_eval", + StandardizedDateValidator, + default=f"{{{v.upload_date_standardized.variable_name}}}", + ) if ( self._keep_files_before or self._keep_files_after or self._keep_max_files @@ -272,6 +285,18 @@ class OutputOptions(StrictDictValidator): """ return self._keep_files_after + @property + def keep_files_date_eval(self) -> StandardizedDateValidator: + """ + :expected type: str + :description: + Uses this standardized date in the form of YYYY-MM-DD to record in the + download archive for a given entry. Subsequently, uses this value to + perform evaluation for keep_files_before/after and keep_max_files. Defaults + to the entry's upload_date_standardized variable. + """ + return self._keep_files_date_eval + @property def keep_max_files(self) -> Optional[OverridesIntegerFormatterValidator]: """ @@ -283,3 +308,10 @@ class OutputOptions(StrictDictValidator): applied. Can be used in conjunction with ``keep_files_before`` and ``keep_files_after``. """ return self._keep_max_files + + def added_variables(self, unresolved_variables: Set[str]) -> Dict[PluginOperation, Set[str]]: + return { + # PluginOperation.MODIFY_ENTRY_METADATA: { + # VARIABLES.ytdl_sub_entry_date_eval.variable_name + # } + } diff --git a/src/ytdl_sub/config/validators/variable_validation.py b/src/ytdl_sub/config/validators/variable_validation.py index 60488394..e2d5e85d 100644 --- a/src/ytdl_sub/config/validators/variable_validation.py +++ b/src/ytdl_sub/config/validators/variable_validation.py @@ -57,13 +57,14 @@ def _add_dummy_overrides(overrides: Overrides) -> Dict[str, str]: def _get_added_and_modified_variables( - plugins: PresetPlugins, downloader_options: MultiUrlValidator + plugins: PresetPlugins, downloader_options: MultiUrlValidator, output_options: OutputOptions ) -> Iterable[Tuple[OptionsValidator, Set[str], Set[str]]]: """ Iterates and returns the plugin options, added variables, modified variables """ options: List[OptionsValidator] = plugins.plugin_options options.append(downloader_options) + options.append(output_options) for plugin_options in options: added_variables: Set[str] = set() @@ -117,6 +118,7 @@ class VariableValidation: ) in _get_added_and_modified_variables( plugins=self.plugins, downloader_options=self.downloader_options, + output_options=self.output_options, ): for added_variable in added_variables: @@ -183,6 +185,9 @@ class VariableValidation: self._add_variables(PluginOperation.DOWNLOADER, options=self.downloader_options) self._add_subscription_override_variables() + # Always add output options first + self._add_variables(PluginOperation.MODIFY_ENTRY_METADATA, options=self.output_options) + # Metadata variables to be added for plugin_options in PluginMapping.order_options_by( self.plugins.zipped(), PluginOperation.MODIFY_ENTRY_METADATA diff --git a/src/ytdl_sub/entries/script/variable_definitions.py b/src/ytdl_sub/entries/script/variable_definitions.py index 5526780a..cc86737d 100644 --- a/src/ytdl_sub/entries/script/variable_definitions.py +++ b/src/ytdl_sub/entries/script/variable_definitions.py @@ -823,6 +823,17 @@ class YtdlSubVariableDefinitions(ABC): variable_name="upload_date_index_reversed_padded", pad=2 ) + @cached_property + def ytdl_sub_keep_files_date_eval(self: "VariableDefinitions") -> StringVariable: + """ + :description: + The standardized date variable supplied in ``output_options.keep_files_date_eval``. + """ + return StringVariable( + variable_name="ytdl_sub_entry_date_eval", + definition=f"{{%string({self.upload_date_standardized.variable_name})}}", + ) + class EntryVariableDefinitions(ABC): @cached_property @@ -1121,6 +1132,7 @@ class VariableDefinitions( self.ytdl_sub_input_url, self.ytdl_sub_input_url_index, self.ytdl_sub_input_url_count, + self.ytdl_sub_keep_files_date_eval, } @cache diff --git a/src/ytdl_sub/subscriptions/subscription_download.py b/src/ytdl_sub/subscriptions/subscription_download.py index 5d1b4cb3..9acabda4 100644 --- a/src/ytdl_sub/subscriptions/subscription_download.py +++ b/src/ytdl_sub/subscriptions/subscription_download.py @@ -17,6 +17,7 @@ from ytdl_sub.downloaders.source_plugin import SourcePlugin from ytdl_sub.downloaders.url.downloader import MultiUrlDownloader from ytdl_sub.downloaders.ytdl_options_builder import YTDLOptionsBuilder from ytdl_sub.entries.entry import Entry +from ytdl_sub.entries.script.variable_definitions import VARIABLES from ytdl_sub.subscriptions.base_subscription import BaseSubscription from ytdl_sub.subscriptions.subscription_ytdl_options import SubscriptionYTDLOptions from ytdl_sub.utils.datetime import to_date_range @@ -215,9 +216,23 @@ class SubscriptionDownload(BaseSubscription, ABC): FileHandler.delete(entry.get_download_thumbnail_path()) FileHandler.delete(entry.get_download_info_json_path()) - @classmethod - def _preprocess_entry(cls, plugins: List[Plugin], entry: Entry) -> Optional[Entry]: + def _preprocess_entry(self, plugins: List[Plugin], entry: Entry) -> Optional[Entry]: maybe_entry: Optional[Entry] = entry + + # Inject OutputOption variables here + entry.add( + { + VARIABLES.ytdl_sub_keep_files_date_eval: ( + self.output_options.keep_files_date_eval.format_string + ) + } + ) + + # Run it to make sure it's actually a standardized date + _ = self.overrides.apply_formatter( + formatter=self.output_options.keep_files_date_eval, entry=entry + ) + for plugin in PluginMapping.order_plugins_by( plugins, PluginOperation.MODIFY_ENTRY_METADATA ): diff --git a/src/ytdl_sub/validators/string_formatter_validators.py b/src/ytdl_sub/validators/string_formatter_validators.py index 2c709ebb..855a607f 100644 --- a/src/ytdl_sub/validators/string_formatter_validators.py +++ b/src/ytdl_sub/validators/string_formatter_validators.py @@ -1,3 +1,4 @@ +from datetime import datetime from typing import Dict from typing import Set from typing import Union @@ -73,6 +74,20 @@ class StringFormatterValidator(StringValidator): return resolved +class StandardizedDateValidator(StringFormatterValidator): + _expected_value_type_name = "standardized_date" + + def post_process(self, resolved: str) -> str: + try: + datetime.strptime(resolved, "%Y-%m-%d") + except ValueError as exc: + raise self._validation_exception( + f"Expected a standardized date in the form of YYYY-MM-DD, but received '{resolved}'" + ) from exc + + return resolved + + # pylint: disable=line-too-long class OverridesStringFormatterValidator(StringFormatterValidator): """ diff --git a/src/ytdl_sub/ytdl_additions/enhanced_download_archive.py b/src/ytdl_sub/ytdl_additions/enhanced_download_archive.py index ec155245..2c633a74 100644 --- a/src/ytdl_sub/ytdl_additions/enhanced_download_archive.py +++ b/src/ytdl_sub/ytdl_additions/enhanced_download_archive.py @@ -75,7 +75,7 @@ class DownloadMapping: DownloadMapping for the entry """ return DownloadMapping( - upload_date=entry.get(v.upload_date_standardized, str), + upload_date=entry.get(v.ytdl_sub_keep_files_date_eval, str), extractor=entry.download_archive_extractor, file_names=set(), ) diff --git a/tests/integration/plugins/test_output_options.py b/tests/integration/plugins/test_output_options.py index c8665c1e..458416b4 100644 --- a/tests/integration/plugins/test_output_options.py +++ b/tests/integration/plugins/test_output_options.py @@ -1,3 +1,4 @@ +import re from pathlib import Path from typing import Dict from unittest.mock import patch @@ -11,6 +12,7 @@ from ytdl_sub.config.config_file import ConfigFile from ytdl_sub.downloaders.ytdlp import YTDLP from ytdl_sub.entries.entry import Entry from ytdl_sub.subscriptions.subscription import Subscription +from ytdl_sub.utils.exceptions import ValidationException from ytdl_sub.utils.file_handler import FileHandler from ytdl_sub.utils.thumbnail import try_convert_download_thumbnail @@ -223,3 +225,35 @@ class TestOutputOptions: dry_run=False, expected_download_summary_file_name="plugins/output_options/test_missing_thumb.json", ) + + def test_invalid_keep_files_date_eval( + self, + config: ConfigFile, + subscription_name: str, + output_options_subscription_dict: Dict, + output_directory: str, + mock_download_collection_entries, + ): + output_options_subscription_dict["output_options"]["keep_files_date_eval"] = "nope" + + subscription = Subscription.from_dict( + config=config, + preset_name=subscription_name, + preset_dict=output_options_subscription_dict, + ) + + expected_error_msg = ( + "Validation error in subscription_test.output_options.keep_files_date_eval: " + "Expected a standardized date in the form of YYYY-MM-DD, but received 'nope'" + ) + + with ( + mock_download_collection_entries( + is_youtube_channel=False, + num_urls=1, + is_extracted_audio=False, + is_dry_run=True, + ), + pytest.raises(ValidationException, match=re.escape(expected_error_msg)), + ): + subscription.download(dry_run=True)