From 9fd46bf929242fecbd977e54e6bab6ce1e0ca5b6 Mon Sep 17 00:00:00 2001 From: Jesse Bannon Date: Sat, 31 May 2025 08:03:21 -0700 Subject: [PATCH] working? --- src/ytdl_sub/config/preset_options.py | 29 ++++++++---- .../config/validators/variable_validation.py | 7 ++- .../entries/script/variable_definitions.py | 8 +++- .../subscriptions/base_subscription.py | 1 - .../subscriptions/subscription_download.py | 15 ++++++- .../validators/string_formatter_validators.py | 27 ++++++------ .../enhanced_download_archive.py | 44 +++++-------------- 7 files changed, 72 insertions(+), 59 deletions(-) diff --git a/src/ytdl_sub/config/preset_options.py b/src/ytdl_sub/config/preset_options.py index 66e2b0a2..c6a6d65e 100644 --- a/src/ytdl_sub/config/preset_options.py +++ b/src/ytdl_sub/config/preset_options.py @@ -1,16 +1,19 @@ from typing import Any from typing import Dict from typing import Optional +from typing import Set from ytdl_sub.config.defaults import DEFAULT_DOWNLOAD_ARCHIVE_NAME from ytdl_sub.config.overrides import Overrides +from ytdl_sub.config.plugin.plugin_operation import PluginOperation +from ytdl_sub.config.validators.options import OptionsDictValidator +from ytdl_sub.entries.script.variable_definitions import VARIABLES as v from ytdl_sub.validators.file_path_validators import OverridesStringFormatterFilePathValidator from ytdl_sub.validators.file_path_validators import StringFormatterFileNameValidator -from ytdl_sub.validators.strict_dict_validator import StrictDictValidator from ytdl_sub.validators.string_datetime import StringDatetimeValidator -from ytdl_sub.validators.string_formatter_validators import OverridesIntegerFormatterValidator, \ - OverridesStandardizedDateValidator +from ytdl_sub.validators.string_formatter_validators import OverridesIntegerFormatterValidator from ytdl_sub.validators.string_formatter_validators import OverridesStringFormatterValidator +from ytdl_sub.validators.string_formatter_validators import StandardizedDateValidator from ytdl_sub.validators.string_formatter_validators import StringFormatterValidator from ytdl_sub.validators.string_formatter_validators import ( UnstructuredOverridesDictFormatterValidator, @@ -66,7 +69,7 @@ class YTDLOptions(UnstructuredOverridesDictFormatterValidator): # pylint: disable=line-too-long -class OutputOptions(StrictDictValidator): +class OutputOptions(OptionsDictValidator): """ Defines where to output files and thumbnails after all post-processing has completed. @@ -89,7 +92,7 @@ class OutputOptions(StrictDictValidator): keep_files_before: now keep_files_after: 19000101 keep_max_files: 1000 - keep_files_date_eval: "upload_date" + keep_files_date_eval: "{upload_date_standardized}" """ _required_keys = {"output_directory", "file_name"} @@ -103,6 +106,7 @@ class OutputOptions(StrictDictValidator): "keep_files_after", "keep_max_files", "download_archive_standardized_date", + "entry_date_eval", } @classmethod @@ -160,8 +164,10 @@ class OutputOptions(StrictDictValidator): self._keep_max_files = self._validate_key_if_present( "keep_max_files", OverridesIntegerFormatterValidator ) - self._entry_date_eval = self._validate_key_if_present( - "entry_date_eval", OverridesStandardizedDateValidator + self._entry_date_eval = self._validate_key( + "entry_date_eval", + StandardizedDateValidator, + default=f"{{{v.upload_date_standardized.variable_name}}}" ) if ( @@ -280,7 +286,7 @@ class OutputOptions(StrictDictValidator): return self._keep_files_after @property - def entry_date_eval(self) -> Optional[OverridesStandardizedDateValidator]: + def entry_date_eval(self) -> StandardizedDateValidator: """ :expected type: str :description: @@ -302,3 +308,10 @@ class OutputOptions(StrictDictValidator): applied. Can be used in conjunction with ``keep_files_before`` and ``keep_files_after``. """ return self._keep_max_files + + def added_variables(self, unresolved_variables: Set[str]) -> Dict[PluginOperation, Set[str]]: + return { + # PluginOperation.MODIFY_ENTRY_METADATA: { + # VARIABLES.ytdl_sub_entry_date_eval.variable_name + # } + } diff --git a/src/ytdl_sub/config/validators/variable_validation.py b/src/ytdl_sub/config/validators/variable_validation.py index 60488394..e2d5e85d 100644 --- a/src/ytdl_sub/config/validators/variable_validation.py +++ b/src/ytdl_sub/config/validators/variable_validation.py @@ -57,13 +57,14 @@ def _add_dummy_overrides(overrides: Overrides) -> Dict[str, str]: def _get_added_and_modified_variables( - plugins: PresetPlugins, downloader_options: MultiUrlValidator + plugins: PresetPlugins, downloader_options: MultiUrlValidator, output_options: OutputOptions ) -> Iterable[Tuple[OptionsValidator, Set[str], Set[str]]]: """ Iterates and returns the plugin options, added variables, modified variables """ options: List[OptionsValidator] = plugins.plugin_options options.append(downloader_options) + options.append(output_options) for plugin_options in options: added_variables: Set[str] = set() @@ -117,6 +118,7 @@ class VariableValidation: ) in _get_added_and_modified_variables( plugins=self.plugins, downloader_options=self.downloader_options, + output_options=self.output_options, ): for added_variable in added_variables: @@ -183,6 +185,9 @@ class VariableValidation: self._add_variables(PluginOperation.DOWNLOADER, options=self.downloader_options) self._add_subscription_override_variables() + # Always add output options first + self._add_variables(PluginOperation.MODIFY_ENTRY_METADATA, options=self.output_options) + # Metadata variables to be added for plugin_options in PluginMapping.order_options_by( self.plugins.zipped(), PluginOperation.MODIFY_ENTRY_METADATA diff --git a/src/ytdl_sub/entries/script/variable_definitions.py b/src/ytdl_sub/entries/script/variable_definitions.py index 50ba44c7..4c939801 100644 --- a/src/ytdl_sub/entries/script/variable_definitions.py +++ b/src/ytdl_sub/entries/script/variable_definitions.py @@ -827,9 +827,12 @@ class YtdlSubVariableDefinitions(ABC): def ytdl_sub_entry_date_eval(self: "VariableDefinitions") -> StringVariable: """ :description: - The standardized + The standardized date variable supplied in ``output_options.entry_date_eval`` """ - return StringVariable(variable_name="ytdl_sub_input_url", definition="{ %string('') }") + return StringVariable( + variable_name="ytdl_sub_entry_date_eval", + definition=f"{{%string({self.upload_date_standardized.variable_name})}}", + ) class EntryVariableDefinitions(ABC): @@ -1129,6 +1132,7 @@ class VariableDefinitions( self.ytdl_sub_input_url, self.ytdl_sub_input_url_index, self.ytdl_sub_input_url_count, + self.ytdl_sub_entry_date_eval, } @cache diff --git a/src/ytdl_sub/subscriptions/base_subscription.py b/src/ytdl_sub/subscriptions/base_subscription.py index 3123ecfd..970f142d 100644 --- a/src/ytdl_sub/subscriptions/base_subscription.py +++ b/src/ytdl_sub/subscriptions/base_subscription.py @@ -31,7 +31,6 @@ def _initialize_download_archive( file_name=overrides.apply_formatter(output_options.download_archive_name), working_directory=working_directory, output_directory=output_directory, - entry_date_eval=output_options.entry_date_eval, migrated_file_name=migrated_file_name, ).reinitialize(dry_run=True) diff --git a/src/ytdl_sub/subscriptions/subscription_download.py b/src/ytdl_sub/subscriptions/subscription_download.py index 5d1b4cb3..f65538d3 100644 --- a/src/ytdl_sub/subscriptions/subscription_download.py +++ b/src/ytdl_sub/subscriptions/subscription_download.py @@ -17,6 +17,7 @@ from ytdl_sub.downloaders.source_plugin import SourcePlugin from ytdl_sub.downloaders.url.downloader import MultiUrlDownloader from ytdl_sub.downloaders.ytdl_options_builder import YTDLOptionsBuilder from ytdl_sub.entries.entry import Entry +from ytdl_sub.entries.script.variable_definitions import VARIABLES from ytdl_sub.subscriptions.base_subscription import BaseSubscription from ytdl_sub.subscriptions.subscription_ytdl_options import SubscriptionYTDLOptions from ytdl_sub.utils.datetime import to_date_range @@ -215,9 +216,19 @@ class SubscriptionDownload(BaseSubscription, ABC): FileHandler.delete(entry.get_download_thumbnail_path()) FileHandler.delete(entry.get_download_info_json_path()) - @classmethod - def _preprocess_entry(cls, plugins: List[Plugin], entry: Entry) -> Optional[Entry]: + def _preprocess_entry(self, plugins: List[Plugin], entry: Entry) -> Optional[Entry]: maybe_entry: Optional[Entry] = entry + + # Inject OutputOption variables here + entry.add( + {VARIABLES.ytdl_sub_entry_date_eval: self.output_options.entry_date_eval.format_string} + ) + + # Run it to make sure it's actually a standardized date + _ = self.overrides.apply_formatter( + formatter=self.output_options.entry_date_eval, entry=entry + ) + for plugin in PluginMapping.order_plugins_by( plugins, PluginOperation.MODIFY_ENTRY_METADATA ): diff --git a/src/ytdl_sub/validators/string_formatter_validators.py b/src/ytdl_sub/validators/string_formatter_validators.py index 15edd217..855a607f 100644 --- a/src/ytdl_sub/validators/string_formatter_validators.py +++ b/src/ytdl_sub/validators/string_formatter_validators.py @@ -74,6 +74,20 @@ class StringFormatterValidator(StringValidator): return resolved +class StandardizedDateValidator(StringFormatterValidator): + _expected_value_type_name = "standardized_date" + + def post_process(self, resolved: str) -> str: + try: + datetime.strptime(resolved, "%Y-%m-%d") + except ValueError as exc: + raise self._validation_exception( + f"Expected a standardized date in the form of YYYY-MM-DD, but received '{resolved}'" + ) from exc + + return resolved + + # pylint: disable=line-too-long class OverridesStringFormatterValidator(StringFormatterValidator): """ @@ -104,19 +118,6 @@ class OverridesIntegerFormatterValidator(OverridesStringFormatterValidator): return resolved -class OverridesStandardizedDateValidator(OverridesStringFormatterValidator): - _expected_value_type_name = "standardized_date" - - def post_process(self, resolved: str) -> str: - try: - datetime.strptime(resolved, "%Y-%m-%d") - except ValueError as exc: - raise self._validation_exception( - f"Expected a standardized date in the form of YYYY-MM-DD, but received '{resolved}'" - ) from exc - - return resolved - class OverridesBooleanFormatterValidator(OverridesStringFormatterValidator): _expected_value_type_name = "boolean" diff --git a/src/ytdl_sub/ytdl_additions/enhanced_download_archive.py b/src/ytdl_sub/ytdl_additions/enhanced_download_archive.py index 4ebc795e..25f1a923 100644 --- a/src/ytdl_sub/ytdl_additions/enhanced_download_archive.py +++ b/src/ytdl_sub/ytdl_additions/enhanced_download_archive.py @@ -21,7 +21,6 @@ from ytdl_sub.utils.file_handler import FileHandler from ytdl_sub.utils.file_handler import FileHandlerTransactionLog from ytdl_sub.utils.file_handler import FileMetadata from ytdl_sub.utils.logger import Logger -from ytdl_sub.validators.string_formatter_validators import OverridesStandardizedDateValidator logger = Logger.get("archive") @@ -64,21 +63,19 @@ class DownloadMapping: ) @classmethod - def from_entry(cls, entry: Entry, entry_date: str) -> "DownloadMapping": + def from_entry(cls, entry: Entry) -> "DownloadMapping": """ Parameters ---------- entry Entry to create a download mapping for - entry_date: - Date to use for the entry Returns ------- DownloadMapping for the entry """ return DownloadMapping( - upload_date=entry_date, + upload_date=entry.get(v.ytdl_sub_entry_date_eval, str), extractor=entry.download_archive_extractor, file_names=set(), ) @@ -154,25 +151,19 @@ class DownloadArchive: class DownloadMappings: _strptime_format = "%Y-%m-%d" - def __init__(self, entry_date_eval: Optional[OverridesStandardizedDateValidator]): + def __init__(self): """ Initializes an empty mapping - - entry_date_eval - Which date to use for download mapping logging """ - self._entry_date_eval = entry_date_eval self._entry_mappings: Dict[str, DownloadMapping] = {} @classmethod - def from_file(cls, json_file_path: str, entry_date_eval: Optional[OverridesStandardizedDateValidator]) -> "DownloadMappings": + def from_file(cls, json_file_path: str) -> "DownloadMappings": """ Parameters ---------- json_file_path Path to a json file that contains download mappings - entry_date_eval - Which date to use for download mapping logging Returns ------- @@ -186,7 +177,7 @@ class DownloadMappings: mapping_dict=entry_mappings_json[uid] ) - download_mappings = DownloadMappings(entry_date_eval=entry_date_eval) + download_mappings = DownloadMappings() download_mappings._entry_mappings = entry_mappings_json return download_mappings @@ -227,6 +218,7 @@ class DownloadMappings: Entry that this file belongs to entry_file_path Relative path to the file that lives in the output directory + Returns ------- self @@ -236,15 +228,7 @@ class DownloadMappings: uid = parent_uid if uid not in self.entry_ids: - entry_date = entry.get(v.upload_date_standardized, str) - if self._entry_date_eval is not None: - entry_date = entry - elif self._entry_date_eval.is_release_date: - entry_date = entry.get(v.release_date_standardized, str) - else: - raise AssertionError("Unsupported entry_date_eval. Should not reach.") - - self._entry_mappings[uid] = DownloadMapping.from_entry(entry=entry, entry_date=entry_date) + self._entry_mappings[uid] = DownloadMapping.from_entry(entry=entry) self._entry_mappings[uid].file_names.add(entry_file_path) return self @@ -386,7 +370,7 @@ class EnhancedDownloadArchive: @classmethod def _maybe_load_download_mappings( - cls, mapping_file_path: str, migrated_mapping_file_path: Optional[str], entry_date_eval: Optional[OverridesStandardizedDateValidator] + cls, mapping_file_path: str, migrated_mapping_file_path: Optional[str] ) -> DownloadMappings: """ Tries to load download mappings if a file exists. Otherwise returns empty mappings. @@ -398,22 +382,21 @@ class EnhancedDownloadArchive: "`output_options.migrated_download_archive` to " "`output_options.download_archive`" ) - return DownloadMappings.from_file(json_file_path=migrated_mapping_file_path, entry_date_eval=entry_date_eval) + return DownloadMappings.from_file(migrated_mapping_file_path) logger.warning( "MIGRATION DETECTED, will write archive file to %s", migrated_mapping_file_path ) if os.path.isfile(mapping_file_path): - return DownloadMappings.from_file(json_file_path=mapping_file_path, entry_date_eval=entry_date_eval) - return DownloadMappings(entry_date_eval=entry_date_eval) + return DownloadMappings.from_file(json_file_path=mapping_file_path) + return DownloadMappings() def __init__( self, file_name: str, working_directory: str, output_directory: str, - entry_date_eval: Optional[OverridesStandardizedDateValidator], dry_run: bool = False, migrated_file_name: Optional[str] = None, ): @@ -421,9 +404,7 @@ class EnhancedDownloadArchive: self._file_handler = FileHandler( working_directory=working_directory, output_directory=output_directory, dry_run=dry_run ) - self._entry_date_eval = entry_date_eval - # gets reinitialized - self._download_mapping = DownloadMappings(entry_date_eval=entry_date_eval) + self._download_mapping = DownloadMappings() # gets reinitialized self._migrated_file_name = migrated_file_name self.num_entries_added: int = 0 @@ -461,7 +442,6 @@ class EnhancedDownloadArchive: self._download_mapping = self._maybe_load_download_mappings( mapping_file_path=self._output_file_path, migrated_mapping_file_path=self._migrated_file_path, - entry_date_eval=self._entry_date_eval, ) return self