working?

src/ytdl_sub/config/preset_options.py

@@ -1,16 +1,19 @@
 from typing import Any
 from typing import Dict
 from typing import Optional
+from typing import Set
 
 from ytdl_sub.config.defaults import DEFAULT_DOWNLOAD_ARCHIVE_NAME
 from ytdl_sub.config.overrides import Overrides
+from ytdl_sub.config.plugin.plugin_operation import PluginOperation
+from ytdl_sub.config.validators.options import OptionsDictValidator
+from ytdl_sub.entries.script.variable_definitions import VARIABLES as v
 from ytdl_sub.validators.file_path_validators import OverridesStringFormatterFilePathValidator
 from ytdl_sub.validators.file_path_validators import StringFormatterFileNameValidator
-from ytdl_sub.validators.strict_dict_validator import StrictDictValidator
 from ytdl_sub.validators.string_datetime import StringDatetimeValidator
-from ytdl_sub.validators.string_formatter_validators import OverridesIntegerFormatterValidator, \
+from ytdl_sub.validators.string_formatter_validators import OverridesIntegerFormatterValidator
-    OverridesStandardizedDateValidator
 from ytdl_sub.validators.string_formatter_validators import OverridesStringFormatterValidator
+from ytdl_sub.validators.string_formatter_validators import StandardizedDateValidator
 from ytdl_sub.validators.string_formatter_validators import StringFormatterValidator
 from ytdl_sub.validators.string_formatter_validators import (
     UnstructuredOverridesDictFormatterValidator,
@@ -66,7 +69,7 @@ class YTDLOptions(UnstructuredOverridesDictFormatterValidator):
 # pylint: disable=line-too-long
 
 
-class OutputOptions(StrictDictValidator):
+class OutputOptions(OptionsDictValidator):
     """
     Defines where to output files and thumbnails after all post-processing has completed.
 
@@ -89,7 +92,7 @@ class OutputOptions(StrictDictValidator):
              keep_files_before: now
              keep_files_after: 19000101
              keep_max_files: 1000
-             keep_files_date_eval: "upload_date"
+             keep_files_date_eval: "{upload_date_standardized}"
     """
 
     _required_keys = {"output_directory", "file_name"}
@@ -103,6 +106,7 @@ class OutputOptions(StrictDictValidator):
         "keep_files_after",
         "keep_max_files",
         "download_archive_standardized_date",
+        "entry_date_eval",
     }
 
     @classmethod
@@ -160,8 +164,10 @@ class OutputOptions(StrictDictValidator):
         self._keep_max_files = self._validate_key_if_present(
             "keep_max_files", OverridesIntegerFormatterValidator
         )
-        self._entry_date_eval = self._validate_key_if_present(
+        self._entry_date_eval = self._validate_key(
-            "entry_date_eval", OverridesStandardizedDateValidator
+            "entry_date_eval",
+            StandardizedDateValidator,
+            default=f"{{{v.upload_date_standardized.variable_name}}}"
         )
 
         if (
@@ -280,7 +286,7 @@ class OutputOptions(StrictDictValidator):
         return self._keep_files_after
 
     @property
-    def entry_date_eval(self) -> Optional[OverridesStandardizedDateValidator]:
+    def entry_date_eval(self) -> StandardizedDateValidator:
         """
         :expected type: str
         :description:
@@ -302,3 +308,10 @@ class OutputOptions(StrictDictValidator):
           applied. Can be used in conjunction with ``keep_files_before`` and ``keep_files_after``.
         """
         return self._keep_max_files
+
+    def added_variables(self, unresolved_variables: Set[str]) -> Dict[PluginOperation, Set[str]]:
+        return {
+            # PluginOperation.MODIFY_ENTRY_METADATA: {
+            #     VARIABLES.ytdl_sub_entry_date_eval.variable_name
+            # }
+        }

src/ytdl_sub/config/validators/variable_validation.py

@@ -57,13 +57,14 @@ def _add_dummy_overrides(overrides: Overrides) -> Dict[str, str]:
 
 
 def _get_added_and_modified_variables(
-    plugins: PresetPlugins, downloader_options: MultiUrlValidator
+    plugins: PresetPlugins, downloader_options: MultiUrlValidator, output_options: OutputOptions
 ) -> Iterable[Tuple[OptionsValidator, Set[str], Set[str]]]:
     """
     Iterates and returns the plugin options, added variables, modified variables
     """
     options: List[OptionsValidator] = plugins.plugin_options
     options.append(downloader_options)
+    options.append(output_options)
 
     for plugin_options in options:
         added_variables: Set[str] = set()
@@ -117,6 +118,7 @@ class VariableValidation:
         ) in _get_added_and_modified_variables(
             plugins=self.plugins,
             downloader_options=self.downloader_options,
+            output_options=self.output_options,
         ):
 
             for added_variable in added_variables:
@@ -183,6 +185,9 @@ class VariableValidation:
         self._add_variables(PluginOperation.DOWNLOADER, options=self.downloader_options)
         self._add_subscription_override_variables()
 
+        # Always add output options first
+        self._add_variables(PluginOperation.MODIFY_ENTRY_METADATA, options=self.output_options)
+
         # Metadata variables to be added
         for plugin_options in PluginMapping.order_options_by(
             self.plugins.zipped(), PluginOperation.MODIFY_ENTRY_METADATA

src/ytdl_sub/entries/script/variable_definitions.py

@@ -827,9 +827,12 @@ class YtdlSubVariableDefinitions(ABC):
     def ytdl_sub_entry_date_eval(self: "VariableDefinitions") -> StringVariable:
         """
         :description:
-          The standardized
+          The standardized date variable supplied in ``output_options.entry_date_eval``
         """
-        return StringVariable(variable_name="ytdl_sub_input_url", definition="{ %string('') }")
+        return StringVariable(
+            variable_name="ytdl_sub_entry_date_eval",
+            definition=f"{{%string({self.upload_date_standardized.variable_name})}}",
+        )
 
 
 class EntryVariableDefinitions(ABC):
@@ -1129,6 +1132,7 @@ class VariableDefinitions(
             self.ytdl_sub_input_url,
             self.ytdl_sub_input_url_index,
             self.ytdl_sub_input_url_count,
+            self.ytdl_sub_entry_date_eval,
         }
 
     @cache

src/ytdl_sub/subscriptions/base_subscription.py

@@ -31,7 +31,6 @@ def _initialize_download_archive(
         file_name=overrides.apply_formatter(output_options.download_archive_name),
         working_directory=working_directory,
         output_directory=output_directory,
-        entry_date_eval=output_options.entry_date_eval,
         migrated_file_name=migrated_file_name,
     ).reinitialize(dry_run=True)
 

src/ytdl_sub/subscriptions/subscription_download.py

@@ -17,6 +17,7 @@ from ytdl_sub.downloaders.source_plugin import SourcePlugin
 from ytdl_sub.downloaders.url.downloader import MultiUrlDownloader
 from ytdl_sub.downloaders.ytdl_options_builder import YTDLOptionsBuilder
 from ytdl_sub.entries.entry import Entry
+from ytdl_sub.entries.script.variable_definitions import VARIABLES
 from ytdl_sub.subscriptions.base_subscription import BaseSubscription
 from ytdl_sub.subscriptions.subscription_ytdl_options import SubscriptionYTDLOptions
 from ytdl_sub.utils.datetime import to_date_range
@@ -215,9 +216,19 @@ class SubscriptionDownload(BaseSubscription, ABC):
         FileHandler.delete(entry.get_download_thumbnail_path())
         FileHandler.delete(entry.get_download_info_json_path())
 
-    @classmethod
+    def _preprocess_entry(self, plugins: List[Plugin], entry: Entry) -> Optional[Entry]:
-    def _preprocess_entry(cls, plugins: List[Plugin], entry: Entry) -> Optional[Entry]:
         maybe_entry: Optional[Entry] = entry
+
+        # Inject OutputOption variables here
+        entry.add(
+            {VARIABLES.ytdl_sub_entry_date_eval: self.output_options.entry_date_eval.format_string}
+        )
+
+        # Run it to make sure it's actually a standardized date
+        _ = self.overrides.apply_formatter(
+            formatter=self.output_options.entry_date_eval, entry=entry
+        )
+
         for plugin in PluginMapping.order_plugins_by(
             plugins, PluginOperation.MODIFY_ENTRY_METADATA
         ):

src/ytdl_sub/validators/string_formatter_validators.py

@@ -74,6 +74,20 @@ class StringFormatterValidator(StringValidator):
         return resolved
 
 
+class StandardizedDateValidator(StringFormatterValidator):
+    _expected_value_type_name = "standardized_date"
+
+    def post_process(self, resolved: str) -> str:
+        try:
+            datetime.strptime(resolved, "%Y-%m-%d")
+        except ValueError as exc:
+            raise self._validation_exception(
+                f"Expected a standardized date in the form of YYYY-MM-DD, but received '{resolved}'"
+            ) from exc
+
+        return resolved
+
+
 # pylint: disable=line-too-long
 class OverridesStringFormatterValidator(StringFormatterValidator):
     """
@@ -104,19 +118,6 @@ class OverridesIntegerFormatterValidator(OverridesStringFormatterValidator):
         return resolved
 
 
-class OverridesStandardizedDateValidator(OverridesStringFormatterValidator):
-    _expected_value_type_name = "standardized_date"
-
-    def post_process(self, resolved: str) -> str:
-        try:
-            datetime.strptime(resolved, "%Y-%m-%d")
-        except ValueError as exc:
-            raise self._validation_exception(
-                f"Expected a standardized date in the form of YYYY-MM-DD, but received '{resolved}'"
-            ) from exc
-
-        return resolved
-
 class OverridesBooleanFormatterValidator(OverridesStringFormatterValidator):
     _expected_value_type_name = "boolean"
 

src/ytdl_sub/ytdl_additions/enhanced_download_archive.py

@@ -21,7 +21,6 @@ from ytdl_sub.utils.file_handler import FileHandler
 from ytdl_sub.utils.file_handler import FileHandlerTransactionLog
 from ytdl_sub.utils.file_handler import FileMetadata
 from ytdl_sub.utils.logger import Logger
-from ytdl_sub.validators.string_formatter_validators import OverridesStandardizedDateValidator
 
 logger = Logger.get("archive")
 
@@ -64,21 +63,19 @@ class DownloadMapping:
         )
 
     @classmethod
-    def from_entry(cls, entry: Entry, entry_date: str) -> "DownloadMapping":
+    def from_entry(cls, entry: Entry) -> "DownloadMapping":
         """
         Parameters
         ----------
         entry
             Entry to create a download mapping for
-        entry_date:
-            Date to use for the entry
 
         Returns
         -------
         DownloadMapping for the entry
         """
         return DownloadMapping(
-            upload_date=entry_date,
+            upload_date=entry.get(v.ytdl_sub_entry_date_eval, str),
             extractor=entry.download_archive_extractor,
             file_names=set(),
         )
@@ -154,25 +151,19 @@ class DownloadArchive:
 class DownloadMappings:
     _strptime_format = "%Y-%m-%d"
 
-    def __init__(self, entry_date_eval: Optional[OverridesStandardizedDateValidator]):
+    def __init__(self):
         """
         Initializes an empty mapping
-
-        entry_date_eval
-            Which date to use for download mapping logging
         """
-        self._entry_date_eval = entry_date_eval
         self._entry_mappings: Dict[str, DownloadMapping] = {}
 
     @classmethod
-    def from_file(cls, json_file_path: str, entry_date_eval: Optional[OverridesStandardizedDateValidator]) -> "DownloadMappings":
+    def from_file(cls, json_file_path: str) -> "DownloadMappings":
         """
         Parameters
         ----------
         json_file_path
             Path to a json file that contains download mappings
-        entry_date_eval
-            Which date to use for download mapping logging
 
         Returns
         -------
@@ -186,7 +177,7 @@ class DownloadMappings:
                 mapping_dict=entry_mappings_json[uid]
             )
 
-        download_mappings = DownloadMappings(entry_date_eval=entry_date_eval)
+        download_mappings = DownloadMappings()
         download_mappings._entry_mappings = entry_mappings_json
         return download_mappings
 
@@ -227,6 +218,7 @@ class DownloadMappings:
             Entry that this file belongs to
         entry_file_path
             Relative path to the file that lives in the output directory
+
         Returns
         -------
         self
@@ -236,15 +228,7 @@ class DownloadMappings:
             uid = parent_uid
 
         if uid not in self.entry_ids:
-            entry_date = entry.get(v.upload_date_standardized, str)
+            self._entry_mappings[uid] = DownloadMapping.from_entry(entry=entry)
-            if self._entry_date_eval is not None:
-                entry_date = entry
-            elif self._entry_date_eval.is_release_date:
-                entry_date = entry.get(v.release_date_standardized, str)
-            else:
-                raise AssertionError("Unsupported entry_date_eval. Should not reach.")
-
-            self._entry_mappings[uid] = DownloadMapping.from_entry(entry=entry, entry_date=entry_date)
 
         self._entry_mappings[uid].file_names.add(entry_file_path)
         return self
@@ -386,7 +370,7 @@ class EnhancedDownloadArchive:
 
     @classmethod
     def _maybe_load_download_mappings(
-        cls, mapping_file_path: str, migrated_mapping_file_path: Optional[str], entry_date_eval: Optional[OverridesStandardizedDateValidator]
+        cls, mapping_file_path: str, migrated_mapping_file_path: Optional[str]
     ) -> DownloadMappings:
         """
         Tries to load download mappings if a file exists. Otherwise returns empty mappings.
@@ -398,22 +382,21 @@ class EnhancedDownloadArchive:
                     "`output_options.migrated_download_archive` to "
                     "`output_options.download_archive`"
                 )
-                return DownloadMappings.from_file(json_file_path=migrated_mapping_file_path, entry_date_eval=entry_date_eval)
+                return DownloadMappings.from_file(migrated_mapping_file_path)
 
             logger.warning(
                 "MIGRATION DETECTED, will write archive file to %s", migrated_mapping_file_path
             )
 
         if os.path.isfile(mapping_file_path):
-            return DownloadMappings.from_file(json_file_path=mapping_file_path, entry_date_eval=entry_date_eval)
+            return DownloadMappings.from_file(json_file_path=mapping_file_path)
-        return DownloadMappings(entry_date_eval=entry_date_eval)
+        return DownloadMappings()
 
     def __init__(
         self,
         file_name: str,
         working_directory: str,
         output_directory: str,
-        entry_date_eval: Optional[OverridesStandardizedDateValidator],
         dry_run: bool = False,
         migrated_file_name: Optional[str] = None,
     ):
@@ -421,9 +404,7 @@ class EnhancedDownloadArchive:
         self._file_handler = FileHandler(
             working_directory=working_directory, output_directory=output_directory, dry_run=dry_run
         )
-        self._entry_date_eval = entry_date_eval
+        self._download_mapping = DownloadMappings()  # gets reinitialized
-        # gets reinitialized
-        self._download_mapping = DownloadMappings(entry_date_eval=entry_date_eval)
         self._migrated_file_name = migrated_file_name
 
         self.num_entries_added: int = 0
@@ -461,7 +442,6 @@ class EnhancedDownloadArchive:
         self._download_mapping = self._maybe_load_download_mappings(
             mapping_file_path=self._output_file_path,
             migrated_mapping_file_path=self._migrated_file_path,
-            entry_date_eval=self._entry_date_eval,
         )
         return self