added_variables by validation

src/ytdl_sub/config/plugin/plugin_operation.py

@@ -2,6 +2,7 @@ from enum import Enum
 
 
 class PluginOperation(Enum):
+    ANY = -2
     DOWNLOADER = -1
     MODIFY_ENTRY_METADATA = 0
     MODIFY_ENTRY = 1

src/ytdl_sub/config/validators/options.py

@@ -41,7 +41,10 @@ class OptionsValidator(Validator, ABC):
         return {}
 
     def added_variables(
-        self, resolved_variables: Set[str], unresolved_variables: Set[str]
+        self,
+        resolved_variables: Set[str],
+        unresolved_variables: Set[str],
+        plugin_op: PluginOperation,
     ) -> Dict[PluginOperation, Set[str]]:
         """
         If the plugin adds source variables, list them here.

src/ytdl_sub/config/validators/variable_validation.py

@@ -28,7 +28,7 @@ def _add_dummy_variables(variables: Iterable[str]) -> Dict[str, str]:
 
 
 def _get_added_and_modified_variables(
-    plugins: PresetPlugins, downloader_options: MultiUrlValidator, resolved_variables: Set[str]
+    plugins: PresetPlugins, downloader_options: MultiUrlValidator
 ) -> Iterable[Tuple[OptionsValidator, Set[str], Set[str]]]:
     """
     Iterates and returns the plugin options, added variables, modified variables
@@ -41,8 +41,9 @@ def _get_added_and_modified_variables(
         modified_variables: Set[str] = set()
 
         for plugin_added_variables in plugin_options.added_variables(
-            resolved_variables=resolved_variables,
+            resolved_variables=set(),
             unresolved_variables=set(),
+            plugin_op=PluginOperation.ANY,
         ).values():
             added_variables |= set(plugin_added_variables)
 
@@ -93,7 +94,6 @@ class VariableValidation:
         ) in _get_added_and_modified_variables(
             plugins=self.plugins,
             downloader_options=self.downloader_options,
-            resolved_variables=self.resolved_variables,
         ):
 
             for added_variable in added_variables:
@@ -129,12 +129,14 @@ class VariableValidation:
         added_variables = options.added_variables(
             resolved_variables=self.resolved_variables,
             unresolved_variables=self.unresolved_variables,
+            plugin_op=plugin_op,
         ).get(plugin_op, set())
         modified_variables = options.modified_variables().get(plugin_op, set())
 
         resolved_variables = added_variables | modified_variables
 
         self.script.add(_add_dummy_variables(resolved_variables))
+        self.resolved_variables |= resolved_variables
         self.unresolved_variables -= resolved_variables
 
         return added_variables

src/ytdl_sub/downloaders/url/validators.py

@@ -6,6 +6,7 @@ from typing import Set
 
 from ytdl_sub.config.plugin.plugin_operation import PluginOperation
 from ytdl_sub.config.validators.options import OptionsValidator
+from ytdl_sub.script.parser import parse
 from ytdl_sub.validators.strict_dict_validator import StrictDictValidator
 from ytdl_sub.validators.string_formatter_validators import DictFormatterValidator
 from ytdl_sub.validators.string_formatter_validators import OverridesStringFormatterValidator
@@ -245,11 +246,25 @@ class MultiUrlValidator(OptionsValidator):
         return self._urls.list[0].variables
 
     def added_variables(
-        self, resolved_variables: Set[str], unresolved_variables: Set[str]
+        self,
+        resolved_variables: Set[str],
+        unresolved_variables: Set[str],
+        plugin_op: PluginOperation,
     ) -> Dict[PluginOperation, Set[str]]:
         """
         Returns
         -------
         List of variables added. The first collection url always contains all the variables.
         """
+        if plugin_op != PluginOperation.ANY:
+            for url in self._urls.list:
+                for variable_name, definition in url.variables.dict_with_format_strings.items():
+                    used_variables = set(var.name for var in parse(definition).variables)
+                    if unresolved := used_variables & unresolved_variables:
+                        raise self._validation_exception(
+                            f"variable {variable_name} cannot use the variables "
+                            f"{', '.join(sorted(list(unresolved)))} because they depend on other"
+                            " variables that are computed later in execution"
+                        )
+
         return {PluginOperation.DOWNLOADER: set(self._urls.list[0].variables.keys)}

src/ytdl_sub/plugins/chapters.py

@@ -197,7 +197,10 @@ class ChaptersOptions(OptionsDictValidator):
         return self._force_key_frames
 
     def added_variables(
-        self, resolved_variables: Set[str], unresolved_variables: Set[str]
+        self,
+        resolved_variables: Set[str],
+        unresolved_variables: Set[str],
+        plugin_op: PluginOperation,
     ) -> Dict[PluginOperation, Set[str]]:
         return {PluginOperation.MODIFY_ENTRY: {ytdl_sub_chapters_from_comments.variable_name}}
 

src/ytdl_sub/plugins/regex.py

@@ -203,6 +203,9 @@ class RegexOptions(OptionsDictValidator):
             key="skip_if_match_fails", validator=BoolValidator, default=True
         ).value
 
+        # Variables added by the regex plugin
+        self._added_variable_names: Set[str] = set()
+
     @property
     def skip_if_match_fails(self) -> Optional[bool]:
         """
@@ -221,7 +224,7 @@ class RegexOptions(OptionsDictValidator):
         return self._from.variable_capture_dict
 
     @classmethod
-    def _can_evaluate_at_metadata_time(
+    def _can_resolve(
         cls, unresolved_variables: Set[str], input_variable_name: str, regex_options: VariableRegex
     ) -> bool:
         if input_variable_name in unresolved_variables:
@@ -233,7 +236,10 @@ class RegexOptions(OptionsDictValidator):
         return True
 
     def added_variables(
-        self, resolved_variables: Set[str], unresolved_variables: Set[str]
+        self,
+        resolved_variables: Set[str],
+        unresolved_variables: Set[str],
+        plugin_op: PluginOperation,
     ) -> Dict[PluginOperation, Set[str]]:
         """
         Returns
@@ -245,28 +251,43 @@ class RegexOptions(OptionsDictValidator):
             PluginOperation.MODIFY_ENTRY: set(),
         }
         for input_variable_name, regex_options in self.source_variable_capture_dict.items():
+            variables_to_add = set(regex_options.capture_group_names)
 
-            if input_variable_name not in resolved_variables:
+            if plugin_op != PluginOperation.ANY and input_variable_name not in (
+                resolved_variables | unresolved_variables
+            ):
                 raise self._validation_exception(
-                    f"cannot regex capture '{input_variable_name}' because it is not a "
-                    f"defined variable"
+                    f"cannot regex capture '{input_variable_name}' because it is not a"
+                    " defined variable."
+                )
+            if (
+                plugin_op.value >= PluginOperation.MODIFY_ENTRY.value
+                and input_variable_name in unresolved_variables
+            ):
+                raise self._validation_exception(
+                    f"cannot regex capture '{input_variable_name}' because it is not "
+                    f"computed until later in execution."
                 )
 
-            key = PluginOperation.MODIFY_ENTRY
-            if self._can_evaluate_at_metadata_time(
+            if plugin_op == PluginOperation.ANY:
+                added_source_vars[PluginOperation.MODIFY_ENTRY_METADATA] |= variables_to_add
+                self._added_variable_names |= variables_to_add
+                continue
+
+            if not self._can_resolve(
                 unresolved_variables=unresolved_variables,
                 input_variable_name=input_variable_name,
                 regex_options=regex_options,
             ):
-                key = PluginOperation.MODIFY_ENTRY_METADATA
+                continue
 
             for capture_group_name in regex_options.capture_group_names:
-                if capture_group_name in resolved_variables:
+                if capture_group_name in (resolved_variables - self._added_variable_names):
                     raise self._validation_exception(
                         f"cannot use '{capture_group_name}' as a capture group name because it is "
-                        f"a defined variable"
+                        f"an already defined variable."
                     )
-            added_source_vars[key] |= set(regex_options.capture_group_names)
+            added_source_vars[plugin_op] |= set(regex_options.capture_group_names)
 
         return added_source_vars
 

src/ytdl_sub/plugins/split_by_chapters.py

@@ -82,7 +82,10 @@ class SplitByChaptersOptions(OptionsDictValidator):
         ).value
 
     def added_variables(
-        self, resolved_variables: Set[str], unresolved_variables: Set[str]
+        self,
+        resolved_variables: Set[str],
+        unresolved_variables: Set[str],
+        plugin_op: PluginOperation,
     ) -> Dict[PluginOperation, Set[str]]:
         return {
             PluginOperation.MODIFY_ENTRY: {

src/ytdl_sub/plugins/subtitles.py

@@ -120,7 +120,10 @@ class SubtitleOptions(OptionsDictValidator):
         return self._allow_auto_generated_subtitles
 
     def added_variables(
-        self, resolved_variables: Set[str], unresolved_variables: Set[str]
+        self,
+        resolved_variables: Set[str],
+        unresolved_variables: Set[str],
+        plugin_op: PluginOperation,
     ) -> Dict[PluginOperation, Set[str]]:
         """
         Returns

tests/e2e/plugins/test_split_by_chapters.py

@@ -34,7 +34,7 @@ def yt_album_as_chapters_with_regex_preset_dict(yt_album_as_chapters_preset_dict
                 "from": {
                     # Ensure regex can handle override variables that come from the
                     # post-metadata stage
-                    "override_chapter_title": {
+                    "chapter_title": {
                         "match": r"\d+\. (.+)",
                         "capture_group_names": "captured_track_title",
                         "capture_group_defaults": "{chapter_title}",
@@ -56,7 +56,6 @@ def yt_album_as_chapters_with_regex_preset_dict(yt_album_as_chapters_preset_dict
                 }
             },
             "overrides": {
-                "override_chapter_title": "{chapter_title}",
                 "track_title": "{captured_track_title}",
                 "track_album": "{captured_track_album}",
                 "track_artist": "{captured_track_artist}",