almost regex ready

src/ytdl_sub/config/plugin/preset_plugins.py

@@ -1,7 +1,11 @@
-from typing import List, Type, Tuple, Optional
+from typing import List
+from typing import Optional
+from typing import Tuple
+from typing import Type
 
 from ytdl_sub.config.plugin.plugin import Plugin
-from ytdl_sub.config.validators.options import OptionsValidator, TOptionsValidator
+from ytdl_sub.config.validators.options import OptionsValidator
+from ytdl_sub.config.validators.options import TOptionsValidator
 
 
 class PresetPlugins:

src/ytdl_sub/config/validators/variable_validation.py

@@ -12,93 +12,116 @@ from ytdl_sub.config.validators.options import OptionsValidator
 from ytdl_sub.downloaders.url.validators import MultiUrlValidator
 from ytdl_sub.entries.script.variable_scripts import VARIABLE_SCRIPTS
 from ytdl_sub.script.script import Script
+from ytdl_sub.utils.exceptions import ValidationException
 from ytdl_sub.utils.script import ScriptUtils
 from ytdl_sub.validators.string_formatter_validators import validate_formatters
 
 
+def _get_added_variables(plugins: PresetPlugins, downloader_options: MultiUrlValidator) -> Set[str]:
+    added_variables: Set[str] = set()
+    options: List[OptionsValidator] = plugins.plugin_options
+    options.append(downloader_options)
+
+    for plugin_options in options:
+        for plugin_added_variables in plugin_options.added_source_variables(
+            unresolved_variables=set()
+        ).values():
+            added_variables |= set(plugin_added_variables)
+
+    return added_variables
+
+
+def _override_variables(overrides: Overrides) -> Set[str]:
+    return set(list(overrides.initial_variables(unresolved_variables={}).keys()))
+
+
+def _entry_variables() -> Set[str]:
+    return set(list(VARIABLE_SCRIPTS.keys()))
+
+
 class VariableValidation:
-    @classmethod
-    def _get_added_variables(
-        cls, plugins: PresetPlugins, downloader_options: MultiUrlValidator
-    ) -> Set[str]:
-        added_variables: Set[str] = set()
-        options: List[OptionsValidator] = plugins.plugin_options
-        options.append(downloader_options)
-
-        for plugin_options in options:
-            for plugin_added_variables in plugin_options.added_source_variables(
-                unresolved_variables=set()
-            ).values():
-                added_variables |= set(plugin_added_variables)
-
-        return added_variables
-
     def __init__(
         self,
         downloader_options: MultiUrlValidator,
         output_options: OutputOptions,
         plugins: PresetPlugins,
     ):
-        self.script: Optional[Script] = None
-        self.resolved_variables: Set[str] = set()
-
         self.downloader_options = downloader_options
         self.output_options = output_options
         self.plugins = plugins
-        self.unresolved_variables = VariableValidation._get_added_variables(
+
-            plugins=plugins, downloader_options=downloader_options
+        self.script: Optional[Script] = None
-        )
+        self.resolved_variables: Set[str] = set()
+        self.unresolved_variables: Set[str] = set()
 
     def initialize_overrides(self, overrides: Overrides) -> "VariableValidation":
+        override_variables = _override_variables(overrides)
+        entry_variables = _entry_variables()
+
+        # Set unresolved as variables that are added but do not exist as entry/override variables
+        self.unresolved_variables = (
+            _get_added_variables(plugins=self.plugins, downloader_options=self.downloader_options)
+            - override_variables
+            - entry_variables
+        )
+
+        # Initialize overrides with unresolved variables to throw an error
         overrides = overrides.initialize_script(
             unresolved_variables={
                 var_name: f"{{%throw('Plugin variable {var_name} has not been created yet')}}"
                 for var_name in self.unresolved_variables
             }
         )
+
+        # copy the script and mock entry variables
         self.script = copy.deepcopy(overrides.script).add(
-            ScriptUtils.add_dummy_variables(list(VARIABLE_SCRIPTS.keys()))
+            ScriptUtils.add_dummy_variables(entry_variables)
         )
         self.resolved_variables = self.script.variable_names - self.unresolved_variables
 
         return self
 
+    def _update_script(self) -> None:
+        _ = self.script.resolve(unresolvable=self.unresolved_variables, update=True)
+
+    def _add_variables(self, plugin_op: PluginOperation, options: OptionsValidator) -> Set[str]:
+        added_variables = options.added_source_variables(
+            unresolved_variables=self.unresolved_variables
+        ).get(plugin_op, set())
+
+        if added_variables:
+            for added_variable in added_variables:
+                if added_variable in self.resolved_variables:
+                    raise ValidationException(
+                        f"Tried added the variable '{added_variable}', but it already "
+                        f"exists as a defined variable."
+                    )
+
+            self.script.add(ScriptUtils.add_dummy_variables(added_variables))
+            self.unresolved_variables -= added_variables
+
+        return added_variables
+
     def ensure_proper_usage(self) -> None:
         """
         Validate variables resolve as plugins are executed, and return
         a mock script which contains actualized added variables from the plugins
         """
+        self._add_variables(PluginOperation.DOWNLOADER, options=self.downloader_options)
 
-        added_variables: Set[str] = self.downloader_options.added_source_variables(
+        # Metadata variables to be added
-            self.unresolved_variables
-        ).get(PluginOperation.DOWNLOADER, set())
-        self.script.add(ScriptUtils.add_dummy_variables(added_variables))
-        self.unresolved_variables -= added_variables
-
         for plugin_options in PluginMapping.order_options_by(
             self.plugins.zipped(), PluginOperation.MODIFY_ENTRY_METADATA
         ):
-            added_variables = plugin_options.added_source_variables(
+            self._add_variables(PluginOperation.MODIFY_ENTRY_METADATA, options=plugin_options)
-                unresolved_variables=self.unresolved_variables
-            ).get(PluginOperation.MODIFY_ENTRY_METADATA, set())
 
-            if added_variables:
+        self._update_script()
-                self.script.add(ScriptUtils.add_dummy_variables(added_variables))
-                self.unresolved_variables -= added_variables
-
-        _ = self.script.resolve(unresolvable=self.unresolved_variables, update=True)
         for plugin_options in PluginMapping.order_options_by(
             self.plugins.zipped(), PluginOperation.MODIFY_ENTRY
         ):
-            added_variables = plugin_options.added_source_variables(
+            added = self._add_variables(PluginOperation.MODIFY_ENTRY, options=plugin_options)
-                unresolved_variables=self.unresolved_variables
+            if added:
-            ).get(PluginOperation.MODIFY_ENTRY, set())
+                self._update_script()
-
-            if added_variables:
-                self.script.add(ScriptUtils.add_dummy_variables(added_variables))
-                self.unresolved_variables -= added_variables
-
-                _ = self.script.resolve(unresolvable=self.unresolved_variables, update=True)
 
             # Validate that any formatter in the plugin options can resolve
             validate_formatters(

src/ytdl_sub/subscriptions/base_subscription.py

@@ -4,8 +4,8 @@ from typing import Optional
 
 from ytdl_sub.config.config_validator import ConfigOptions
 from ytdl_sub.config.overrides import Overrides
-from ytdl_sub.config.preset import Preset
 from ytdl_sub.config.plugin.preset_plugins import PresetPlugins
+from ytdl_sub.config.preset import Preset
 from ytdl_sub.config.preset_options import OutputOptions
 from ytdl_sub.config.preset_options import YTDLOptions
 from ytdl_sub.downloaders.url.validators import MultiUrlValidator

tests/e2e/plugins/test_regex.py

@@ -302,32 +302,39 @@ class TestRegex:
                 preset_dict=regex_subscription_dict,
             )
 
-    def test_regex_fails_capture_group_is_source_variable(
+    def test_regex_fails_capture_group_is_entry_variable(
         self, regex_subscription_dict, default_config
     ):
-        regex_subscription_dict["regex"]["from"]["title"]["capture_group_names"][0] = "uid"
+        regex_subscription_dict["regex"]["from"]["playlist_id"] = {
+            "match": [".*http:\\/\\/(.+).com.*"],
+            "capture_group_names": ["uid"],
+        }
+
         with pytest.raises(
             ValidationException,
             match=re.escape(
-                "'uid' cannot be used as a capture group name because it is a source variable"
+                "Tried added the variable 'uid', but it already exists as a defined variable"
             ),
         ):
             _ = Subscription.from_dict(
                 config=default_config,
-                preset_name="test_regex_fails_capture_group_is_source_variable",
+                preset_name="test_regex_fails_capture_group_is_entry_variable",
                 preset_dict=regex_subscription_dict,
             )
 
     def test_regex_fails_capture_group_is_override_variable(
         self, regex_subscription_dict, default_config
     ):
-        regex_subscription_dict["regex"]["from"]["title"]["capture_group_names"][
+        regex_subscription_dict["regex"]["from"]["playlist_id"] = {
-            0
+            "match": [".*http:\\/\\/(.+).com.*"],
-        ] = "in_regex_default"
+            "capture_group_names": ["contains_regex_default"],
+        }
+
         with pytest.raises(
             ValidationException,
             match=re.escape(
-                "'in_regex_default' cannot be used as a capture group name because it is an override variable"
+                "Tried added the variable 'contains_regex_default', but it already exists "
+                "as a defined variable"
             ),
         ):
             _ = Subscription.from_dict(