somehow working

src/ytdl_sub/config/overrides.py

@@ -127,8 +127,8 @@ class Overrides(DictFormatterValidator, Scriptable):
         if function_overrides:
             script.add(function_overrides)
 
-        return str(
+        return (
-            script.add({"tmp_var": formatter.format_string}).resolve(unresolvable=unresolvable)[
+            script.add({"tmp_var": formatter.format_string})
-                "tmp_var"
+            .resolve(unresolvable=unresolvable)
-            ]
+            .get_str("tmp_var")
         )

src/ytdl_sub/config/preset.py

@@ -27,10 +27,12 @@ from ytdl_sub.entries.script.variable_scripts import VARIABLE_SCRIPTS
 from ytdl_sub.prebuilt_presets import PREBUILT_PRESET_NAMES
 from ytdl_sub.prebuilt_presets import PUBLISHED_PRESET_NAMES
 from ytdl_sub.script.script import Script
+from ytdl_sub.script.script import ScriptBuilder
 from ytdl_sub.script.utils.exceptions import VariableDoesNotExist
 from ytdl_sub.utils.exceptions import StringFormattingVariableNotFoundException
 from ytdl_sub.utils.exceptions import ValidationException
 from ytdl_sub.utils.logger import Logger
+from ytdl_sub.utils.scriptable import Scriptable
 from ytdl_sub.utils.yaml import dump_yaml
 from ytdl_sub.validators.strict_dict_validator import StrictDictValidator
 from ytdl_sub.validators.string_formatter_validators import DictFormatterValidator
@@ -178,22 +180,41 @@ class Preset(_PresetShell):
         return added_variables
 
     @functools.cached_property
-    def _mock_script(self) -> Script:
+    def _cached_script_builder(self) -> ScriptBuilder:
         # Set the formatter variables to be the overrides
-        variable_dict = copy.deepcopy(self.overrides.dict_with_format_strings)
+        script = ScriptBuilder(
-
+            Scriptable.add_sanitized_variables(self.overrides.dict_with_format_strings)
-        source_variables = {
+        )
-            source_var: "dummy_string"
+        script.add(
-            for source_var in self._source_variables
+            Scriptable.add_sanitized_variables(
-            + self.downloader_options.added_source_variables()
+                {source_var: "dummy_string" for source_var in self._source_variables}
-        }
+            )
-        variable_dict = dict(source_variables, **variable_dict)
+        )
-        variable_dict = dict(variable_dict, **self._added_variables)
-
-        script = Script(variable_dict)
-        script.resolve(update=True)
         return script
 
+    @property
+    def _script_builder(self) -> ScriptBuilder:
+        return copy.deepcopy(self._cached_script_builder)
+
+    @functools.cached_property
+    def _script_builder_with_added_variables(self) -> ScriptBuilder:
+        return self._script_builder.add(
+            Scriptable.add_sanitized_variables(
+                {source_var: "dummy_string" for source_var in self._added_variables}
+            )
+        )
+
+    @functools.cached_property
+    def _cached_script(self) -> Script:
+        """
+        Contains actualized script which should hold all Override variables
+        """
+        return self._script_builder_with_added_variables.partial_build(update=True)
+
+    @property
+    def _script(self) -> Script:
+        return copy.deepcopy(self._cached_script)
+
     def __validate_and_get_plugins(self) -> PresetPlugins:
         preset_plugins = PresetPlugins()
 
@@ -209,30 +230,43 @@ class Preset(_PresetShell):
         return preset_plugins
 
     def __validate_added_variables(self):
-        self.downloader_options.validate_with_variables(script=copy.deepcopy(self._mock_script))
+        script_builder = self._script_builder
+        self.downloader_options.validate_with_variables(script=copy.deepcopy(script_builder))
+        script_builder.add(
+            Scriptable.add_sanitized_variables(
+                {name: "dummy_string" for name in self.downloader_options.added_source_variables()}
+            )
+        )
 
         for _, plugin_options in sorted(
             self.plugins.zipped(), key=lambda pl: pl[0].priority.modify_entry
         ):
             # Validate current plugin using source + added plugin variables
-            plugin_options.validate_with_variables(script=copy.deepcopy(self._mock_script))
+            plugin_options.validate_with_variables(script=copy.deepcopy(script_builder))
+            script_builder.add(
+                Scriptable.add_sanitized_variables(
+                    {
+                        name: "dummy_string"
+                        for name in self.downloader_options.added_source_variables()
+                    }
+                )
+            )
 
     def __validate_override_string_formatter_validator(
         self,
         formatter_validator: Union[StringFormatterValidator, OverridesStringFormatterValidator],
-    ):
+    ) -> None:
-        script = copy.deepcopy(self._mock_script)
+        unresolvable = (
-        try:
+            set([VARIABLES.entry_metadata.variable_name] + list(self._added_variables.keys()))
-            script.add({"tmp_var": formatter_validator.format_string})
-        except VariableDoesNotExist as exc:
-            raise StringFormattingVariableNotFoundException(exc) from exc
-
-        unresolved: Optional[Set[str]] = (
-            {VARIABLES.entry_metadata.variable_name}
             if isinstance(formatter_validator, OverridesStringFormatterValidator)
             else None
         )
-        _ = script.resolve(unresolvable=unresolved)["tmp_var"]  # TODO: error if not present
+        try:
+            self._script.add({"tmp_var": formatter_validator.format_string}).resolve(
+                unresolvable=unresolvable
+            ).get("tmp_var")
+        except VariableDoesNotExist as exc:
+            raise StringFormattingVariableNotFoundException(exc) from exc
 
     def __recursive_preset_validate(
         self,

src/ytdl_sub/config/preset_options.py

@@ -6,6 +6,7 @@ from typing import TypeVar
 
 from ytdl_sub.config.defaults import DEFAULT_DOWNLOAD_ARCHIVE_NAME
 from ytdl_sub.script.script import Script
+from ytdl_sub.script.script import ScriptBuilder
 from ytdl_sub.utils.exceptions import ValidationException
 from ytdl_sub.validators.file_path_validators import OverridesStringFormatterFilePathValidator
 from ytdl_sub.validators.file_path_validators import StringFormatterFileNameValidator
@@ -53,7 +54,7 @@ class OptionsValidator(Validator, ABC):
         """
         return []
 
-    def validate_with_variables(self, script: Script) -> None:
+    def validate_with_variables(self, script: ScriptBuilder) -> None:
         """
         Optional validation after init with the session's source and override variables.
 

src/ytdl_sub/downloaders/url/validators.py

@@ -6,6 +6,7 @@ from typing import Optional
 
 from ytdl_sub.config.preset_options import OptionsValidator
 from ytdl_sub.script.script import Script
+from ytdl_sub.script.script import ScriptBuilder
 from ytdl_sub.validators.strict_dict_validator import StrictDictValidator
 from ytdl_sub.validators.string_formatter_validators import DictFormatterValidator
 from ytdl_sub.validators.string_formatter_validators import OverridesStringFormatterValidator
@@ -252,20 +253,21 @@ class MultiUrlValidator(OptionsValidator):
         """
         return list(self._urls.list[0].variables.keys)
 
-    def validate_with_variables(self, script: Script) -> None:
+    def validate_with_variables(self, script: ScriptBuilder) -> None:
         """
         Ensures new variables added are not existing variables
         """
         # Apply formatting to each new source variable, ensure it resolves
         for collection_url in self.urls.list:
             script.add(collection_url.variables.dict_with_format_strings)
-            script.resolve(update=True)
+
+        resolved_script = script.partial_build(update=True)
 
         # Ensure at least URL is non-empty
         has_non_empty_url = False
         for url_validator in self.urls.list:
-            script.add({"tmp_var_url": url_validator.url.format_string})
+            resolved_script.add({"tmp_var_url": url_validator.url.format_string})
-            has_non_empty_url |= bool(str(script.resolve().get("tmp_var_url")))
+            has_non_empty_url |= bool(str(resolved_script.resolve().get_native("tmp_var_url")))
 
         if not has_non_empty_url:
             raise self._validation_exception("Must contain at least one url that is non-empty")

src/ytdl_sub/plugins/regex.py

@@ -12,6 +12,7 @@ from ytdl_sub.entries.entry import Entry
 from ytdl_sub.entries.script.variable_scripts import VARIABLE_SCRIPTS
 from ytdl_sub.entries.variables.kwargs import YTDL_SUB_REGEX_SOURCE_VARS
 from ytdl_sub.script.script import Script
+from ytdl_sub.script.script import ScriptBuilder
 from ytdl_sub.utils.exceptions import RegexNoMatchException
 from ytdl_sub.utils.exceptions import StringFormattingVariableNotFoundException
 from ytdl_sub.utils.logger import Logger
@@ -214,7 +215,7 @@ class RegexOptions(OptionsDictValidator):
         """
         return self._skip_if_match_fails
 
-    def validate_with_variables(self, script: Script) -> None:
+    def validate_with_variables(self, script: ScriptBuilder) -> None:
         for key, regex_options in self.source_variable_capture_dict.items():
             # Ensure each variable getting captured is a source variable
             if key not in script._variables:

src/ytdl_sub/script/script.py

@@ -1,8 +1,10 @@
 import copy
+from collections import defaultdict
 from typing import Dict
 from typing import List
 from typing import Optional
 from typing import Set
+from typing import Tuple
 
 from ytdl_sub.script.functions import Functions
 from ytdl_sub.script.parser import parse
@@ -369,28 +371,45 @@ class ScriptBuilder:
         return self
 
     @property
-    def _missing_metadata(self) -> Dict[str, Set[str]]:
+    def _missing_metadata(self) -> Tuple[Dict[str, Set[str]], Dict[str, Set[str]]]:
-        missing_metadata: Dict[str, Set[str]] = {}
+        variables_missing_metadata: Dict[str, Set[str]] = defaultdict(set)
+        functions_missing_metadata: Dict[str, Set[str]] = defaultdict(set)
 
         defined_variables: Set[str] = set(self._variables.keys())
         defined_functions: Set[str] = set(self._functions.keys())
+
+        while True:
+            variables_missing_metadata_snapshot = copy.deepcopy(variables_missing_metadata)
+            functions_missing_metadata_snapshot = copy.deepcopy(functions_missing_metadata)
+
             for name, variable in self._variables.items():
-            missing_metadata[name] = {var.name for var in variable.variables}.difference(
+                if diff := {var.name for var in variable.variables}.difference(defined_variables):
-                defined_variables
+                    variables_missing_metadata[name].update(diff)
-            )
+
-            missing_metadata[name].update(
+                if diff := {fun.name for fun in variable.custom_functions}.difference(
-                {fun.name for fun in variable.custom_functions}.difference(defined_functions)
+                    defined_functions
-            )
+                ):
+                    variables_missing_metadata[name].update(diff)
 
             for name, function in self._functions.items():
-            missing_metadata[name] = {var.name for var in function.variables}.difference(
+                if diff := {var.name for var in function.variables}.difference(defined_variables):
-                defined_variables
+                    functions_missing_metadata[name].update(diff)
-            )
-            missing_metadata[name].update(
-                {fun.name for fun in function.custom_functions}.difference(defined_functions)
-            )
 
-        return missing_metadata
+                if diff := {fun.name for fun in function.custom_functions}.difference(
+                    defined_functions
+                ):
+                    functions_missing_metadata[name].update(diff)
+
+            if (
+                variables_missing_metadata == variables_missing_metadata_snapshot
+                and functions_missing_metadata == functions_missing_metadata_snapshot
+            ):
+                break
+
+            defined_variables -= set(variables_missing_metadata.keys())
+            defined_functions -= set(functions_missing_metadata.keys())
+
+        return variables_missing_metadata, functions_missing_metadata
 
     @classmethod
     def _build(cls, variables: Dict[str, SyntaxTree], functions: Dict[str, SyntaxTree]) -> Script:
@@ -400,26 +419,29 @@ class ScriptBuilder:
         script._validate()
         return script
 
-    def partial_build(self) -> Script:
+    def partial_build(self, update: bool = False) -> Script:
-        missing_metadata = self._missing_metadata
+        missing_variables, missing_functions = self._missing_metadata
         maybe_resolvable_variables: Dict[str, SyntaxTree] = {
             name: variable
             for name, variable in self._variables.items()
-            if name not in missing_metadata
+            if name not in missing_variables
         }
         maybe_resolvable_functions: Dict[str, SyntaxTree] = {
             name: function
             for name, function in self._functions.items()
-            if name not in missing_metadata
+            if name not in missing_functions
         }
 
-        return self._build(
+        script = self._build(
             variables=maybe_resolvable_variables, functions=maybe_resolvable_functions
         )
+        if update:
+            script.resolve(update=True)
+        return script
 
     def build(self) -> Script:
-        for name, missing_metadata in self._missing_metadata.items():
+        for missing_metadata in self._missing_metadata:
-            if missing_metadata:
+            for name, missing in missing_metadata.items():
                 raise ScriptBuilderMissingDefinitions(
                     f"{name} is missing the following definitions: {', '.join(missing_metadata)}"
                 )

src/ytdl_sub/utils/scriptable.py

@@ -1,4 +1,5 @@
 from abc import ABC
+from typing import Dict
 from typing import Set
 
 from ytdl_sub.entries.script.variable_scripts import VARIABLE_SCRIPTS
@@ -6,6 +7,13 @@ from ytdl_sub.script.script import Script
 
 
 class Scriptable(ABC):
+    @classmethod
+    def add_sanitized_variables(cls, variables: Dict[str, str]) -> Dict[str, str]:
+        sanitized_variables = {
+            f"{name}_sanitized": f"{{%sanitize({name})}}" for name in variables.keys()
+        }
+        return dict(variables, **sanitized_variables)
+
     def __init__(self):
         self.script = Script(VARIABLE_SCRIPTS)
         self.unresolvable: Set[str] = set()