parsed once optimized

src/ytdl_sub/config/overrides.py

@@ -4,15 +4,16 @@ from typing import Iterable
 from typing import Optional
 from typing import Set
 
-import mergedeep
-
 from ytdl_sub.entries.entry import Entry
 from ytdl_sub.entries.script.variable_definitions import VARIABLES
 from ytdl_sub.entries.variables.override_variables import REQUIRED_OVERRIDE_VARIABLE_NAMES
 from ytdl_sub.entries.variables.override_variables import OverrideHelpers
 from ytdl_sub.script.parser import parse
 from ytdl_sub.script.script import Script
+from ytdl_sub.script.types.function import BuiltInFunction
 from ytdl_sub.script.types.resolvable import Resolvable
+from ytdl_sub.script.types.resolvable import String
+from ytdl_sub.script.types.syntax_tree import SyntaxTree
 from ytdl_sub.script.utils.exceptions import ScriptVariableNotResolved
 from ytdl_sub.utils.exceptions import InvalidVariableNameException
 from ytdl_sub.utils.exceptions import StringFormattingException
@@ -134,29 +135,35 @@ class Overrides(UnstructuredDictFormatterValidator, Scriptable):
             )
 
     def initial_variables(
-        self, unresolved_variables: Optional[Dict[str, str]] = None
+        self, unresolved_variables: Optional[Dict[str, SyntaxTree]] = None
-    ) -> Dict[str, str]:
+    ) -> Dict[str, SyntaxTree]:
         """
         Returns
         -------
         Variables and format strings for all Override variables + additional variables (Optional)
         """
-        initial_variables: Dict[str, str] = {}
+        initial_variables: Dict[str, SyntaxTree] = self.dict_with_parsed_format_strings
-        mergedeep.merge(
+        if unresolved_variables:
-            initial_variables,
+            initial_variables |= unresolved_variables
-            self.dict_with_format_strings,
+        return ScriptUtils.add_sanitized_parsed_variables(initial_variables)
-            unresolved_variables if unresolved_variables else {},
-        )
-        return ScriptUtils.add_sanitized_variables(initial_variables)
 
     def initialize_script(self, unresolved_variables: Set[str]) -> "Overrides":
         """
         Initialize the override script with any unresolved variables
         """
-        self.script.add(
+        self.script.add_parsed(
             self.initial_variables(
                 unresolved_variables={
-                    var_name: f"{{%throw('Plugin variable {var_name} has not been created yet')}}"
+                    var_name: SyntaxTree(
+                        ast=[
+                            BuiltInFunction(
+                                name="throw",
+                                args=[
+                                    String(f"Plugin variable {var_name} has not been created yet")
+                                ],
+                            )
+                        ]
+                    )
                     for var_name in unresolved_variables
                 }
             )

src/ytdl_sub/script/parser.py

@@ -31,6 +31,8 @@ from ytdl_sub.script.utils.exceptions import InvalidSyntaxException
 from ytdl_sub.script.utils.exceptions import InvalidVariableName
 from ytdl_sub.script.utils.exceptions import UserException
 from ytdl_sub.script.utils.exceptions import VariableDoesNotExist
+from ytdl_sub.script.utils.name_validation import function_name
+from ytdl_sub.script.utils.name_validation import is_function
 from ytdl_sub.script.utils.name_validation import validate_variable_name
 
 # pylint: disable=invalid-name
@@ -144,6 +146,9 @@ class _Parser:
     ):
         self._text = text
         self._name = name
+        if name and is_function(name):
+            self._name = function_name(name)
+
         self._custom_function_names = custom_function_names
         self._variable_names = variable_names
         self._pos = 0

src/ytdl_sub/script/script.py

@@ -20,28 +20,13 @@ from ytdl_sub.script.utils.exceptions import IncompatibleFunctionArguments
 from ytdl_sub.script.utils.exceptions import InvalidCustomFunctionArguments
 from ytdl_sub.script.utils.exceptions import RuntimeException
 from ytdl_sub.script.utils.exceptions import ScriptVariableNotResolved
+from ytdl_sub.script.utils.name_validation import function_name
+from ytdl_sub.script.utils.name_validation import is_function
+from ytdl_sub.script.utils.name_validation import to_function_definition_name
 from ytdl_sub.script.utils.name_validation import validate_variable_name
 from ytdl_sub.script.utils.type_checking import FunctionSpec
 
 
-def _is_function(override_name: str):
-    return override_name.startswith("%")
-
-
-def _function_name(function_key: str) -> str:
-    """
-    Drop the % in %custom_function
-    """
-    return function_key[1:]
-
-
-def _to_function_definition_name(function_key: str) -> str:
-    """
-    Add % in %custom_function
-    """
-    return f"%{function_key}"
-
-
 class Script:
     """
     Takes a dictionary of both
@@ -241,23 +226,23 @@ class Script:
 
     def __init__(self, script: Dict[str, str]):
         function_names: Set[str] = {
-            _function_name(name) for name in script.keys() if _is_function(name)
+            function_name(name) for name in script.keys() if is_function(name)
         }
         variable_names: Set[str] = {
-            validate_variable_name(name) for name in script.keys() if not _is_function(name)
+            validate_variable_name(name) for name in script.keys() if not is_function(name)
         }
 
         self._functions: Dict[str, SyntaxTree] = {
             # custom_function_name must be passed to properly type custom function
             # arguments uniquely if they're nested (i.e. $0 to $custom_func___0)
-            _function_name(function_key): parse(
+            function_name(function_key): parse(
                 text=function_value,
-                name=_function_name(function_key),
+                name=function_name(function_key),
                 custom_function_names=function_names,
                 variable_names=variable_names,
             )
             for function_key, function_value in script.items()
-            if _is_function(function_key)
+            if is_function(function_key)
         }
 
         self._variables: Dict[str, SyntaxTree] = {
@@ -268,7 +253,7 @@ class Script:
                 variable_names=variable_names,
             )
             for variable_key, variable_value in script.items()
-            if not _is_function(variable_key)
+            if not is_function(variable_key)
         }
         self._validate()
 
@@ -485,12 +470,12 @@ class Script:
         added_variables_to_validate: Set[str] = set()
 
         functions_to_add = {
-            _function_name(name): definition
+            function_name(name): definition
             for name, definition in variables.items()
-            if _is_function(name)
+            if is_function(name)
         }
         variables_to_add = {
-            name: definition for name, definition in variables.items() if not _is_function(name)
+            name: definition for name, definition in variables.items() if not is_function(name)
         }
 
         custom_function_names = set(self._functions.keys()) | functions_to_add.keys()
@@ -520,6 +505,46 @@ class Script:
 
         return self
 
+    def add_parsed(self, variables: Dict[str, SyntaxTree]) -> "Script":
+        """
+        Adds already parsed, new variables to the script.
+
+        Parameters
+        ----------
+        variables
+            Mapping containing variable name to definition.
+
+        Returns
+        -------
+        Script
+            self
+        """
+        added_variables_to_validate: Set[str] = set()
+
+        functions_to_add = {
+            function_name(name): definition
+            for name, definition in variables.items()
+            if is_function(name)
+        }
+        variables_to_add = {
+            name: definition for name, definition in variables.items() if not is_function(name)
+        }
+
+        for definitions in [functions_to_add, variables_to_add]:
+            for name, parsed in definitions.items():
+                if parsed.maybe_resolvable is None:
+                    added_variables_to_validate.add(name)
+
+                if name in functions_to_add:
+                    self._functions[name] = parsed
+                else:
+                    self._variables[name] = parsed
+
+        if added_variables_to_validate:
+            self._validate(added_variables=added_variables_to_validate)
+
+        return self
+
     def resolve_once(
         self,
         variable_definitions: Dict[str, str],
@@ -560,6 +585,46 @@ class Script:
             for name in variable_definitions.keys():
                 self._variables.pop(name, None)
 
+    def resolve_once_parsed(
+        self,
+        variable_definitions: Dict[str, SyntaxTree],
+        resolved: Optional[Dict[str, Resolvable]] = None,
+        unresolvable: Optional[Set[str]] = None,
+        update: bool = False,
+    ) -> Dict[str, Resolvable]:
+        """
+        Given a new set of variable definitions, resolve them using the Script, but do not
+        add them to the Script itself.
+
+        Parameters
+        ----------
+        variable_definitions
+            Variables to resolve, but not store in the Script
+        resolved
+            Optional. Pre-resolved variables that should be used instead of what is in the script.
+        unresolvable
+            Optional. Unresolvable variables that will be ignored in resolution, including all
+            variables with a dependency to them.
+        update
+            Whether to update the script's state with resolved variables. Defaults to False.
+
+        Returns
+        -------
+        Dict[str, Resolvable]
+            Dict containing the variable names to their resolved values.
+        """
+        try:
+            self.add_parsed(variable_definitions)
+            return self._resolve(
+                pre_resolved=resolved,
+                unresolvable=unresolvable,
+                output_filter=set(list(variable_definitions.keys())),
+                update=update,
+            ).output
+        finally:
+            for name in variable_definitions.keys():
+                self._variables.pop(name, None)
+
     def get(self, variable_name: str) -> Resolvable:
         """
         Parameters
@@ -605,4 +670,4 @@ class Script:
         Set[str]
             Names of all functions within the Script.
         """
-        return set(_to_function_definition_name(name) for name in self._functions.keys())
+        return set(to_function_definition_name(name) for name in self._functions.keys())

src/ytdl_sub/script/utils/name_validation.py

@@ -58,3 +58,24 @@ def validate_custom_function_name(custom_function_name: str) -> None:
             f"Custom function name '%{custom_function_name}' is invalid:"
             " The name is used by a built-in function and cannot be overwritten."
         )
+
+
+def is_function(override_name: str):
+    """
+    Whether the definition is a function or not.
+    """
+    return override_name.startswith("%")
+
+
+def function_name(function_key: str) -> str:
+    """
+    Drop the % in %custom_function
+    """
+    return function_key[1:]
+
+
+def to_function_definition_name(function_key: str) -> str:
+    """
+    Add % in %custom_function
+    """
+    return f"%{function_key}"

src/ytdl_sub/utils/script.py

@@ -4,7 +4,6 @@ from typing import Any
 from typing import Dict
 
 from ytdl_sub.script.parser import parse
-from ytdl_sub.script.script import _is_function
 from ytdl_sub.script.types.array import UnresolvedArray
 from ytdl_sub.script.types.function import BuiltInFunction
 from ytdl_sub.script.types.function import Function
@@ -15,8 +14,10 @@ from ytdl_sub.script.types.resolvable import Float
 from ytdl_sub.script.types.resolvable import Integer
 from ytdl_sub.script.types.resolvable import Lambda
 from ytdl_sub.script.types.resolvable import String
+from ytdl_sub.script.types.syntax_tree import SyntaxTree
 from ytdl_sub.script.types.variable import Variable
 from ytdl_sub.script.utils.exceptions import UNREACHABLE
+from ytdl_sub.script.utils.name_validation import is_function
 
 # pylint: disable=too-many-return-statements
 
@@ -30,10 +31,26 @@ class ScriptUtils:
         sanitized_variables = {
             f"{name}_sanitized": f"{{%sanitize({name})}}"
             for name in variables.keys()
-            if not _is_function(name)
+            if not is_function(name)
         }
         return dict(variables, **sanitized_variables)
 
+    @classmethod
+    def add_sanitized_parsed_variables(
+        cls, variables: Dict[str, SyntaxTree]
+    ) -> Dict[str, SyntaxTree]:
+        """
+        Helper to add sanitized variables to a Script
+        """
+        sanitized_variables = {
+            f"{name}_sanitized": SyntaxTree(
+                ast=[BuiltInFunction(name="sanitize", args=[Variable(name)])]
+            )
+            for name in variables.keys()
+            if not is_function(name)
+        }
+        return variables | sanitized_variables
+
     @classmethod
     def to_script(cls, value: Any, sort_keys: bool = True) -> str:
         """

src/ytdl_sub/validators/string_formatter_validators.py

@@ -181,9 +181,14 @@ class DictFormatterValidator(LiteralDictValidator):
 
     @property
     def dict_with_format_strings(self) -> Dict[str, str]:
-        """Returns dict with the format strings themselves"""
+        """Returns dict with the format strings themselves."""
         return {key: string_formatter.format_string for key, string_formatter in self.dict.items()}
 
+    @property
+    def dict_with_parsed_format_strings(self) -> Dict[str, SyntaxTree]:
+        """Returns dict with the parsed format strings."""
+        return {key: string_formatter.parsed for key, string_formatter in self.dict.items()}
+
 
 class OverridesDictFormatterValidator(DictFormatterValidator):
     """
@@ -253,8 +258,8 @@ def _validate_formatter(
         )
     try:
         if is_static_formatter:
-            return mock_script.resolve_once(
+            return mock_script.resolve_once_parsed(
-                {"tmp_var": formatter_validator.format_string},
+                {"tmp_var": formatter_validator.parsed},
                 unresolvable=unresolved_variables,
                 update=True,
             )["tmp_var"].native

tools/docgen/plugins.py

@@ -37,6 +37,7 @@ def should_filter_property(property_name: str) -> bool:
         "dict",
         "keys",
         "dict_with_format_strings",
+        "dict_with_parsed_format_strings",
         "subscription_name",
         "list",
         "script",