[BACKEND] Optimize script interpreter (#1390)

Makes scripting runtime much faster.

src/ytdl_sub/config/overrides.py

@@ -158,10 +158,15 @@ class Overrides(UnstructuredDictFormatterValidator, Scriptable):
             script = entry.script
             unresolvable = entry.unresolvable
 
+        # Update the script internally so long as we are not supplying overrides
+        # that could alter the script with one-off state
+        update = function_overrides is None
+
         try:
             return script.resolve_once(
                 dict({"tmp_var": formatter.format_string}, **(function_overrides or {})),
                 unresolvable=unresolvable,
+                update=update,
             )["tmp_var"]
         except ScriptVariableNotResolved as exc:
             raise StringFormattingException(

src/ytdl_sub/script/parser.py

@@ -605,7 +605,7 @@ def parse(
         name=name,
         custom_function_names=custom_function_names,
         variable_names=variable_names,
-    ).ast
+    ).ast.maybe_resolvable_casted()
 
 
 # pylint: enable=invalid-name

src/ytdl_sub/script/script.py

@@ -10,6 +10,7 @@ from ytdl_sub.script.script_output import ScriptOutput
 from ytdl_sub.script.types.resolvable import BuiltInFunctionType
 from ytdl_sub.script.types.resolvable import Lambda
 from ytdl_sub.script.types.resolvable import Resolvable
+from ytdl_sub.script.types.syntax_tree import ResolvedSyntaxTree
 from ytdl_sub.script.types.syntax_tree import SyntaxTree
 from ytdl_sub.script.types.variable import FunctionArgument
 from ytdl_sub.script.types.variable import Variable
@@ -273,7 +274,7 @@ class Script:
 
     def _update_internally(self, resolved_variables: Dict[str, Resolvable]) -> None:
         for variable_name, resolved in resolved_variables.items():
-            self._variables[variable_name] = SyntaxTree(ast=[resolved])
+            self._variables[variable_name] = ResolvedSyntaxTree(ast=[resolved])
 
     def _recursive_get_unresolved_output_filter_variables(
         self, current_var: SyntaxTree, subset_to_resolve: Set[str], unresolvable: Set[Variable]
@@ -398,8 +399,8 @@ class Script:
 
                 # Otherwise, if it has dependencies that are all resolved, then
                 # resolve the definition
-                elif not definition.is_subset_of(
-                    variables=resolved.keys(), custom_function_definitions=self._functions
+                elif definition.is_subset_of(
+                    variables=resolved, custom_function_definitions=self._functions
                 ):
                     resolved[variable] = unresolved[variable].resolve(
                         resolved_variables=resolved,
@@ -522,6 +523,7 @@ class Script:
         variable_definitions: Dict[str, str],
         resolved: Optional[Dict[str, Resolvable]] = None,
         unresolvable: Optional[Set[str]] = None,
+        update: bool = False,
     ) -> Dict[str, Resolvable]:
         """
         Given a new set of variable definitions, resolve them using the Script, but do not
@@ -536,6 +538,8 @@ class Script:
         unresolvable
             Optional. Unresolvable variables that will be ignored in resolution, including all
             variables with a dependency to them.
+        update
+            Whether to update the script's state with resolved variables. Defaults to False.
 
         Returns
         -------
@@ -548,6 +552,7 @@ class Script:
                 pre_resolved=resolved,
                 unresolvable=unresolvable,
                 output_filter=set(list(variable_definitions.keys())),
+                update=update,
             ).output
         finally:
             for name in variable_definitions.keys():

src/ytdl_sub/script/types/function.py

@@ -1,4 +1,3 @@
-import copy
 import functools
 from abc import ABC
 from dataclasses import dataclass
@@ -58,23 +57,29 @@ class CustomFunction(Function, NamedCustomFunction):
                 # Should be validated in the Script
                 raise UNREACHABLE
 
-            resolved_variables_with_args = copy.deepcopy(resolved_variables)
+            function_args: List[FunctionArgument] = []
             for i, arg in enumerate(resolved_args):
                 function_arg = FunctionArgument.from_idx(idx=i, custom_function_name=self.name)
 
-                if function_arg in resolved_variables_with_args:
+                if function_arg in resolved_variables:
                     # function args should always be unique since they are only defined once
                     # in the custom function as %custom_function_name___idx
                     # and returned as a set from each custom function.
                     raise UNREACHABLE
 
-                resolved_variables_with_args[function_arg] = arg
+                resolved_variables[function_arg] = arg
+                function_args.append(function_arg)
 
-            return custom_functions[self.name].resolve(
-                resolved_variables=resolved_variables_with_args,
+            out = custom_functions[self.name].resolve(
+                resolved_variables=resolved_variables,
                 custom_functions=custom_functions,
             )
 
+            for function_arg in function_args:
+                del resolved_variables[function_arg]
+
+            return out
+
         # Implies the custom function does not exist. This should have
         # been checked in the parser with
         raise UNREACHABLE

src/ytdl_sub/script/types/syntax_tree.py

@@ -47,6 +47,32 @@ class SyntaxTree(VariableDependency):
         -------
         A resolvable if the AST contains a single type that is resolvable. None otherwise.
         """
-        if len(self.ast) == 1 and isinstance(self.ast[0], Resolvable):
-            return self.ast[0]
         return None
+
+    def maybe_resolvable_casted(self) -> "SyntaxTree":
+        """
+        Returns
+        -------
+        Optimized SyntaxTree if its deemed resolvable
+        """
+        if len(self.ast) == 1 and isinstance(self.ast[0], Resolvable):
+            return ResolvedSyntaxTree(self.ast)
+        return self
+
+
+@dataclass(frozen=True)
+class ResolvedSyntaxTree(SyntaxTree):
+    """
+    SyntaxTree with optimized helper functions if it's known to be resolved.
+    """
+
+    def resolve(
+        self,
+        resolved_variables: Dict[Variable, Resolvable],
+        custom_functions: Dict[str, VariableDependency],
+    ) -> Resolvable:
+        return self.ast[0]
+
+    @property
+    def maybe_resolvable(self) -> Optional[Resolvable]:
+        return self.ast[0]

src/ytdl_sub/script/types/variable_dependency.py

@@ -162,7 +162,7 @@ class VariableDependency(ABC):
     @final
     def is_subset_of(
         self,
-        variables: Iterable[Variable],
+        variables: Dict[Variable, Resolvable],
         custom_function_definitions: Dict[str, "VariableDependency"],
     ) -> bool:
         """
@@ -171,12 +171,12 @@ class VariableDependency(ABC):
         True if it contains all input variables as a dependency. False otherwise.
         """
         for custom_function in self.custom_functions:
-            if custom_function_definitions[custom_function.name].is_subset_of(
+            if not custom_function_definitions[custom_function.name].is_subset_of(
                 variables=variables, custom_function_definitions=custom_function_definitions
             ):
-                return True
+                return False
 
-        return not self.variables.issubset(variables)
+        return all(var in variables for var in self.variables)
 
     @final
     def contains(

src/ytdl_sub/validators/string_formatter_validators.py

@@ -246,6 +246,7 @@ def _validate_formatter(
                 )
             },
             unresolvable=unresolvable,
+            update=True,
         )
     except RuntimeException as exc:
         if isinstance(exc, ScriptVariableNotResolved) and is_static_formatter:

tests/unit/script/test_parser.py

@@ -16,6 +16,7 @@ from ytdl_sub.script.types.resolvable import Float
 from ytdl_sub.script.types.resolvable import Integer
 from ytdl_sub.script.types.resolvable import Lambda
 from ytdl_sub.script.types.resolvable import String
+from ytdl_sub.script.types.syntax_tree import ResolvedSyntaxTree
 from ytdl_sub.script.types.syntax_tree import SyntaxTree
 from ytdl_sub.script.types.variable import Variable
 from ytdl_sub.script.utils.exceptions import InvalidSyntaxException
@@ -24,7 +25,7 @@ from ytdl_sub.script.utils.exceptions import InvalidSyntaxException
 class TestParser:
     def test_simple(self):
         parsed = parse("hello world")
-        assert parsed == SyntaxTree([String(value="hello world")])
+        assert parsed == ResolvedSyntaxTree([String(value="hello world")])
         assert parsed.variables == set()
 
     def test_single_function_one_arg(self):