From 19f47cd9141531247a6eca567057a8ac9202c810 Mon Sep 17 00:00:00 2001 From: Jesse Bannon Date: Thu, 27 Nov 2025 18:43:27 -0800 Subject: [PATCH] [BACKEND] Optimize script interpreter (#1390) Makes scripting runtime much faster. --- src/ytdl_sub/config/overrides.py | 5 ++++ src/ytdl_sub/script/parser.py | 2 +- src/ytdl_sub/script/script.py | 11 +++++-- src/ytdl_sub/script/types/function.py | 17 +++++++---- src/ytdl_sub/script/types/syntax_tree.py | 30 +++++++++++++++++-- .../script/types/variable_dependency.py | 8 ++--- .../validators/string_formatter_validators.py | 1 + tests/unit/script/test_parser.py | 3 +- 8 files changed, 60 insertions(+), 17 deletions(-) diff --git a/src/ytdl_sub/config/overrides.py b/src/ytdl_sub/config/overrides.py index 8ed6990c..06ca8fce 100644 --- a/src/ytdl_sub/config/overrides.py +++ b/src/ytdl_sub/config/overrides.py @@ -158,10 +158,15 @@ class Overrides(UnstructuredDictFormatterValidator, Scriptable): script = entry.script unresolvable = entry.unresolvable + # Update the script internally so long as we are not supplying overrides + # that could alter the script with one-off state + update = function_overrides is None + try: return script.resolve_once( dict({"tmp_var": formatter.format_string}, **(function_overrides or {})), unresolvable=unresolvable, + update=update, )["tmp_var"] except ScriptVariableNotResolved as exc: raise StringFormattingException( diff --git a/src/ytdl_sub/script/parser.py b/src/ytdl_sub/script/parser.py index 737d6ffa..9dd3f186 100644 --- a/src/ytdl_sub/script/parser.py +++ b/src/ytdl_sub/script/parser.py @@ -605,7 +605,7 @@ def parse( name=name, custom_function_names=custom_function_names, variable_names=variable_names, - ).ast + ).ast.maybe_resolvable_casted() # pylint: enable=invalid-name diff --git a/src/ytdl_sub/script/script.py b/src/ytdl_sub/script/script.py index ab506318..e92e7fdd 100644 --- a/src/ytdl_sub/script/script.py +++ b/src/ytdl_sub/script/script.py @@ -10,6 +10,7 @@ from ytdl_sub.script.script_output import ScriptOutput from ytdl_sub.script.types.resolvable import BuiltInFunctionType from ytdl_sub.script.types.resolvable import Lambda from ytdl_sub.script.types.resolvable import Resolvable +from ytdl_sub.script.types.syntax_tree import ResolvedSyntaxTree from ytdl_sub.script.types.syntax_tree import SyntaxTree from ytdl_sub.script.types.variable import FunctionArgument from ytdl_sub.script.types.variable import Variable @@ -273,7 +274,7 @@ class Script: def _update_internally(self, resolved_variables: Dict[str, Resolvable]) -> None: for variable_name, resolved in resolved_variables.items(): - self._variables[variable_name] = SyntaxTree(ast=[resolved]) + self._variables[variable_name] = ResolvedSyntaxTree(ast=[resolved]) def _recursive_get_unresolved_output_filter_variables( self, current_var: SyntaxTree, subset_to_resolve: Set[str], unresolvable: Set[Variable] @@ -398,8 +399,8 @@ class Script: # Otherwise, if it has dependencies that are all resolved, then # resolve the definition - elif not definition.is_subset_of( - variables=resolved.keys(), custom_function_definitions=self._functions + elif definition.is_subset_of( + variables=resolved, custom_function_definitions=self._functions ): resolved[variable] = unresolved[variable].resolve( resolved_variables=resolved, @@ -522,6 +523,7 @@ class Script: variable_definitions: Dict[str, str], resolved: Optional[Dict[str, Resolvable]] = None, unresolvable: Optional[Set[str]] = None, + update: bool = False, ) -> Dict[str, Resolvable]: """ Given a new set of variable definitions, resolve them using the Script, but do not @@ -536,6 +538,8 @@ class Script: unresolvable Optional. Unresolvable variables that will be ignored in resolution, including all variables with a dependency to them. + update + Whether to update the script's state with resolved variables. Defaults to False. Returns ------- @@ -548,6 +552,7 @@ class Script: pre_resolved=resolved, unresolvable=unresolvable, output_filter=set(list(variable_definitions.keys())), + update=update, ).output finally: for name in variable_definitions.keys(): diff --git a/src/ytdl_sub/script/types/function.py b/src/ytdl_sub/script/types/function.py index 7ecd1999..5f33006e 100644 --- a/src/ytdl_sub/script/types/function.py +++ b/src/ytdl_sub/script/types/function.py @@ -1,4 +1,3 @@ -import copy import functools from abc import ABC from dataclasses import dataclass @@ -58,23 +57,29 @@ class CustomFunction(Function, NamedCustomFunction): # Should be validated in the Script raise UNREACHABLE - resolved_variables_with_args = copy.deepcopy(resolved_variables) + function_args: List[FunctionArgument] = [] for i, arg in enumerate(resolved_args): function_arg = FunctionArgument.from_idx(idx=i, custom_function_name=self.name) - if function_arg in resolved_variables_with_args: + if function_arg in resolved_variables: # function args should always be unique since they are only defined once # in the custom function as %custom_function_name___idx # and returned as a set from each custom function. raise UNREACHABLE - resolved_variables_with_args[function_arg] = arg + resolved_variables[function_arg] = arg + function_args.append(function_arg) - return custom_functions[self.name].resolve( - resolved_variables=resolved_variables_with_args, + out = custom_functions[self.name].resolve( + resolved_variables=resolved_variables, custom_functions=custom_functions, ) + for function_arg in function_args: + del resolved_variables[function_arg] + + return out + # Implies the custom function does not exist. This should have # been checked in the parser with raise UNREACHABLE diff --git a/src/ytdl_sub/script/types/syntax_tree.py b/src/ytdl_sub/script/types/syntax_tree.py index d058852c..eff39990 100644 --- a/src/ytdl_sub/script/types/syntax_tree.py +++ b/src/ytdl_sub/script/types/syntax_tree.py @@ -47,6 +47,32 @@ class SyntaxTree(VariableDependency): ------- A resolvable if the AST contains a single type that is resolvable. None otherwise. """ - if len(self.ast) == 1 and isinstance(self.ast[0], Resolvable): - return self.ast[0] return None + + def maybe_resolvable_casted(self) -> "SyntaxTree": + """ + Returns + ------- + Optimized SyntaxTree if its deemed resolvable + """ + if len(self.ast) == 1 and isinstance(self.ast[0], Resolvable): + return ResolvedSyntaxTree(self.ast) + return self + + +@dataclass(frozen=True) +class ResolvedSyntaxTree(SyntaxTree): + """ + SyntaxTree with optimized helper functions if it's known to be resolved. + """ + + def resolve( + self, + resolved_variables: Dict[Variable, Resolvable], + custom_functions: Dict[str, VariableDependency], + ) -> Resolvable: + return self.ast[0] + + @property + def maybe_resolvable(self) -> Optional[Resolvable]: + return self.ast[0] diff --git a/src/ytdl_sub/script/types/variable_dependency.py b/src/ytdl_sub/script/types/variable_dependency.py index eb5f646e..21716124 100644 --- a/src/ytdl_sub/script/types/variable_dependency.py +++ b/src/ytdl_sub/script/types/variable_dependency.py @@ -162,7 +162,7 @@ class VariableDependency(ABC): @final def is_subset_of( self, - variables: Iterable[Variable], + variables: Dict[Variable, Resolvable], custom_function_definitions: Dict[str, "VariableDependency"], ) -> bool: """ @@ -171,12 +171,12 @@ class VariableDependency(ABC): True if it contains all input variables as a dependency. False otherwise. """ for custom_function in self.custom_functions: - if custom_function_definitions[custom_function.name].is_subset_of( + if not custom_function_definitions[custom_function.name].is_subset_of( variables=variables, custom_function_definitions=custom_function_definitions ): - return True + return False - return not self.variables.issubset(variables) + return all(var in variables for var in self.variables) @final def contains( diff --git a/src/ytdl_sub/validators/string_formatter_validators.py b/src/ytdl_sub/validators/string_formatter_validators.py index 17436700..567f0f8d 100644 --- a/src/ytdl_sub/validators/string_formatter_validators.py +++ b/src/ytdl_sub/validators/string_formatter_validators.py @@ -246,6 +246,7 @@ def _validate_formatter( ) }, unresolvable=unresolvable, + update=True, ) except RuntimeException as exc: if isinstance(exc, ScriptVariableNotResolved) and is_static_formatter: diff --git a/tests/unit/script/test_parser.py b/tests/unit/script/test_parser.py index b40e13ca..c6694562 100644 --- a/tests/unit/script/test_parser.py +++ b/tests/unit/script/test_parser.py @@ -16,6 +16,7 @@ from ytdl_sub.script.types.resolvable import Float from ytdl_sub.script.types.resolvable import Integer from ytdl_sub.script.types.resolvable import Lambda from ytdl_sub.script.types.resolvable import String +from ytdl_sub.script.types.syntax_tree import ResolvedSyntaxTree from ytdl_sub.script.types.syntax_tree import SyntaxTree from ytdl_sub.script.types.variable import Variable from ytdl_sub.script.utils.exceptions import InvalidSyntaxException @@ -24,7 +25,7 @@ from ytdl_sub.script.utils.exceptions import InvalidSyntaxException class TestParser: def test_simple(self): parsed = parse("hello world") - assert parsed == SyntaxTree([String(value="hello world")]) + assert parsed == ResolvedSyntaxTree([String(value="hello world")]) assert parsed.variables == set() def test_single_function_one_arg(self):