danvics/ytdl-sub
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

[BUGFIX] Fix recursion limit when applying large reduce functions (#851)

Browse source 
Python would think a recursive error occurred, but in reality, the stack got too large due to poor optimization when executing array reduce functions. Thanks Melissa from Discord for the bug report!
This commit is contained in:
Jesse Bannon 2023-12-20 23:46:17 -08:00 committed by GitHub
parent b8fb119f21
commit 873ecc0147
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
3 changed files with 62 additions and 16 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

27
src/ytdl_sub/script/types/function.py
View file

@ -232,20 +232,25 @@ class BuiltInFunction(Function, BuiltInFunctionType):
if len(lambda_array.value) == 1:
return lambda_array.value[0]
reduced = self._instantiate_lambda(
lambda_function_name=lambda_function_name,
args=[lambda_array.value[0], lambda_array.value[1]],
)
for idx in range(2, len(lambda_array.value)):
reduced = self._instantiate_lambda(
lambda_function_name=lambda_function_name, args=[reduced, lambda_array.value[idx]]
)
return self._resolve_argument_type(
arg=reduced,
reduced: Resolvable = self._resolve_argument_type(
arg=self._instantiate_lambda(
lambda_function_name=lambda_function_name,
args=[lambda_array.value[0], lambda_array.value[1]],
),
resolved_variables=resolved_variables,
custom_functions=custom_functions,
)
for idx in range(2, len(lambda_array.value)):
reduced = self._resolve_argument_type(
arg=self._instantiate_lambda(
lambda_function_name=lambda_function_name,
args=[reduced, lambda_array.value[idx]],
),
resolved_variables=resolved_variables,
custom_functions=custom_functions,
)
return reduced
def resolve(
self,

12
src/ytdl_sub/utils/scriptable.py
View file

@ -19,12 +19,14 @@ class Scriptable(ABC):
Shared class between Entry and Overrides to manage their underlying Script.
"""
def __init__(self):
self.script = Script(
ScriptUtils.add_sanitized_variables(
dict(copy.deepcopy(VARIABLE_SCRIPTS), **copy.deepcopy(CUSTOM_FUNCTION_SCRIPTS))
)
_BASE_SCRIPT: Script = Script(
ScriptUtils.add_sanitized_variables(
dict(copy.deepcopy(VARIABLE_SCRIPTS), **copy.deepcopy(CUSTOM_FUNCTION_SCRIPTS))
)
)
def __init__(self):
self.script = copy.deepcopy(Scriptable._BASE_SCRIPT)
self.unresolvable: Set[str] = copy.deepcopy(UNRESOLVED_VARIABLES)
def update_script(self) -> None:

39
tests/unit/script/functions/test_array_functions.py
View file

@ -58,6 +58,45 @@ class TestArrayFunctions:
output = single_variable_output("{%array_reduce([1, 2, 3, 4], %add)}")
assert output == 10
def test_array_reduce_complex(self):
output = (
Script(
{
"%custom_get": """{
%if(
%bool(siblings_array),
%array_apply_fixed(
siblings_array,
%string($0),
%map_get
)
[]
)
}""",
"siblings_array": """{
[
{'upload_date': '20200101'},
{'upload_date': '19940101'}
]
}""",
"upload_date": "20230101",
"output": """{
%array_reduce(
%if_passthrough(
%custom_get('upload_date'),
[ upload_date ]
),
%max
)
}""",
}
)
.resolve(update=True)
.get("output")
.native
)
assert output == "20200101"
def test_array_enumerate(self):
output = (
Script(