reduce function, needs more validation tests
This commit is contained in:
parent
f0420acef1
commit
c4362afda5
7 changed files with 101 additions and 16 deletions
|
|
@ -6,6 +6,7 @@ from ytdl_sub.script.types.resolvable import AnyArgument
|
||||||
from ytdl_sub.script.types.resolvable import Boolean
|
from ytdl_sub.script.types.resolvable import Boolean
|
||||||
from ytdl_sub.script.types.resolvable import Integer
|
from ytdl_sub.script.types.resolvable import Integer
|
||||||
from ytdl_sub.script.types.resolvable import Lambda
|
from ytdl_sub.script.types.resolvable import Lambda
|
||||||
|
from ytdl_sub.script.types.resolvable import LambdaReduce
|
||||||
from ytdl_sub.script.types.resolvable import LambdaTwo
|
from ytdl_sub.script.types.resolvable import LambdaTwo
|
||||||
from ytdl_sub.script.types.resolvable import Resolvable
|
from ytdl_sub.script.types.resolvable import Resolvable
|
||||||
from ytdl_sub.script.utils.exceptions import UNREACHABLE
|
from ytdl_sub.script.utils.exceptions import UNREACHABLE
|
||||||
|
|
@ -101,4 +102,10 @@ class ArrayFunctions:
|
||||||
"""
|
"""
|
||||||
return Array([Array([Integer(idx), val]) for idx, val in enumerate(array.value)])
|
return Array([Array([Integer(idx), val]) for idx, val in enumerate(array.value)])
|
||||||
|
|
||||||
# pylint: enable=unused-argument
|
@staticmethod
|
||||||
|
def array_reduce(array: Array, lambda_reduce_function: LambdaReduce) -> AnyArgument:
|
||||||
|
"""
|
||||||
|
Apply a reduce function on pairs of elements in the Array, until one element remains.
|
||||||
|
Executes using the left-most and reduces in the right direction.
|
||||||
|
"""
|
||||||
|
return array
|
||||||
|
|
|
||||||
|
|
@ -149,7 +149,7 @@ class Script:
|
||||||
for name, definition in definitions.items():
|
for name, definition in definitions.items():
|
||||||
for function in definition.built_in_functions:
|
for function in definition.built_in_functions:
|
||||||
spec = FunctionSpec.from_callable(Functions.get(function.name))
|
spec = FunctionSpec.from_callable(Functions.get(function.name))
|
||||||
if lambda_type := spec.is_lambda_function:
|
if lambda_type := spec.is_lambda_like:
|
||||||
|
|
||||||
lambda_function_names = set(
|
lambda_function_names = set(
|
||||||
[
|
[
|
||||||
|
|
|
||||||
|
|
@ -107,6 +107,14 @@ class BuiltInFunction(Function, BuiltInFunctionType):
|
||||||
return arg.output_type()
|
return arg.output_type()
|
||||||
return type(arg)
|
return type(arg)
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def _instantiate_lambda(cls, lambda_function_name: str, args: List[Argument]) -> Function:
|
||||||
|
return (
|
||||||
|
BuiltInFunction(name=lambda_function_name, args=args)
|
||||||
|
if Functions.is_built_in(lambda_function_name)
|
||||||
|
else CustomFunction(name=lambda_function_name, args=args)
|
||||||
|
)
|
||||||
|
|
||||||
def _output_type(self, union_args: List[Type[Argument]]) -> Type[Resolvable]:
|
def _output_type(self, union_args: List[Type[Argument]]) -> Type[Resolvable]:
|
||||||
union_types_list = set()
|
union_types_list = set()
|
||||||
for union_type in union_args:
|
for union_type in union_args:
|
||||||
|
|
@ -160,9 +168,9 @@ class BuiltInFunction(Function, BuiltInFunctionType):
|
||||||
return self._resolve_argument_type(
|
return self._resolve_argument_type(
|
||||||
arg=UnresolvedArray(
|
arg=UnresolvedArray(
|
||||||
[
|
[
|
||||||
BuiltInFunction(name=lambda_function_name, args=lambda_arg.value)
|
self._instantiate_lambda(
|
||||||
if Functions.is_built_in(lambda_function_name)
|
lambda_function_name=lambda_function_name, args=lambda_arg.value
|
||||||
else CustomFunction(name=lambda_function_name, args=lambda_arg.value)
|
)
|
||||||
for lambda_arg in lambda_args.value
|
for lambda_arg in lambda_args.value
|
||||||
]
|
]
|
||||||
),
|
),
|
||||||
|
|
@ -170,6 +178,50 @@ class BuiltInFunction(Function, BuiltInFunctionType):
|
||||||
custom_functions=custom_functions,
|
custom_functions=custom_functions,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
def _resolve_lambda_reduce_function(
|
||||||
|
self,
|
||||||
|
resolved_arguments: List[Resolvable | Lambda],
|
||||||
|
resolved_variables: Dict[Variable, Resolvable],
|
||||||
|
custom_functions: Dict[str, "VariableDependency"],
|
||||||
|
) -> Resolvable:
|
||||||
|
"""
|
||||||
|
Resolve the lambda reduce function by
|
||||||
|
1. Preemptively create the 'reduce-like' call-stack as unresolvable
|
||||||
|
2. Resolve it like any other syntax
|
||||||
|
"""
|
||||||
|
function_input_lambda_args = [arg for arg in resolved_arguments if isinstance(arg, Lambda)]
|
||||||
|
if not self.function_spec.is_lambda_reduce_function or len(function_input_lambda_args) != 1:
|
||||||
|
raise UNREACHABLE
|
||||||
|
|
||||||
|
lambda_function_name = function_input_lambda_args[0].value
|
||||||
|
|
||||||
|
try:
|
||||||
|
lambda_array = self.callable(*resolved_arguments)
|
||||||
|
except Exception as exc:
|
||||||
|
raise FunctionRuntimeException(
|
||||||
|
f"Runtime error occurred when executing the function %{self.name}: {str(exc)}"
|
||||||
|
) from exc
|
||||||
|
|
||||||
|
assert isinstance(lambda_array, Array)
|
||||||
|
|
||||||
|
if len(lambda_array.value) == 1:
|
||||||
|
return lambda_array.value[0]
|
||||||
|
|
||||||
|
reduced = self._instantiate_lambda(
|
||||||
|
lambda_function_name=lambda_function_name,
|
||||||
|
args=[lambda_array.value[0], lambda_array.value[1]],
|
||||||
|
)
|
||||||
|
for idx in range(2, len(lambda_array.value)):
|
||||||
|
reduced = self._instantiate_lambda(
|
||||||
|
lambda_function_name=lambda_function_name, args=[reduced, lambda_array.value[idx]]
|
||||||
|
)
|
||||||
|
|
||||||
|
return self._resolve_argument_type(
|
||||||
|
arg=reduced,
|
||||||
|
resolved_variables=resolved_variables,
|
||||||
|
custom_functions=custom_functions,
|
||||||
|
)
|
||||||
|
|
||||||
def resolve(
|
def resolve(
|
||||||
self,
|
self,
|
||||||
resolved_variables: Dict[Variable, Resolvable],
|
resolved_variables: Dict[Variable, Resolvable],
|
||||||
|
|
@ -193,6 +245,14 @@ class BuiltInFunction(Function, BuiltInFunctionType):
|
||||||
custom_functions=custom_functions,
|
custom_functions=custom_functions,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
# If a lambda is in a function's arg, resolve it differently
|
||||||
|
if self.function_spec.is_lambda_reduce_function:
|
||||||
|
return self._resolve_lambda_reduce_function(
|
||||||
|
resolved_arguments=resolved_arguments,
|
||||||
|
resolved_variables=resolved_variables,
|
||||||
|
custom_functions=custom_functions,
|
||||||
|
)
|
||||||
|
|
||||||
try:
|
try:
|
||||||
return self.callable(*resolved_arguments)
|
return self.callable(*resolved_arguments)
|
||||||
except (UserThrownRuntimeError, RuntimeException):
|
except (UserThrownRuntimeError, RuntimeException):
|
||||||
|
|
|
||||||
|
|
@ -194,3 +194,8 @@ class LambdaThree(Lambda):
|
||||||
@classmethod
|
@classmethod
|
||||||
def num_input_args(cls) -> int:
|
def num_input_args(cls) -> int:
|
||||||
return 3
|
return 3
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class LambdaReduce(LambdaTwo):
|
||||||
|
pass
|
||||||
|
|
|
||||||
|
|
@ -14,6 +14,7 @@ from ytdl_sub.script.types.resolvable import BuiltInFunctionType
|
||||||
from ytdl_sub.script.types.resolvable import FunctionType
|
from ytdl_sub.script.types.resolvable import FunctionType
|
||||||
from ytdl_sub.script.types.resolvable import FutureResolvable
|
from ytdl_sub.script.types.resolvable import FutureResolvable
|
||||||
from ytdl_sub.script.types.resolvable import Lambda
|
from ytdl_sub.script.types.resolvable import Lambda
|
||||||
|
from ytdl_sub.script.types.resolvable import LambdaReduce
|
||||||
from ytdl_sub.script.types.resolvable import LambdaThree
|
from ytdl_sub.script.types.resolvable import LambdaThree
|
||||||
from ytdl_sub.script.types.resolvable import LambdaTwo
|
from ytdl_sub.script.types.resolvable import LambdaTwo
|
||||||
from ytdl_sub.script.types.resolvable import NamedType
|
from ytdl_sub.script.types.resolvable import NamedType
|
||||||
|
|
@ -163,7 +164,11 @@ class FunctionSpec:
|
||||||
return 0 # varargs can take any number
|
return 0 # varargs can take any number
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def is_lambda_function(self) -> Optional[Type[TLambda]]:
|
def is_lambda_reduce_function(self) -> Optional[Type[LambdaReduce]]:
|
||||||
|
return LambdaReduce if LambdaReduce in (self.args or []) else None
|
||||||
|
|
||||||
|
@property
|
||||||
|
def is_lambda_function(self) -> Optional[Type[Lambda | LambdaTwo | LambdaThree]]:
|
||||||
if LambdaThree in (self.args or []):
|
if LambdaThree in (self.args or []):
|
||||||
return LambdaThree
|
return LambdaThree
|
||||||
if LambdaTwo in (self.args or []):
|
if LambdaTwo in (self.args or []):
|
||||||
|
|
@ -172,6 +177,14 @@ class FunctionSpec:
|
||||||
return Lambda
|
return Lambda
|
||||||
return None
|
return None
|
||||||
|
|
||||||
|
@property
|
||||||
|
def is_lambda_like(self) -> Optional[Type[TLambda]]:
|
||||||
|
if l_type := self.is_lambda_reduce_function:
|
||||||
|
return l_type
|
||||||
|
if l_type := self.is_lambda_function:
|
||||||
|
return l_type
|
||||||
|
return None
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def from_callable(cls, callable_ref: Callable[..., Resolvable]) -> "FunctionSpec":
|
def from_callable(cls, callable_ref: Callable[..., Resolvable]) -> "FunctionSpec":
|
||||||
arg_spec: FullArgSpec = inspect.getfullargspec(callable_ref)
|
arg_spec: FullArgSpec = inspect.getfullargspec(callable_ref)
|
||||||
|
|
|
||||||
|
|
@ -12,16 +12,12 @@ from resources import copy_file_fixture
|
||||||
from ytdl_sub.config.config_file import ConfigFile
|
from ytdl_sub.config.config_file import ConfigFile
|
||||||
from ytdl_sub.downloaders.url.downloader import MultiUrlDownloader
|
from ytdl_sub.downloaders.url.downloader import MultiUrlDownloader
|
||||||
from ytdl_sub.downloaders.ytdlp import YTDLP
|
from ytdl_sub.downloaders.ytdlp import YTDLP
|
||||||
|
from ytdl_sub.entries.script.variable_definitions import VARIABLES as v
|
||||||
from ytdl_sub.entries.variables.kwargs import DESCRIPTION
|
from ytdl_sub.entries.variables.kwargs import DESCRIPTION
|
||||||
from ytdl_sub.entries.variables.kwargs import EPOCH
|
from ytdl_sub.entries.variables.kwargs import EPOCH
|
||||||
from ytdl_sub.entries.variables.kwargs import EXT
|
from ytdl_sub.entries.variables.kwargs import EXT
|
||||||
from ytdl_sub.entries.variables.kwargs import EXTRACTOR
|
from ytdl_sub.entries.variables.kwargs import EXTRACTOR
|
||||||
from ytdl_sub.entries.variables.kwargs import EXTRACTOR_KEY
|
from ytdl_sub.entries.variables.kwargs import EXTRACTOR_KEY
|
||||||
from ytdl_sub.entries.variables.kwargs import IE_KEY
|
|
||||||
from ytdl_sub.entries.variables.kwargs import PLAYLIST_COUNT
|
|
||||||
from ytdl_sub.entries.variables.kwargs import PLAYLIST_ENTRY
|
|
||||||
from ytdl_sub.entries.variables.kwargs import PLAYLIST_INDEX
|
|
||||||
from ytdl_sub.entries.variables.kwargs import PLAYLIST_TITLE
|
|
||||||
from ytdl_sub.entries.variables.kwargs import TITLE
|
from ytdl_sub.entries.variables.kwargs import TITLE
|
||||||
from ytdl_sub.entries.variables.kwargs import UID
|
from ytdl_sub.entries.variables.kwargs import UID
|
||||||
from ytdl_sub.entries.variables.kwargs import UPLOAD_DATE
|
from ytdl_sub.entries.variables.kwargs import UPLOAD_DATE
|
||||||
|
|
@ -66,21 +62,21 @@ def mock_entry_dict_factory(mock_downloaded_file_path) -> Callable:
|
||||||
entry_dict = {
|
entry_dict = {
|
||||||
UID: uid,
|
UID: uid,
|
||||||
EPOCH: 1596878400,
|
EPOCH: 1596878400,
|
||||||
PLAYLIST_TITLE: playlist_title,
|
v.playlist_title.metadata_key: playlist_title,
|
||||||
PLAYLIST_INDEX: playlist_index,
|
v.playlist_index.metadata_key: playlist_index,
|
||||||
PLAYLIST_COUNT: playlist_count,
|
v.playlist_count.metadata_key: playlist_count,
|
||||||
EXTRACTOR: "mock-entry-dict",
|
EXTRACTOR: "mock-entry-dict",
|
||||||
EXTRACTOR_KEY: "mock-extractor-key",
|
EXTRACTOR_KEY: "mock-extractor-key",
|
||||||
TITLE: f"Mock Entry {uid}",
|
TITLE: f"Mock Entry {uid}",
|
||||||
EXT: "mp4",
|
EXT: "mp4",
|
||||||
UPLOAD_DATE: upload_date,
|
UPLOAD_DATE: upload_date,
|
||||||
WEBPAGE_URL: f"https://{uid}.com",
|
WEBPAGE_URL: f"https://{uid}.com",
|
||||||
PLAYLIST_ENTRY: {"thumbnails": []},
|
v.playlist_metadata.metadata_key: {"thumbnails": []},
|
||||||
DESCRIPTION: "The Description",
|
DESCRIPTION: "The Description",
|
||||||
}
|
}
|
||||||
|
|
||||||
if is_youtube_channel:
|
if is_youtube_channel:
|
||||||
entry_dict[PLAYLIST_ENTRY]["thumbnails"] = [
|
entry_dict[v.playlist_metadata.metadata_key]["thumbnails"] = [
|
||||||
{
|
{
|
||||||
"id": "avatar_uncropped",
|
"id": "avatar_uncropped",
|
||||||
"url": "https://avatar_uncropped.com",
|
"url": "https://avatar_uncropped.com",
|
||||||
|
|
|
||||||
|
|
@ -48,6 +48,10 @@ class TestArrayFunctions:
|
||||||
output = single_variable_output("{%array_apply(['a', 'b', 'c'], %capitalize)}")
|
output = single_variable_output("{%array_apply(['a', 'b', 'c'], %capitalize)}")
|
||||||
assert output == ["A", "B", "C"]
|
assert output == ["A", "B", "C"]
|
||||||
|
|
||||||
|
def test_array_reduce(self):
|
||||||
|
output = single_variable_output("{%array_reduce([1, 2, 3, 4], %add)}")
|
||||||
|
assert output == 10
|
||||||
|
|
||||||
def test_array_enumerate(self):
|
def test_array_enumerate(self):
|
||||||
output = (
|
output = (
|
||||||
Script(
|
Script(
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue