docs docs docs
This commit is contained in:
parent
ea701efd91
commit
720d4e97e4
11 changed files with 176 additions and 24 deletions
|
|
@ -1,3 +1,4 @@
|
||||||
|
# pylint: disable=protected-access
|
||||||
from abc import ABC
|
from abc import ABC
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from typing import Any
|
from typing import Any
|
||||||
|
|
@ -35,7 +36,7 @@ class BaseEntry(ABC):
|
||||||
self._kwargs = entry_dict
|
self._kwargs = entry_dict
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def uid(self: "BaseEntry") -> str:
|
def uid(self) -> str:
|
||||||
"""
|
"""
|
||||||
Returns
|
Returns
|
||||||
-------
|
-------
|
||||||
|
|
@ -45,7 +46,7 @@ class BaseEntry(ABC):
|
||||||
return str(self._kwargs[v.uid.metadata_key])
|
return str(self._kwargs[v.uid.metadata_key])
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def download_archive_extractor(self: "BaseEntry") -> str:
|
def download_archive_extractor(self) -> str:
|
||||||
"""
|
"""
|
||||||
The extractor name used in yt-dlp download archives
|
The extractor name used in yt-dlp download archives
|
||||||
"""
|
"""
|
||||||
|
|
@ -59,14 +60,14 @@ class BaseEntry(ABC):
|
||||||
).lower()
|
).lower()
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def title(self: "BaseEntry") -> str:
|
def title(self) -> str:
|
||||||
"""
|
"""
|
||||||
The title of the entry. If a title does not exist, returns its unique ID.
|
The title of the entry. If a title does not exist, returns its unique ID.
|
||||||
"""
|
"""
|
||||||
return self._kwargs_get(v.title.metadata_key, self.uid)
|
return self._kwargs_get(v.title.metadata_key, self.uid)
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def webpage_url(self: "BaseEntry") -> str:
|
def webpage_url(self) -> str:
|
||||||
"""
|
"""
|
||||||
The url to the webpage.
|
The url to the webpage.
|
||||||
"""
|
"""
|
||||||
|
|
@ -78,7 +79,7 @@ class BaseEntry(ABC):
|
||||||
return "info.json"
|
return "info.json"
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def uploader_id(self: "BaseEntry") -> str:
|
def uploader_id(self) -> str:
|
||||||
"""
|
"""
|
||||||
The uploader id if it exists, otherwise return the unique ID.
|
The uploader id if it exists, otherwise return the unique ID.
|
||||||
"""
|
"""
|
||||||
|
|
|
||||||
|
|
@ -1,3 +1,4 @@
|
||||||
|
# pylint: disable=protected-access
|
||||||
import copy
|
import copy
|
||||||
import json
|
import json
|
||||||
import os
|
import os
|
||||||
|
|
@ -75,6 +76,10 @@ class Entry(BaseEntry, Scriptable):
|
||||||
def add_injected_variables(
|
def add_injected_variables(
|
||||||
self, download_entry: "Entry", download_idx: int, upload_date_idx: int
|
self, download_entry: "Entry", download_idx: int, upload_date_idx: int
|
||||||
) -> "Entry":
|
) -> "Entry":
|
||||||
|
"""
|
||||||
|
Adds variables that get injected into the Entry script that aren't available at
|
||||||
|
metadata scrape time (only after the actual download).
|
||||||
|
"""
|
||||||
self.add(
|
self.add(
|
||||||
{
|
{
|
||||||
# Tracks number of entries downloaded
|
# Tracks number of entries downloaded
|
||||||
|
|
@ -211,7 +216,7 @@ class Entry(BaseEntry, Scriptable):
|
||||||
return maybe_prior_variables
|
return maybe_prior_variables
|
||||||
|
|
||||||
@final
|
@final
|
||||||
def to_dict(self) -> Dict[str, str]:
|
def to_dict(self) -> Dict[str, Any]:
|
||||||
"""
|
"""
|
||||||
Returns
|
Returns
|
||||||
-------
|
-------
|
||||||
|
|
|
||||||
|
|
@ -31,10 +31,25 @@ class Functions(
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def is_built_in(cls, name: str) -> bool:
|
def is_built_in(cls, name: str) -> bool:
|
||||||
|
"""
|
||||||
|
Returns
|
||||||
|
-------
|
||||||
|
True if the name exists as a built-in function or custom function. False otherwise.
|
||||||
|
"""
|
||||||
return hasattr(cls, name) or hasattr(cls, f"{name}_") or name in cls._custom_functions
|
return hasattr(cls, name) or hasattr(cls, f"{name}_") or name in cls._custom_functions
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def get(cls, name: str) -> Callable[..., Resolvable]:
|
def get(cls, name: str) -> Callable[..., Resolvable]:
|
||||||
|
"""
|
||||||
|
Returns
|
||||||
|
-------
|
||||||
|
The actual Python callable for the function of the given name.
|
||||||
|
|
||||||
|
Raises
|
||||||
|
------
|
||||||
|
FunctionDoesNotExistRuntimeException
|
||||||
|
If the function does not exist.
|
||||||
|
"""
|
||||||
if hasattr(cls, name):
|
if hasattr(cls, name):
|
||||||
return getattr(cls, name)
|
return getattr(cls, name)
|
||||||
if hasattr(cls, f"{name}_"):
|
if hasattr(cls, f"{name}_"):
|
||||||
|
|
@ -46,6 +61,14 @@ class Functions(
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def register_function(cls, function: Callable[..., Resolvable]) -> None:
|
def register_function(cls, function: Callable[..., Resolvable]) -> None:
|
||||||
|
"""
|
||||||
|
Adds a function to the suite of offered functions.
|
||||||
|
|
||||||
|
Parameters
|
||||||
|
----------
|
||||||
|
function
|
||||||
|
A static function whose name will be used as the offered function name.
|
||||||
|
"""
|
||||||
if cls.is_built_in(function.__name__):
|
if cls.is_built_in(function.__name__):
|
||||||
raise ValueError(
|
raise ValueError(
|
||||||
f"Cannot register a function with name {function.__name__} "
|
f"Cannot register a function with name {function.__name__} "
|
||||||
|
|
|
||||||
|
|
@ -18,6 +18,14 @@ from ytdl_sub.script.utils.exceptions import FunctionRuntimeException
|
||||||
class ArrayFunctions:
|
class ArrayFunctions:
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def array(maybe_array: AnyArgument) -> Array:
|
def array(maybe_array: AnyArgument) -> Array:
|
||||||
|
"""
|
||||||
|
Tries to cast an unknown variable type to an Array.
|
||||||
|
|
||||||
|
Raises
|
||||||
|
------
|
||||||
|
FunctionRuntimeException
|
||||||
|
If the input type is not actually an Array.
|
||||||
|
"""
|
||||||
if not isinstance(maybe_array, Array):
|
if not isinstance(maybe_array, Array):
|
||||||
raise FunctionRuntimeException(
|
raise FunctionRuntimeException(
|
||||||
f"Tried and failed to cast {maybe_array.type_name()} as an Array"
|
f"Tried and failed to cast {maybe_array.type_name()} as an Array"
|
||||||
|
|
@ -26,6 +34,9 @@ class ArrayFunctions:
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def array_size(array: Array) -> Integer:
|
def array_size(array: Array) -> Integer:
|
||||||
|
"""
|
||||||
|
Returns the size of an Array.
|
||||||
|
"""
|
||||||
return Integer(len(array.value))
|
return Integer(len(array.value))
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
|
|
|
||||||
|
|
@ -33,5 +33,8 @@ def _from_json(out: Any) -> Resolvable:
|
||||||
|
|
||||||
class JsonFunctions:
|
class JsonFunctions:
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def from_json(argument: AnyArgument) -> AnyArgument:
|
def from_json(argument: String) -> AnyArgument:
|
||||||
|
"""
|
||||||
|
Converts a JSON string into an actual type.
|
||||||
|
"""
|
||||||
return _from_json(json.loads(argument.value))
|
return _from_json(json.loads(argument.value))
|
||||||
|
|
|
||||||
|
|
@ -17,6 +17,14 @@ from ytdl_sub.script.utils.exceptions import KeyNotHashableRuntimeException
|
||||||
class MapFunctions:
|
class MapFunctions:
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def map(maybe_mapping: AnyArgument) -> Map:
|
def map(maybe_mapping: AnyArgument) -> Map:
|
||||||
|
"""
|
||||||
|
Tries to cast an unknown variable type to a Map.
|
||||||
|
|
||||||
|
Raises
|
||||||
|
------
|
||||||
|
FunctionRuntimeException
|
||||||
|
If the input type is not actually a Map.
|
||||||
|
"""
|
||||||
if not isinstance(maybe_mapping, Map):
|
if not isinstance(maybe_mapping, Map):
|
||||||
raise FunctionRuntimeException(
|
raise FunctionRuntimeException(
|
||||||
f"Tried and failed to cast {maybe_mapping.type_name()} as a Map"
|
f"Tried and failed to cast {maybe_mapping.type_name()} as a Map"
|
||||||
|
|
@ -25,6 +33,9 @@ class MapFunctions:
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def map_size(mapping: Map) -> Integer:
|
def map_size(mapping: Map) -> Integer:
|
||||||
|
"""
|
||||||
|
Returns the size of a Map.
|
||||||
|
"""
|
||||||
return Integer(len(mapping.value))
|
return Integer(len(mapping.value))
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
|
|
|
||||||
|
|
@ -144,11 +144,9 @@ class Script:
|
||||||
if lambda_type := spec.is_lambda_like:
|
if lambda_type := spec.is_lambda_like:
|
||||||
|
|
||||||
lambda_function_names = set(
|
lambda_function_names = set(
|
||||||
[
|
|
||||||
lamb.value
|
lamb.value
|
||||||
for lamb in SyntaxTree(function.args).lambdas
|
for lamb in SyntaxTree(function.args).lambdas
|
||||||
if isinstance(lamb, Lambda)
|
if isinstance(lamb, Lambda)
|
||||||
]
|
|
||||||
)
|
)
|
||||||
|
|
||||||
# Only case len(lambda_function_names) > 1 is when used in if-statements
|
# Only case len(lambda_function_names) > 1 is when used in if-statements
|
||||||
|
|
@ -342,11 +340,46 @@ class Script:
|
||||||
unresolvable: Optional[Set[str]] = None,
|
unresolvable: Optional[Set[str]] = None,
|
||||||
update: bool = False,
|
update: bool = False,
|
||||||
) -> ScriptOutput:
|
) -> ScriptOutput:
|
||||||
|
"""
|
||||||
|
Resolves the script
|
||||||
|
|
||||||
|
Parameters
|
||||||
|
----------
|
||||||
|
resolved
|
||||||
|
Optional. Pre-resolved variables that should be used instead of what is in the script.
|
||||||
|
unresolvable
|
||||||
|
Optional. Unresolvable variables that will be ignored in resolution, including all
|
||||||
|
variables with a dependency to them.
|
||||||
|
update
|
||||||
|
Whether to update the script's internal values with the resolved variables instead of
|
||||||
|
their original definition. This helps avoid re-evaluated the same variables repeatedly.
|
||||||
|
|
||||||
|
Returns
|
||||||
|
-------
|
||||||
|
ScriptOutput
|
||||||
|
Containing all resolved variables.
|
||||||
|
"""
|
||||||
return self._resolve(
|
return self._resolve(
|
||||||
pre_resolved=resolved, unresolvable=unresolvable, update=update, output_filter=None
|
pre_resolved=resolved, unresolvable=unresolvable, update=update, output_filter=None
|
||||||
)
|
)
|
||||||
|
|
||||||
def add(self, variables: Dict[str, str], unresolvable: Optional[Set[str]] = None) -> "Script":
|
def add(self, variables: Dict[str, str], unresolvable: Optional[Set[str]] = None) -> "Script":
|
||||||
|
"""
|
||||||
|
Adds parses and adds new variables to the script.
|
||||||
|
|
||||||
|
Parameters
|
||||||
|
----------
|
||||||
|
variables
|
||||||
|
Mapping containing variable name to definition.
|
||||||
|
unresolvable
|
||||||
|
Optional. Set of unresolved variables that the new variables may contain, but the
|
||||||
|
script does not (yet).
|
||||||
|
|
||||||
|
Returns
|
||||||
|
-------
|
||||||
|
Script
|
||||||
|
self
|
||||||
|
"""
|
||||||
added_variables_to_validate: Set[str] = set()
|
added_variables_to_validate: Set[str] = set()
|
||||||
for variable_name, variable_definition in variables.items():
|
for variable_name, variable_definition in variables.items():
|
||||||
self._variables[variable_name] = parse(
|
self._variables[variable_name] = parse(
|
||||||
|
|
|
||||||
|
|
@ -10,13 +10,25 @@ from ytdl_sub.script.utils.exceptions import ScriptVariableNotResolved
|
||||||
class ScriptOutput:
|
class ScriptOutput:
|
||||||
output: Dict[str, Resolvable]
|
output: Dict[str, Resolvable]
|
||||||
|
|
||||||
def as_resolvable(self) -> Dict[str, Resolvable]:
|
|
||||||
return self.output
|
|
||||||
|
|
||||||
def as_native(self) -> Dict[str, Any]:
|
def as_native(self) -> Dict[str, Any]:
|
||||||
|
"""
|
||||||
|
Returns
|
||||||
|
-------
|
||||||
|
The script output as native python types
|
||||||
|
"""
|
||||||
return {name: out.native for name, out in self.output.items()}
|
return {name: out.native for name, out in self.output.items()}
|
||||||
|
|
||||||
def get(self, name: str) -> Resolvable:
|
def get(self, name: str) -> Resolvable:
|
||||||
|
"""
|
||||||
|
Returns
|
||||||
|
-------
|
||||||
|
The script output's variable as a resolvable type
|
||||||
|
|
||||||
|
Raises
|
||||||
|
------
|
||||||
|
ScriptVariableNotResolved
|
||||||
|
The variable name requested did not resolve
|
||||||
|
"""
|
||||||
if name not in self.output:
|
if name not in self.output:
|
||||||
raise ScriptVariableNotResolved(
|
raise ScriptVariableNotResolved(
|
||||||
f"Tried to access resolved variable {name}, but it has not resolved"
|
f"Tried to access resolved variable {name}, but it has not resolved"
|
||||||
|
|
@ -24,12 +36,17 @@ class ScriptOutput:
|
||||||
return self.output[name]
|
return self.output[name]
|
||||||
|
|
||||||
def get_native(self, name: str) -> Any:
|
def get_native(self, name: str) -> Any:
|
||||||
|
"""
|
||||||
|
Returns
|
||||||
|
-------
|
||||||
|
The script output's variable as native python type
|
||||||
|
"""
|
||||||
return self.get(name).native
|
return self.get(name).native
|
||||||
|
|
||||||
def get_str(self, name: str) -> str:
|
def get_str(self, name: str) -> str:
|
||||||
|
"""
|
||||||
|
Returns
|
||||||
|
-------
|
||||||
|
The script output's variable as a string
|
||||||
|
"""
|
||||||
return str(self.get(name))
|
return str(self.get(name))
|
||||||
|
|
||||||
def get_int(self, name: str) -> int:
|
|
||||||
out = self.get_native(name)
|
|
||||||
assert isinstance(out, int)
|
|
||||||
return out
|
|
||||||
|
|
|
||||||
|
|
@ -73,22 +73,36 @@ class ReturnableArgumentB(ValueArgument, NamedType, ABC):
|
||||||
@dataclass(frozen=True)
|
@dataclass(frozen=True)
|
||||||
class AnyArgument(ReturnableArgument, ReturnableArgumentA, ReturnableArgumentB, ABC):
|
class AnyArgument(ReturnableArgument, ReturnableArgumentA, ReturnableArgumentB, ABC):
|
||||||
"""
|
"""
|
||||||
Human-readable name for FutureResolvable
|
Human-readable name for Resolvable
|
||||||
"""
|
"""
|
||||||
|
|
||||||
|
|
||||||
@dataclass(frozen=True)
|
@dataclass(frozen=True)
|
||||||
class Resolvable(AnyArgument, ABC):
|
class Resolvable(AnyArgument, ABC):
|
||||||
|
"""
|
||||||
|
A type that is resolved into a native Python type (and have no dependencies to other types).
|
||||||
|
"""
|
||||||
|
|
||||||
def __str__(self) -> str:
|
def __str__(self) -> str:
|
||||||
return str(self.value)
|
return str(self.value)
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def native(self) -> Any:
|
def native(self) -> Any:
|
||||||
|
"""
|
||||||
|
Returns
|
||||||
|
-------
|
||||||
|
The resolvable in its native form
|
||||||
|
"""
|
||||||
return self.value
|
return self.value
|
||||||
|
|
||||||
|
|
||||||
@dataclass(frozen=True)
|
@dataclass(frozen=True)
|
||||||
class FutureResolvable(AnyArgument, ABC):
|
class FutureResolvable(AnyArgument, ABC):
|
||||||
|
"""
|
||||||
|
Used when parsing, it is an unresolved type that will eventually resolve to a known type
|
||||||
|
(i.e. Maps, Arrays)
|
||||||
|
"""
|
||||||
|
|
||||||
@abstractmethod
|
@abstractmethod
|
||||||
def future_resolvable_type(self) -> Type[Resolvable]:
|
def future_resolvable_type(self) -> Type[Resolvable]:
|
||||||
pass
|
pass
|
||||||
|
|
@ -96,27 +110,47 @@ class FutureResolvable(AnyArgument, ABC):
|
||||||
|
|
||||||
@dataclass(frozen=True)
|
@dataclass(frozen=True)
|
||||||
class Hashable(Resolvable, ABC):
|
class Hashable(Resolvable, ABC):
|
||||||
|
"""
|
||||||
|
Resolvable type that can be used as hashes (i.e. in Maps)
|
||||||
|
"""
|
||||||
|
|
||||||
pass
|
pass
|
||||||
|
|
||||||
|
|
||||||
@dataclass(frozen=True)
|
@dataclass(frozen=True)
|
||||||
class NonHashable(NamedType, ABC):
|
class NonHashable(NamedType, ABC):
|
||||||
|
"""
|
||||||
|
Type that is known to never be hashable.
|
||||||
|
"""
|
||||||
|
|
||||||
pass
|
pass
|
||||||
|
|
||||||
|
|
||||||
@dataclass(frozen=True)
|
@dataclass(frozen=True)
|
||||||
class ResolvableToJson(Resolvable, ABC):
|
class ResolvableToJson(Resolvable, ABC):
|
||||||
|
"""
|
||||||
|
Types whose string values should be resolved to JSON (i.e. Maps, Arrays)
|
||||||
|
"""
|
||||||
|
|
||||||
def __str__(self):
|
def __str__(self):
|
||||||
return json.dumps(self.native)
|
return json.dumps(self.native)
|
||||||
|
|
||||||
|
|
||||||
@dataclass(frozen=True)
|
@dataclass(frozen=True)
|
||||||
class ResolvableT(Hashable, ABC, Generic[T]):
|
class ResolvableT(Hashable, ABC, Generic[T]):
|
||||||
|
"""
|
||||||
|
Resolvable types that resolve to the generic T
|
||||||
|
"""
|
||||||
|
|
||||||
value: T
|
value: T
|
||||||
|
|
||||||
|
|
||||||
@dataclass(frozen=True)
|
@dataclass(frozen=True)
|
||||||
class Numeric(ResolvableT[NumericT], Hashable, ABC, Generic[NumericT]):
|
class Numeric(ResolvableT[NumericT], ABC, Generic[NumericT]):
|
||||||
|
"""
|
||||||
|
Resolvable numeric types (int/float)
|
||||||
|
"""
|
||||||
|
|
||||||
pass
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
|
@ -131,12 +165,12 @@ class Float(Numeric[float], Argument):
|
||||||
|
|
||||||
|
|
||||||
@dataclass(frozen=True)
|
@dataclass(frozen=True)
|
||||||
class Boolean(ResolvableT[bool], Hashable, Argument):
|
class Boolean(ResolvableT[bool], Argument):
|
||||||
pass
|
pass
|
||||||
|
|
||||||
|
|
||||||
@dataclass(frozen=True)
|
@dataclass(frozen=True)
|
||||||
class String(ResolvableT[str], Hashable, Argument):
|
class String(ResolvableT[str], Argument):
|
||||||
pass
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
|
@ -199,4 +233,8 @@ class LambdaThree(Lambda):
|
||||||
|
|
||||||
@dataclass(frozen=True)
|
@dataclass(frozen=True)
|
||||||
class LambdaReduce(LambdaTwo):
|
class LambdaReduce(LambdaTwo):
|
||||||
|
"""
|
||||||
|
Type-hinting for functions that apply a reduce-operation using a lambda (two arguments)
|
||||||
|
"""
|
||||||
|
|
||||||
pass
|
pass
|
||||||
|
|
|
||||||
|
|
@ -42,6 +42,11 @@ class SyntaxTree(VariableDependency):
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def maybe_resolvable(self) -> Optional[Resolvable]:
|
def maybe_resolvable(self) -> Optional[Resolvable]:
|
||||||
|
"""
|
||||||
|
Returns
|
||||||
|
-------
|
||||||
|
A resolvable if the AST contains a single type that is resolvable. None otherwise.
|
||||||
|
"""
|
||||||
if len(self.ast) == 1 and isinstance(self.ast[0], Resolvable):
|
if len(self.ast) == 1 and isinstance(self.ast[0], Resolvable):
|
||||||
return self.ast[0]
|
return self.ast[0]
|
||||||
return None
|
return None
|
||||||
|
|
|
||||||
|
|
@ -106,6 +106,11 @@ def is_type_compatible(
|
||||||
arg: NamedType,
|
arg: NamedType,
|
||||||
expected_arg_type: Type[Resolvable | Optional[Resolvable]],
|
expected_arg_type: Type[Resolvable | Optional[Resolvable]],
|
||||||
) -> bool:
|
) -> bool:
|
||||||
|
"""
|
||||||
|
Returns
|
||||||
|
-------
|
||||||
|
True if arg is compatible with expected_arg_type. False otherwise.
|
||||||
|
"""
|
||||||
arg_type: Type[NamedType] = arg.__class__
|
arg_type: Type[NamedType] = arg.__class__
|
||||||
if isinstance(arg, BuiltInFunctionType):
|
if isinstance(arg, BuiltInFunctionType):
|
||||||
arg_type = arg.output_type() # built-in function
|
arg_type = arg.output_type() # built-in function
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue