danvics/ytdl-sub
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

somehow working

Browse source
This commit is contained in:
Jesse Bannon 2023-12-07 15:49:44 -08:00
parent 21e58dbb25
commit 1ebf91888e
7 changed files with 127 additions and 59 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

8
src/ytdl_sub/config/overrides.py
View file

@ -127,8 +127,8 @@ class Overrides(DictFormatterValidator, Scriptable):
if function_overrides:
script.add(function_overrides)
return str(
script.add({"tmp_var": formatter.format_string}).resolve(unresolvable=unresolvable)[
"tmp_var"
]
return (
script.add({"tmp_var": formatter.format_string})
.resolve(unresolvable=unresolvable)
.get_str("tmp_var")
)

84
src/ytdl_sub/config/preset.py
View file

@ -27,10 +27,12 @@ from ytdl_sub.entries.script.variable_scripts import VARIABLE_SCRIPTS
from ytdl_sub.prebuilt_presets import PREBUILT_PRESET_NAMES
from ytdl_sub.prebuilt_presets import PUBLISHED_PRESET_NAMES
from ytdl_sub.script.script import Script
from ytdl_sub.script.script import ScriptBuilder
from ytdl_sub.script.utils.exceptions import VariableDoesNotExist
from ytdl_sub.utils.exceptions import StringFormattingVariableNotFoundException
from ytdl_sub.utils.exceptions import ValidationException
from ytdl_sub.utils.logger import Logger
from ytdl_sub.utils.scriptable import Scriptable
from ytdl_sub.utils.yaml import dump_yaml
from ytdl_sub.validators.strict_dict_validator import StrictDictValidator
from ytdl_sub.validators.string_formatter_validators import DictFormatterValidator
@ -178,22 +180,41 @@ class Preset(_PresetShell):
return added_variables
@functools.cached_property
def _mock_script(self) -> Script:
def _cached_script_builder(self) -> ScriptBuilder:
# Set the formatter variables to be the overrides
variable_dict = copy.deepcopy(self.overrides.dict_with_format_strings)
source_variables = {
source_var: "dummy_string"
for source_var in self._source_variables
+ self.downloader_options.added_source_variables()
}
variable_dict = dict(source_variables, **variable_dict)
variable_dict = dict(variable_dict, **self._added_variables)
script = Script(variable_dict)
script.resolve(update=True)
script = ScriptBuilder(
Scriptable.add_sanitized_variables(self.overrides.dict_with_format_strings)
)
script.add(
Scriptable.add_sanitized_variables(
{source_var: "dummy_string" for source_var in self._source_variables}
)
)
return script
@property
def _script_builder(self) -> ScriptBuilder:
return copy.deepcopy(self._cached_script_builder)
@functools.cached_property
def _script_builder_with_added_variables(self) -> ScriptBuilder:
return self._script_builder.add(
Scriptable.add_sanitized_variables(
{source_var: "dummy_string" for source_var in self._added_variables}
)
)
@functools.cached_property
def _cached_script(self) -> Script:
"""
Contains actualized script which should hold all Override variables
"""
return self._script_builder_with_added_variables.partial_build(update=True)
@property
def _script(self) -> Script:
return copy.deepcopy(self._cached_script)
def __validate_and_get_plugins(self) -> PresetPlugins:
preset_plugins = PresetPlugins()
@ -209,30 +230,43 @@ class Preset(_PresetShell):
return preset_plugins
def __validate_added_variables(self):
self.downloader_options.validate_with_variables(script=copy.deepcopy(self._mock_script))
script_builder = self._script_builder
self.downloader_options.validate_with_variables(script=copy.deepcopy(script_builder))
script_builder.add(
Scriptable.add_sanitized_variables(
{name: "dummy_string" for name in self.downloader_options.added_source_variables()}
)
)
for _, plugin_options in sorted(
self.plugins.zipped(), key=lambda pl: pl[0].priority.modify_entry
):
# Validate current plugin using source + added plugin variables
plugin_options.validate_with_variables(script=copy.deepcopy(self._mock_script))
plugin_options.validate_with_variables(script=copy.deepcopy(script_builder))
script_builder.add(
Scriptable.add_sanitized_variables(
{
name: "dummy_string"
for name in self.downloader_options.added_source_variables()
}
)
)
def __validate_override_string_formatter_validator(
self,
formatter_validator: Union[StringFormatterValidator, OverridesStringFormatterValidator],
):
script = copy.deepcopy(self._mock_script)
try:
script.add({"tmp_var": formatter_validator.format_string})
except VariableDoesNotExist as exc:
raise StringFormattingVariableNotFoundException(exc) from exc
unresolved: Optional[Set[str]] = (
{VARIABLES.entry_metadata.variable_name}
) -> None:
unresolvable = (
set([VARIABLES.entry_metadata.variable_name] + list(self._added_variables.keys()))
if isinstance(formatter_validator, OverridesStringFormatterValidator)
else None
)
_ = script.resolve(unresolvable=unresolved)["tmp_var"] # TODO: error if not present
try:
self._script.add({"tmp_var": formatter_validator.format_string}).resolve(
unresolvable=unresolvable
).get("tmp_var")
except VariableDoesNotExist as exc:
raise StringFormattingVariableNotFoundException(exc) from exc
def __recursive_preset_validate(
self,

3
src/ytdl_sub/config/preset_options.py
View file

@ -6,6 +6,7 @@ from typing import TypeVar
from ytdl_sub.config.defaults import DEFAULT_DOWNLOAD_ARCHIVE_NAME
from ytdl_sub.script.script import Script
from ytdl_sub.script.script import ScriptBuilder
from ytdl_sub.utils.exceptions import ValidationException
from ytdl_sub.validators.file_path_validators import OverridesStringFormatterFilePathValidator
from ytdl_sub.validators.file_path_validators import StringFormatterFileNameValidator
@ -53,7 +54,7 @@ class OptionsValidator(Validator, ABC):
"""
return []
def validate_with_variables(self, script: Script) -> None:
def validate_with_variables(self, script: ScriptBuilder) -> None:
"""
Optional validation after init with the session's source and override variables.

10
src/ytdl_sub/downloaders/url/validators.py
View file

@ -6,6 +6,7 @@ from typing import Optional
from ytdl_sub.config.preset_options import OptionsValidator
from ytdl_sub.script.script import Script
from ytdl_sub.script.script import ScriptBuilder
from ytdl_sub.validators.strict_dict_validator import StrictDictValidator
from ytdl_sub.validators.string_formatter_validators import DictFormatterValidator
from ytdl_sub.validators.string_formatter_validators import OverridesStringFormatterValidator
@ -252,20 +253,21 @@ class MultiUrlValidator(OptionsValidator):
"""
return list(self._urls.list[0].variables.keys)
def validate_with_variables(self, script: Script) -> None:
def validate_with_variables(self, script: ScriptBuilder) -> None:
"""
Ensures new variables added are not existing variables
"""
# Apply formatting to each new source variable, ensure it resolves
for collection_url in self.urls.list:
script.add(collection_url.variables.dict_with_format_strings)
script.resolve(update=True)
resolved_script = script.partial_build(update=True)
# Ensure at least URL is non-empty
has_non_empty_url = False
for url_validator in self.urls.list:
script.add({"tmp_var_url": url_validator.url.format_string})
has_non_empty_url |= bool(str(script.resolve().get("tmp_var_url")))
resolved_script.add({"tmp_var_url": url_validator.url.format_string})
has_non_empty_url |= bool(str(resolved_script.resolve().get_native("tmp_var_url")))
if not has_non_empty_url:
raise self._validation_exception("Must contain at least one url that is non-empty")

3
src/ytdl_sub/plugins/regex.py
View file

@ -12,6 +12,7 @@ from ytdl_sub.entries.entry import Entry
from ytdl_sub.entries.script.variable_scripts import VARIABLE_SCRIPTS
from ytdl_sub.entries.variables.kwargs import YTDL_SUB_REGEX_SOURCE_VARS
from ytdl_sub.script.script import Script
from ytdl_sub.script.script import ScriptBuilder
from ytdl_sub.utils.exceptions import RegexNoMatchException
from ytdl_sub.utils.exceptions import StringFormattingVariableNotFoundException
from ytdl_sub.utils.logger import Logger
@ -214,7 +215,7 @@ class RegexOptions(OptionsDictValidator):
"""
return self._skip_if_match_fails
def validate_with_variables(self, script: Script) -> None:
def validate_with_variables(self, script: ScriptBuilder) -> None:
for key, regex_options in self.source_variable_capture_dict.items():
# Ensure each variable getting captured is a source variable
if key not in script._variables:

70
src/ytdl_sub/script/script.py
View file

@ -1,8 +1,10 @@
import copy
from collections import defaultdict
from typing import Dict
from typing import List
from typing import Optional
from typing import Set
from typing import Tuple
from ytdl_sub.script.functions import Functions
from ytdl_sub.script.parser import parse
@ -369,28 +371,45 @@ class ScriptBuilder:
return self
@property
def _missing_metadata(self) -> Dict[str, Set[str]]:
missing_metadata: Dict[str, Set[str]] = {}
def _missing_metadata(self) -> Tuple[Dict[str, Set[str]], Dict[str, Set[str]]]:
variables_missing_metadata: Dict[str, Set[str]] = defaultdict(set)
functions_missing_metadata: Dict[str, Set[str]] = defaultdict(set)
defined_variables: Set[str] = set(self._variables.keys())
defined_functions: Set[str] = set(self._functions.keys())
for name, variable in self._variables.items():
missing_metadata[name] = {var.name for var in variable.variables}.difference(
defined_variables
)
missing_metadata[name].update(
{fun.name for fun in variable.custom_functions}.difference(defined_functions)
)
for name, function in self._functions.items():
missing_metadata[name] = {var.name for var in function.variables}.difference(
defined_variables
)
missing_metadata[name].update(
{fun.name for fun in function.custom_functions}.difference(defined_functions)
)
while True:
variables_missing_metadata_snapshot = copy.deepcopy(variables_missing_metadata)
functions_missing_metadata_snapshot = copy.deepcopy(functions_missing_metadata)
return missing_metadata
for name, variable in self._variables.items():
if diff := {var.name for var in variable.variables}.difference(defined_variables):
variables_missing_metadata[name].update(diff)
if diff := {fun.name for fun in variable.custom_functions}.difference(
defined_functions
):
variables_missing_metadata[name].update(diff)
for name, function in self._functions.items():
if diff := {var.name for var in function.variables}.difference(defined_variables):
functions_missing_metadata[name].update(diff)
if diff := {fun.name for fun in function.custom_functions}.difference(
defined_functions
):
functions_missing_metadata[name].update(diff)
if (
variables_missing_metadata == variables_missing_metadata_snapshot
and functions_missing_metadata == functions_missing_metadata_snapshot
):
break
defined_variables -= set(variables_missing_metadata.keys())
defined_functions -= set(functions_missing_metadata.keys())
return variables_missing_metadata, functions_missing_metadata
@classmethod
def _build(cls, variables: Dict[str, SyntaxTree], functions: Dict[str, SyntaxTree]) -> Script:
@ -400,26 +419,29 @@ class ScriptBuilder:
script._validate()
return script
def partial_build(self) -> Script:
missing_metadata = self._missing_metadata
def partial_build(self, update: bool = False) -> Script:
missing_variables, missing_functions = self._missing_metadata
maybe_resolvable_variables: Dict[str, SyntaxTree] = {
name: variable
for name, variable in self._variables.items()
if name not in missing_metadata
if name not in missing_variables
}
maybe_resolvable_functions: Dict[str, SyntaxTree] = {
name: function
for name, function in self._functions.items()
if name not in missing_metadata
if name not in missing_functions
}
return self._build(
script = self._build(
variables=maybe_resolvable_variables, functions=maybe_resolvable_functions
)
if update:
script.resolve(update=True)
return script
def build(self) -> Script:
for name, missing_metadata in self._missing_metadata.items():
if missing_metadata:
for missing_metadata in self._missing_metadata:
for name, missing in missing_metadata.items():
raise ScriptBuilderMissingDefinitions(
f"{name} is missing the following definitions: {', '.join(missing_metadata)}"
)

8
src/ytdl_sub/utils/scriptable.py
View file

@ -1,4 +1,5 @@
from abc import ABC
from typing import Dict
from typing import Set
from ytdl_sub.entries.script.variable_scripts import VARIABLE_SCRIPTS
@ -6,6 +7,13 @@ from ytdl_sub.script.script import Script
class Scriptable(ABC):
@classmethod
def add_sanitized_variables(cls, variables: Dict[str, str]) -> Dict[str, str]:
sanitized_variables = {
f"{name}_sanitized": f"{{%sanitize({name})}}" for name in variables.keys()
}
return dict(variables, **sanitized_variables)
def __init__(self):
self.script = Script(VARIABLE_SCRIPTS)
self.unresolvable: Set[str] = set()