massive
This commit is contained in:
Jesse Bannon
parent
6236fb004a
commit
7d6666a8f2
11 changed files with 139 additions and 72 deletions
|
|
@ -127,17 +127,10 @@ class Overrides(DictFormatterValidator, Scriptable):
|
|||
)
|
||||
return ScriptUtils.add_sanitized_variables(initial_variables)
|
||||
|
||||
def initialize_script(
|
||||
self, subscription_name: str, unresolved_variables: Set[str]
|
||||
) -> "Overrides":
|
||||
def initialize_script(self, unresolved_variables: Set[str]) -> "Overrides":
|
||||
"""
|
||||
Initialize the override script with override variables + any unresolved variables
|
||||
Initialize the override script with any unresolved variables
|
||||
"""
|
||||
self.script.add(
|
||||
ScriptUtils.add_sanitized_variables(
|
||||
{SubscriptionVariables.subscription_name(): subscription_name}
|
||||
)
|
||||
)
|
||||
self.script.add(
|
||||
self.initial_variables(
|
||||
unresolved_variables={
|
||||
|
|
|
|||
|
|
@ -198,9 +198,7 @@ class Preset(_PresetShell):
|
|||
downloader_options=self.downloader_options,
|
||||
output_options=self.output_options,
|
||||
plugins=self.plugins,
|
||||
).initialize_overrides(
|
||||
subscription_name=self.name, overrides=self.overrides
|
||||
).ensure_proper_usage()
|
||||
).initialize_preset_overrides(overrides=self.overrides).ensure_proper_usage()
|
||||
|
||||
@property
|
||||
def name(self) -> str:
|
||||
|
|
|
|||
|
|
@ -13,13 +13,25 @@ from ytdl_sub.config.plugin.preset_plugins import PresetPlugins
|
|||
from ytdl_sub.config.preset_options import OutputOptions
|
||||
from ytdl_sub.config.validators.options import OptionsValidator
|
||||
from ytdl_sub.downloaders.url.validators import MultiUrlValidator
|
||||
from ytdl_sub.entries.variables.override_variables import SubscriptionVariables
|
||||
from ytdl_sub.entries.variables.override_variables import REQUIRED_OVERRIDE_VARIABLE_DEFINITIONS
|
||||
from ytdl_sub.entries.variables.override_variables import REQUIRED_OVERRIDE_VARIABLE_NAMES
|
||||
from ytdl_sub.script.script import Script
|
||||
from ytdl_sub.script.script import _is_function
|
||||
from ytdl_sub.utils.scriptable import BASE_SCRIPT
|
||||
from ytdl_sub.validators.string_formatter_validators import to_variable_dependency_format_string
|
||||
from ytdl_sub.validators.string_formatter_validators import validate_formatters
|
||||
|
||||
# Entry variables to mock during validation
|
||||
_DUMMY_ENTRY_VARIABLES: Dict[str, str] = {
|
||||
name: to_variable_dependency_format_string(
|
||||
# pylint: disable=protected-access
|
||||
script=BASE_SCRIPT,
|
||||
parsed_format_string=BASE_SCRIPT._variables[name]
|
||||
# pylint: enable=protected-access
|
||||
)
|
||||
for name in BASE_SCRIPT.variable_names
|
||||
}
|
||||
|
||||
|
||||
def _add_dummy_variables(variables: Iterable[str]) -> Dict[str, str]:
|
||||
dummy_variables: Dict[str, str] = {}
|
||||
|
|
@ -72,20 +84,7 @@ def _get_added_and_modified_variables(
|
|||
|
||||
|
||||
def _override_variables(overrides: Overrides) -> Set[str]:
|
||||
return set(list(overrides.initial_variables().keys())) | {
|
||||
SubscriptionVariables.subscription_name()
|
||||
}
|
||||
|
||||
|
||||
_DUMMY_ENTRY_VARIABLES: Dict[str, str] = {
|
||||
name: to_variable_dependency_format_string(
|
||||
# pylint: disable=protected-access
|
||||
script=BASE_SCRIPT,
|
||||
parsed_format_string=BASE_SCRIPT._variables[name]
|
||||
# pylint: enable=protected-access
|
||||
)
|
||||
for name in BASE_SCRIPT.variable_names
|
||||
}
|
||||
return set(list(overrides.initial_variables().keys()))
|
||||
|
||||
|
||||
class VariableValidation:
|
||||
|
|
@ -103,13 +102,13 @@ class VariableValidation:
|
|||
self.resolved_variables: Set[str] = set()
|
||||
self.unresolved_variables: Set[str] = set()
|
||||
|
||||
def initialize_overrides(
|
||||
self, subscription_name: str, overrides: Overrides
|
||||
) -> "VariableValidation":
|
||||
def initialize_preset_overrides(self, overrides: Overrides) -> "VariableValidation":
|
||||
"""
|
||||
Do some gymnastics to initialize the Overrides script.
|
||||
"""
|
||||
override_variables = _override_variables(overrides)
|
||||
override_variables = (
|
||||
set(list(overrides.initial_variables().keys())) | REQUIRED_OVERRIDE_VARIABLE_NAMES
|
||||
)
|
||||
|
||||
# Set resolved variables as all entry + override variables
|
||||
# at this point to generate every possible added/modified variable
|
||||
|
|
@ -145,9 +144,7 @@ class VariableValidation:
|
|||
# Initialize overrides with unresolved variables + modified variables to throw an error.
|
||||
# For modified variables, this is to prevent a resolve(update=True) to setting any
|
||||
# dependencies until it has been explicitly added
|
||||
overrides = overrides.initialize_script(
|
||||
subscription_name=subscription_name, unresolved_variables=self.unresolved_variables
|
||||
)
|
||||
overrides = overrides.initialize_script(unresolved_variables=self.unresolved_variables)
|
||||
|
||||
# copy the script and mock entry variables
|
||||
self.script = copy.deepcopy(overrides.script)
|
||||
|
|
@ -162,7 +159,16 @@ class VariableValidation:
|
|||
def _update_script(self) -> None:
|
||||
_ = self.script.resolve(unresolvable=self.unresolved_variables, update=True)
|
||||
|
||||
def _add_variables(self, plugin_op: PluginOperation, options: OptionsValidator) -> Set[str]:
|
||||
def _add_subscription_override_variables(self) -> None:
|
||||
"""
|
||||
Add dummy subscription variables for script validation
|
||||
"""
|
||||
self.resolved_variables |= REQUIRED_OVERRIDE_VARIABLE_NAMES
|
||||
|
||||
def _add_variables(self, plugin_op: PluginOperation, options: OptionsValidator) -> None:
|
||||
"""
|
||||
Add dummy variables for script validation
|
||||
"""
|
||||
added_variables = options.added_variables(
|
||||
resolved_variables=self.resolved_variables,
|
||||
unresolved_variables=self.unresolved_variables,
|
||||
|
|
@ -175,14 +181,14 @@ class VariableValidation:
|
|||
self.resolved_variables |= resolved_variables
|
||||
self.unresolved_variables -= resolved_variables
|
||||
|
||||
return added_variables
|
||||
|
||||
def ensure_proper_usage(self) -> None:
|
||||
"""
|
||||
Validate variables resolve as plugins are executed, and return
|
||||
a mock script which contains actualized added variables from the plugins
|
||||
"""
|
||||
|
||||
self._add_variables(PluginOperation.DOWNLOADER, options=self.downloader_options)
|
||||
self._add_subscription_override_variables()
|
||||
|
||||
# Metadata variables to be added
|
||||
for plugin_options in PluginMapping.order_options_by(
|
||||
|
|
|
|||
|
|
@ -134,7 +134,9 @@ class UrlDownloaderThumbnailPlugin(SourcePluginExtension):
|
|||
try_convert_download_thumbnail(entry=entry)
|
||||
|
||||
self._download_url_thumbnails(
|
||||
collection_url=self.plugin_options.urls.list[entry.get(v.ytdl_sub_input_url_index, int)],
|
||||
collection_url=self.plugin_options.urls.list[
|
||||
entry.get(v.ytdl_sub_input_url_index, int)
|
||||
],
|
||||
entry=entry,
|
||||
)
|
||||
return entry
|
||||
|
|
@ -224,7 +226,12 @@ class MultiUrlDownloader(SourcePlugin[MultiUrlValidator]):
|
|||
return (
|
||||
self._download_ytdl_options_builder.clone()
|
||||
.add(self.ytdl_option_defaults(), before=True)
|
||||
.add(self.plugin_options.urls.list[url_idx].ytdl_options.dict if url_idx is not None else None, before=True)
|
||||
.add(
|
||||
self.plugin_options.urls.list[url_idx].ytdl_options.dict
|
||||
if url_idx is not None
|
||||
else None,
|
||||
before=True,
|
||||
)
|
||||
.to_dict()
|
||||
)
|
||||
|
||||
|
|
@ -463,10 +470,12 @@ class MultiUrlDownloader(SourcePlugin[MultiUrlValidator]):
|
|||
continue
|
||||
|
||||
for entry in self._download_metadata(url=url, validator=url_validator):
|
||||
entry.initialize_script(self.overrides).add({
|
||||
v.ytdl_sub_input_url: url,
|
||||
v.ytdl_sub_input_url_index: idx,
|
||||
})
|
||||
entry.initialize_script(self.overrides).add(
|
||||
{
|
||||
v.ytdl_sub_input_url: url,
|
||||
v.ytdl_sub_input_url_index: idx,
|
||||
}
|
||||
)
|
||||
|
||||
yield entry
|
||||
|
||||
|
|
|
|||
|
|
@ -9,6 +9,7 @@ from typing import TypeVar
|
|||
|
||||
from ytdl_sub.script.types.array import Array
|
||||
from ytdl_sub.script.types.map import Map
|
||||
from ytdl_sub.script.types.resolvable import Boolean
|
||||
from ytdl_sub.script.types.resolvable import Integer
|
||||
from ytdl_sub.script.types.resolvable import String
|
||||
|
||||
|
|
@ -63,6 +64,13 @@ class Variable(ABC):
|
|||
"""
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class BooleanVariable(Variable):
|
||||
@classmethod
|
||||
def human_readable_type(cls) -> str:
|
||||
return Boolean.__name__
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class StringVariable(Variable):
|
||||
@classmethod
|
||||
|
|
|
|||
|
|
@ -1,6 +1,14 @@
|
|||
from typing import Dict
|
||||
from typing import Set
|
||||
|
||||
from ytdl_sub.entries.script.function_scripts import CUSTOM_FUNCTION_SCRIPTS
|
||||
from ytdl_sub.entries.script.variable_definitions import VARIABLE_SCRIPTS
|
||||
from ytdl_sub.entries.script.variable_types import BooleanVariable
|
||||
from ytdl_sub.entries.script.variable_types import MapVariable
|
||||
from ytdl_sub.entries.script.variable_types import StringVariable
|
||||
from ytdl_sub.entries.script.variable_types import Variable
|
||||
from ytdl_sub.script.functions import Functions
|
||||
from ytdl_sub.script.types.syntax_tree import SyntaxTree
|
||||
from ytdl_sub.script.utils.name_validation import is_valid_name
|
||||
|
||||
# TODO: use this
|
||||
|
|
@ -9,15 +17,15 @@ SUBSCRIPTION_ARRAY = "subscription_array"
|
|||
|
||||
class SubscriptionVariables:
|
||||
@staticmethod
|
||||
def subscription_name() -> str:
|
||||
def subscription_name() -> StringVariable:
|
||||
"""
|
||||
Name of the subscription. For subscriptions types that use a prefix (``~``, ``+``),
|
||||
the prefix and all whitespace afterwards is stripped from the subscription name.
|
||||
"""
|
||||
return "subscription_name"
|
||||
return StringVariable(variable_name="subscription_name", definition="{ %string('') }")
|
||||
|
||||
@staticmethod
|
||||
def subscription_value() -> str:
|
||||
def subscription_value() -> StringVariable:
|
||||
"""
|
||||
For subscriptions in the form of
|
||||
|
||||
|
|
@ -27,10 +35,10 @@ class SubscriptionVariables:
|
|||
|
||||
``subscription_value`` gets set to ``https://...``.
|
||||
"""
|
||||
return "subscription_value"
|
||||
return StringVariable(variable_name="subscription_value", definition="{ %string('') }")
|
||||
|
||||
@staticmethod
|
||||
def subscription_indent_i(index: int) -> str:
|
||||
def subscription_indent_i(index: int) -> StringVariable:
|
||||
"""
|
||||
For subscriptions in the form of
|
||||
|
||||
|
|
@ -43,10 +51,12 @@ class SubscriptionVariables:
|
|||
``subscription_indent_1`` and ``subscription_indent_2`` get set to
|
||||
``Indent Value 1`` and ``Indent Value 2``.
|
||||
"""
|
||||
return f"subscription_indent_{index + 1}"
|
||||
return StringVariable(
|
||||
variable_name=f"subscription_indent_{index + 1}", definition="{ %string('') }"
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def subscription_value_i(index: int) -> str:
|
||||
def subscription_value_i(index: int) -> StringVariable:
|
||||
"""
|
||||
For subscriptions in the form of
|
||||
|
||||
|
|
@ -60,10 +70,12 @@ class SubscriptionVariables:
|
|||
and ``https://url2.com/...``. Note that ``subscription_value_1`` also gets set to
|
||||
``subscription_value``.
|
||||
"""
|
||||
return f"subscription_value_{index + 1}"
|
||||
return StringVariable(
|
||||
variable_name=f"subscription_value_{index + 1}", definition="{ %string('') }"
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def subscription_map() -> str:
|
||||
def subscription_map() -> MapVariable:
|
||||
"""
|
||||
For subscriptions in the form of
|
||||
|
||||
|
|
@ -89,7 +101,17 @@ class SubscriptionVariables:
|
|||
]
|
||||
}
|
||||
"""
|
||||
return "subscription_map"
|
||||
return MapVariable(variable_name="subscription_map", definition="{ {} }")
|
||||
|
||||
@staticmethod
|
||||
def subscription_has_download_archive() -> BooleanVariable:
|
||||
"""
|
||||
Returns True if the subscription has any entries recorded in a download archive. False
|
||||
otherwise.
|
||||
"""
|
||||
return BooleanVariable(
|
||||
variable_name="subscription_has_download_archive", definition="{ %bool(True) }"
|
||||
)
|
||||
|
||||
|
||||
class OverrideHelpers:
|
||||
|
|
@ -124,3 +146,17 @@ class OverrideHelpers:
|
|||
return is_valid_name(name=name[1:])
|
||||
|
||||
return is_valid_name(name=name)
|
||||
|
||||
|
||||
REQUIRED_OVERRIDE_VARIABLES: Set[Variable] = {
|
||||
SubscriptionVariables.subscription_name(),
|
||||
SubscriptionVariables.subscription_has_download_archive(),
|
||||
}
|
||||
|
||||
REQUIRED_OVERRIDE_VARIABLE_DEFINITIONS: Dict[str, str] = {
|
||||
var.variable_name: var.definition for var in REQUIRED_OVERRIDE_VARIABLES
|
||||
}
|
||||
|
||||
REQUIRED_OVERRIDE_VARIABLE_NAMES: Set[str] = {
|
||||
var.variable_name for var in REQUIRED_OVERRIDE_VARIABLES
|
||||
}
|
||||
|
|
|
|||
|
|
@ -454,7 +454,9 @@ class Script:
|
|||
added_variables_to_validate: Set[str] = set()
|
||||
|
||||
functions_to_add = {
|
||||
_function_name(name): definition for name, definition in variables.items() if _is_function(name)
|
||||
_function_name(name): definition
|
||||
for name, definition in variables.items()
|
||||
if _is_function(name)
|
||||
}
|
||||
variables_to_add = {
|
||||
name: definition for name, definition in variables.items() if not _is_function(name)
|
||||
|
|
|
|||
|
|
@ -17,6 +17,7 @@ from ytdl_sub.downloaders.source_plugin import SourcePlugin
|
|||
from ytdl_sub.downloaders.url.downloader import MultiUrlDownloader
|
||||
from ytdl_sub.downloaders.ytdl_options_builder import YTDLOptionsBuilder
|
||||
from ytdl_sub.entries.entry import Entry
|
||||
from ytdl_sub.entries.variables.override_variables import SubscriptionVariables
|
||||
from ytdl_sub.subscriptions.base_subscription import BaseSubscription
|
||||
from ytdl_sub.subscriptions.subscription_ytdl_options import SubscriptionYTDLOptions
|
||||
from ytdl_sub.utils.datetime import to_date_range
|
||||
|
|
@ -325,6 +326,16 @@ class SubscriptionDownload(BaseSubscription, ABC):
|
|||
|
||||
return self._enhanced_download_archive.get_file_handler_transaction_log()
|
||||
|
||||
def _initialize_subscription_overrides(self):
|
||||
required_overrides = {
|
||||
SubscriptionVariables.subscription_name(): self.name,
|
||||
SubscriptionVariables.subscription_has_download_archive(): f"""{{
|
||||
%bool({self._enhanced_download_archive.num_entries > 0})
|
||||
}}""",
|
||||
}
|
||||
assert SubscriptionVariables
|
||||
self.overrides.add(required_overrides)
|
||||
|
||||
def download(self, dry_run: bool = False) -> FileHandlerTransactionLog:
|
||||
"""
|
||||
Performs the subscription download
|
||||
|
|
@ -338,6 +349,7 @@ class SubscriptionDownload(BaseSubscription, ABC):
|
|||
self._exception = None
|
||||
self._enhanced_download_archive.reinitialize(dry_run=dry_run)
|
||||
|
||||
self._initialize_subscription_overrides()
|
||||
plugins = self._initialize_plugins()
|
||||
|
||||
subscription_ytdl_options = SubscriptionYTDLOptions(
|
||||
|
|
|
|||
|
|
@ -32,7 +32,7 @@ class SubscriptionOutput(Validator, ABC):
|
|||
indent overrides to merge with the preset dict's overrides
|
||||
"""
|
||||
return {
|
||||
SubscriptionVariables.subscription_indent_i(i): self._indent_overrides[i]
|
||||
SubscriptionVariables.subscription_indent_i(i).variable_name: self._indent_overrides[i]
|
||||
for i in range(len(self._indent_overrides))
|
||||
}
|
||||
|
||||
|
|
@ -143,7 +143,9 @@ class SubscriptionValueValidator(SubscriptionLeafValidator, StringValidator):
|
|||
presets=presets,
|
||||
indent_overrides=indent_overrides,
|
||||
)
|
||||
self._overrides_to_add[SubscriptionVariables.subscription_value()] = self.value
|
||||
self._overrides_to_add[
|
||||
SubscriptionVariables.subscription_value().variable_name
|
||||
] = self.value
|
||||
|
||||
|
||||
class SubscriptionListValuesValidator(SubscriptionLeafValidator, StringListValidator):
|
||||
|
|
@ -169,11 +171,11 @@ class SubscriptionListValuesValidator(SubscriptionLeafValidator, StringListValid
|
|||
# Write the first list value into subscription_value as well
|
||||
if idx == 0:
|
||||
self._overrides_to_add[
|
||||
SubscriptionVariables.subscription_value()
|
||||
SubscriptionVariables.subscription_value().variable_name
|
||||
] = list_value.value
|
||||
|
||||
self._overrides_to_add[
|
||||
SubscriptionVariables.subscription_value_i(index=idx)
|
||||
SubscriptionVariables.subscription_value_i(index=idx).variable_name
|
||||
] = list_value.value
|
||||
|
||||
|
||||
|
|
@ -217,9 +219,9 @@ class SubscriptionMapValidator(SubscriptionLeafValidator, LiteralDictValidator):
|
|||
presets=presets,
|
||||
indent_overrides=indent_overrides,
|
||||
)
|
||||
self._overrides_to_add[SubscriptionVariables.subscription_map()] = ScriptUtils.to_script(
|
||||
self.dict
|
||||
)
|
||||
self._overrides_to_add[
|
||||
SubscriptionVariables.subscription_map().variable_name
|
||||
] = ScriptUtils.to_script(self.dict)
|
||||
|
||||
|
||||
class SubscriptionValidator(SubscriptionOutput):
|
||||
|
|
|
|||
|
|
@ -9,13 +9,16 @@ from ytdl_sub.entries.script.function_scripts import CUSTOM_FUNCTION_SCRIPTS
|
|||
from ytdl_sub.entries.script.variable_definitions import UNRESOLVED_VARIABLES
|
||||
from ytdl_sub.entries.script.variable_definitions import VARIABLE_SCRIPTS
|
||||
from ytdl_sub.entries.script.variable_types import Variable
|
||||
from ytdl_sub.entries.variables.override_variables import REQUIRED_OVERRIDE_VARIABLE_DEFINITIONS
|
||||
from ytdl_sub.script.script import Script
|
||||
from ytdl_sub.script.utils.exceptions import RuntimeException
|
||||
from ytdl_sub.utils.exceptions import StringFormattingException
|
||||
from ytdl_sub.utils.script import ScriptUtils
|
||||
|
||||
BASE_SCRIPT: Script = Script(
|
||||
dict(ScriptUtils.add_sanitized_variables(VARIABLE_SCRIPTS), **CUSTOM_FUNCTION_SCRIPTS)
|
||||
ScriptUtils.add_sanitized_variables(VARIABLE_SCRIPTS)
|
||||
| ScriptUtils.add_sanitized_variables(REQUIRED_OVERRIDE_VARIABLE_DEFINITIONS)
|
||||
| CUSTOM_FUNCTION_SCRIPTS
|
||||
)
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -56,7 +56,7 @@ def playlist_bilateral_dict(output_directory):
|
|||
return {
|
||||
"preset": [
|
||||
"Jellyfin TV Show by Date",
|
||||
],
|
||||
],
|
||||
"format": "worst[ext=mp4]",
|
||||
"overrides": {
|
||||
"url": "https://www.youtube.com/playlist?list=PLd4Q7G88JqoekF0b30NYQcOTnTiIe9Ali",
|
||||
|
|
@ -238,14 +238,12 @@ class TestPlaylist:
|
|||
assert subscriptions[0].transaction_log.is_empty
|
||||
|
||||
def test_tv_show_downloads_bilateral(
|
||||
self,
|
||||
playlist_bilateral_dict: Dict,
|
||||
output_directory: str,
|
||||
default_config: ConfigFile,
|
||||
self,
|
||||
playlist_bilateral_dict: Dict,
|
||||
output_directory: str,
|
||||
default_config: ConfigFile,
|
||||
):
|
||||
playlist_bilateral_dict['filter_include'] = [
|
||||
"{ %contains(title, 'Feb.1') }"
|
||||
]
|
||||
playlist_bilateral_dict["filter_include"] = ["{ %contains(title, 'Feb.1') }"]
|
||||
playlist_subscription = Subscription.from_dict(
|
||||
config=default_config,
|
||||
preset_name="bilateral_test",
|
||||
|
|
@ -261,7 +259,7 @@ class TestPlaylist:
|
|||
|
||||
# Now that one vid is downloaded, attempt to download all and see if bilateral
|
||||
# logic kicks in
|
||||
del playlist_bilateral_dict['filter_include']
|
||||
del playlist_bilateral_dict["filter_include"]
|
||||
playlist_subscription = Subscription.from_dict(
|
||||
config=default_config,
|
||||
preset_name="bilateral_test",
|
||||
|
|
@ -272,4 +270,4 @@ class TestPlaylist:
|
|||
output_directory=output_directory,
|
||||
transaction_log=transaction_log,
|
||||
transaction_log_summary_file_name="youtube/test_playlist_bilateral_p2.txt",
|
||||
)
|
||||
)
|
||||
|
|
|
|||
Loading…
Reference in a new issue