970c74ba45
Subscription file syntax is designed to minimize boiler-plate when authoring new subscriptions. You can now unpack any subscription using the ``inspect`` sub-command to see its boiler-plate *preset format*. ``` ytdl-sub inspect --config /path/to/config.yaml --match "BBC News" /path/to/subscriptions.yaml ``` This can be utilized for numerous purposes including: * Ensuring your custom preset is getting applied correctly. * Figuring out which variables set things like file names, metadata, etc. * Understanding how subscription syntax translates to preset representation. The default ``--level`` of inspect will fill in defined variables. Using ``--level original`` will present the subscription's raw layout with no fill.
218 lines
8.5 KiB
Python
218 lines
8.5 KiB
Python
from typing import Dict, List, Optional, Set
|
|
|
|
from ytdl_sub.config.overrides import Overrides
|
|
from ytdl_sub.config.plugin.plugin_mapping import PluginMapping
|
|
from ytdl_sub.config.plugin.plugin_operation import PluginOperation
|
|
from ytdl_sub.config.plugin.preset_plugins import PresetPlugins
|
|
from ytdl_sub.config.preset_options import OutputOptions
|
|
from ytdl_sub.config.validators.options import OptionsValidator
|
|
from ytdl_sub.downloaders.url.validators import MultiUrlValidator
|
|
from ytdl_sub.entries.script.variable_definitions import UNRESOLVED_VARIABLES, VARIABLES
|
|
from ytdl_sub.script.script import Script
|
|
from ytdl_sub.script.utils.name_validation import is_function
|
|
from ytdl_sub.utils.script import ScriptUtils
|
|
from ytdl_sub.validators.string_formatter_validators import validate_formatters
|
|
|
|
|
|
class ResolutionLevel:
|
|
ORIGINAL = 0
|
|
FILL = 1
|
|
RESOLVE = 2
|
|
INTERNAL = 3
|
|
|
|
@classmethod
|
|
def name_of(cls, resolution_level: int) -> str:
|
|
"""
|
|
Name of the resolution level.
|
|
"""
|
|
if resolution_level == cls.ORIGINAL:
|
|
return "original"
|
|
if resolution_level == cls.FILL:
|
|
return "fill"
|
|
if resolution_level == cls.RESOLVE:
|
|
return "resolve"
|
|
if resolution_level == cls.INTERNAL:
|
|
return "internal"
|
|
raise ValueError("Invalid resolution level")
|
|
|
|
@classmethod
|
|
def level_number(cls, resolution_arg: str) -> int:
|
|
"""
|
|
Numeric resolution level
|
|
"""
|
|
if resolution_arg in ("0", "original"):
|
|
return 0
|
|
if resolution_arg in ("1", "fill"):
|
|
return 1
|
|
if resolution_arg in ("2", "resolve"):
|
|
return 2
|
|
if resolution_arg in ("3", "internal"):
|
|
return 3
|
|
raise ValueError("Invalid resolution level")
|
|
|
|
@classmethod
|
|
def all(cls) -> List[int]:
|
|
"""
|
|
All possible resolution levels.
|
|
"""
|
|
return [cls.ORIGINAL, cls.FILL, cls.RESOLVE, cls.INTERNAL]
|
|
|
|
|
|
class VariableValidation:
|
|
def _get_resolve_partial_filter(self) -> Set[str]:
|
|
# Exclude sanitized variables from partial validation. This lessens the work
|
|
# and prevents double-evaluation, which can lead to bad behavior like double-prints.
|
|
return {
|
|
name
|
|
for name in self.script.variable_names
|
|
if name not in self.unresolved_variables and not name.endswith("_sanitized")
|
|
}
|
|
|
|
def _apply_resolution_level(self, mocks: Optional[Dict[str, str]]) -> None:
|
|
if self._resolution_level == ResolutionLevel.FILL:
|
|
self.unresolved_variables |= VARIABLES.variable_names(include_sanitized=True)
|
|
# Only partial resolve definitions that are already resolved
|
|
self.unresolved_variables |= {
|
|
name
|
|
for name in self.overrides.keys
|
|
if not is_function(name) and not self.script.definition_of(name).maybe_resolvable
|
|
}
|
|
elif self._resolution_level == ResolutionLevel.RESOLVE:
|
|
# Partial resolve everything, but not including internal variables
|
|
self.unresolved_variables |= VARIABLES.variable_names(include_sanitized=True)
|
|
elif self._resolution_level == ResolutionLevel.INTERNAL:
|
|
# Partial resolve everything including internal variables
|
|
pass
|
|
else:
|
|
raise ValueError("Invalid resolution level for validation")
|
|
|
|
if mocks is not None:
|
|
for mock_name in mocks.keys():
|
|
if mock_name in self.unresolved_variables:
|
|
self.unresolved_variables.remove(mock_name)
|
|
|
|
self.script.add(
|
|
variables=mocks,
|
|
unresolvable=self.unresolved_variables,
|
|
)
|
|
|
|
self.script = self.script.resolve_partial(
|
|
unresolvable=self.unresolved_variables,
|
|
output_filter=self._get_resolve_partial_filter(),
|
|
)
|
|
|
|
def __init__(
|
|
self,
|
|
overrides: Overrides,
|
|
downloader_options: MultiUrlValidator,
|
|
output_options: OutputOptions,
|
|
plugins: PresetPlugins,
|
|
resolution_level: int = ResolutionLevel.RESOLVE,
|
|
mocks: Optional[Dict[str, str]] = None,
|
|
):
|
|
self.overrides = overrides
|
|
self.downloader_options = downloader_options
|
|
self.output_options = output_options
|
|
self.plugins = plugins
|
|
|
|
self.script: Script = self.overrides.script
|
|
self.unresolved_variables = (
|
|
self.plugins.get_all_variables(
|
|
additional_options=[self.output_options, self.downloader_options]
|
|
)
|
|
| UNRESOLVED_VARIABLES
|
|
)
|
|
self.unresolved_runtime_variables = self.plugins.get_all_variables(
|
|
additional_options=[self.output_options, self.downloader_options]
|
|
)
|
|
self._resolution_level = resolution_level
|
|
self._apply_resolution_level(mocks=mocks)
|
|
|
|
def _add_runtime_variables(self, plugin_op: PluginOperation, options: OptionsValidator) -> None:
|
|
"""
|
|
Add dummy variables for script validation
|
|
"""
|
|
added_variables = options.added_variables(
|
|
unresolved_variables=self.unresolved_runtime_variables,
|
|
).get(plugin_op, set())
|
|
modified_variables = options.modified_variables().get(plugin_op, set())
|
|
|
|
self.unresolved_runtime_variables -= added_variables | modified_variables
|
|
|
|
def _output_override_variables(self) -> Dict:
|
|
output = {}
|
|
for name in self.overrides.keys:
|
|
value = self.script.definition_of(name)
|
|
if name in self.script.function_names:
|
|
# Keep custom functions as-is
|
|
output[name] = self.overrides.dict_with_format_strings[name]
|
|
elif resolved := value.maybe_resolvable:
|
|
output[name] = resolved.native
|
|
else:
|
|
output[name] = ScriptUtils.to_native_script(value)
|
|
|
|
return output
|
|
|
|
def ensure_proper_usage(self, partial_resolve_formatters: bool = False) -> Dict:
|
|
"""
|
|
Validate variables resolve as plugins are executed, and return
|
|
a mock script which contains actualized added variables from the plugins
|
|
"""
|
|
resolved_subscription: Dict = {}
|
|
|
|
self._add_runtime_variables(PluginOperation.DOWNLOADER, options=self.downloader_options)
|
|
|
|
# Always add output options first
|
|
self._add_runtime_variables(
|
|
PluginOperation.MODIFY_ENTRY_METADATA, options=self.output_options
|
|
)
|
|
|
|
# Metadata variables to be added
|
|
for plugin_options in PluginMapping.order_options_by(
|
|
self.plugins.zipped(), PluginOperation.MODIFY_ENTRY_METADATA
|
|
):
|
|
self._add_runtime_variables(
|
|
PluginOperation.MODIFY_ENTRY_METADATA, options=plugin_options
|
|
)
|
|
|
|
for plugin_options in PluginMapping.order_options_by(
|
|
self.plugins.zipped(), PluginOperation.MODIFY_ENTRY
|
|
):
|
|
self._add_runtime_variables(PluginOperation.MODIFY_ENTRY, options=plugin_options)
|
|
|
|
# Validate that any formatter in the plugin options can resolve
|
|
resolved_subscription |= validate_formatters(
|
|
script=self.script,
|
|
unresolved_variables=self.unresolved_variables,
|
|
unresolved_runtime_variables=self.unresolved_runtime_variables,
|
|
validator=plugin_options,
|
|
partial_resolve_formatters=partial_resolve_formatters,
|
|
)
|
|
|
|
resolved_subscription |= validate_formatters(
|
|
script=self.script,
|
|
unresolved_variables=self.unresolved_variables,
|
|
unresolved_runtime_variables=self.unresolved_runtime_variables,
|
|
validator=self.output_options,
|
|
partial_resolve_formatters=partial_resolve_formatters,
|
|
)
|
|
|
|
# TODO: make this a function
|
|
raw_download_output = validate_formatters(
|
|
script=self.script,
|
|
unresolved_variables=self.unresolved_variables,
|
|
unresolved_runtime_variables=self.unresolved_runtime_variables,
|
|
validator=self.downloader_options.urls,
|
|
partial_resolve_formatters=partial_resolve_formatters,
|
|
)
|
|
resolved_subscription["download"] = []
|
|
for url_output in raw_download_output["download"]:
|
|
if isinstance(url_output["url"], list):
|
|
url_output["url"] = [url for url in url_output["url"] if bool(url)]
|
|
|
|
if url_output["url"]:
|
|
resolved_subscription["download"].append(url_output)
|
|
|
|
resolved_subscription["overrides"] = self._output_override_variables()
|
|
|
|
return resolved_subscription
|