danvics/ytdl-sub
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

[BACKEND][HUGE] Function Support in variable syntax (#838)

Browse source 
A complete gutting of the internals of ytdl-sub to support functions in our variable syntax, in addition to being able to access a yt-dlp entry's .info.json fields using functions. Functionally, ytdl-sub should still look and behave the same from a user-perspective.

With so many lines of code changed (+8927, -2708), no doubt there will be new issues. Please make a GH issue or reach out on Discord if your config/subscriptions break in any way/shape/form.

Details on how to use function support will come soon in the form of proper documentation in our readthedocs.
This commit is contained in:
Jesse Bannon 2023-12-18 16:08:15 -08:00 committed by GitHub
parent 4d9a30779c
commit e92b1cd12a
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
131 changed files with 9015 additions and 2722 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
Makefile
View file

@ -17,12 +17,10 @@ lint:
@-isort .
@-black .
@-pylint src/
@-pydocstyle src/*
check_lint:
isort . --check-only --diff \
&& black . --check \
&& pylint src/ \
&& pydocstyle src/*
&& pylint src/
wheel: clean
$(shell echo "__pypi_version__ = \"$(PYPI_VERSION)\"\n__local_version__ = \"$(LOCAL_VERSION)\"" > src/ytdl_sub/__init__.py)
cat src/ytdl_sub/__init__.py

4
docs/config.rst
View file

@ -95,7 +95,7 @@ ytdl_options
overrides
"""""""""
.. autoclass:: ytdl_sub.config.preset_options.Overrides()
.. autoclass:: ytdl_sub.config.overrides.Overrides()
.. _parent preset:
@ -423,7 +423,7 @@ Traditional subscriptions that can override presets will still work when using `
Source Variables
----------------
.. autoclass:: ytdl_sub.entries.variables.entry_variables.EntryVariables
.. autoclass:: ytdl_sub.entries.script.variable_definitions.VariableDefinitions()
:members:
:inherited-members:
:undoc-members:

6
pyproject.toml
View file

@ -5,6 +5,7 @@ force_single_line = true
[tool.black]
line_length = 100
target-version = ["py310"]
[tool.pylint.MASTER]
disable = [
@ -43,3 +44,8 @@ ignore = [
include = [
"src/*"
]
[tool.coverage.report]
exclude_also = [
"raise UNREACHABLE.*",
]

1
setup.cfg
View file

@ -49,7 +49,6 @@ lint =
black==22.3.0
isort==5.10.1
pylint==2.13.5
pydocstyle[toml]==6.1.1
docs =
sphinx==4.5.0
sphinx-rtd-theme==1.0.0

2
src/ytdl_sub/cli/entrypoint.py
View file

@ -221,6 +221,7 @@ def main() -> List[Subscription]:
"full backup before usage. You have been warned!",
)
logger.info("Validating subscriptions...")
subscriptions = _download_subscriptions_from_yaml_files(
config=config,
subscription_paths=args.subscription_paths,
@ -230,6 +231,7 @@ def main() -> List[Subscription]:
# One-off download
elif args.subparser == "dl":
logger.info("Validating presets...")
subscriptions.append(
_download_subscription_from_cli(
config=config, dry_run=args.dry_run, extra_args=extra_args

4
src/ytdl_sub/config/config_file.py
View file

@ -6,8 +6,8 @@ from ytdl_sub.config.config_validator import ConfigValidator
from ytdl_sub.config.preset import Preset
from ytdl_sub.utils.exceptions import FileNotFoundException
from ytdl_sub.utils.ffmpeg import FFMPEG
from ytdl_sub.utils.file_path import FilePathTruncater
from ytdl_sub.utils.yaml import load_yaml
from ytdl_sub.validators.file_path_validators import FilePathValidatorMixin
class ConfigFile(ConfigValidator):
@ -36,7 +36,7 @@ class ConfigFile(ConfigValidator):
ffprobe_path=self.config_options.ffprobe_path,
)
FilePathValidatorMixin.set_max_file_name_bytes(
FilePathTruncater.set_max_file_name_bytes(
max_file_name_bytes=self.config_options.file_name_max_bytes
)

176
src/ytdl_sub/config/overrides.py Normal file
View file

@ -0,0 +1,176 @@
from typing import Any
from typing import Dict
from typing import Optional
from typing import Set
import mergedeep
from ytdl_sub.entries.entry import Entry
from ytdl_sub.entries.script.variable_definitions import VARIABLES
from ytdl_sub.entries.variables.override_variables import SUBSCRIPTION_NAME
from ytdl_sub.entries.variables.override_variables import OverrideVariables
from ytdl_sub.script.parser import parse
from ytdl_sub.script.script import Script
from ytdl_sub.utils.exceptions import ValidationException
from ytdl_sub.utils.script import ScriptUtils
from ytdl_sub.utils.scriptable import Scriptable
from ytdl_sub.validators.string_formatter_validators import DictFormatterValidator
from ytdl_sub.validators.string_formatter_validators import StringFormatterValidator
class Overrides(DictFormatterValidator, Scriptable):
"""
Optional. This section allows you to define variables that can be used in any string formatter.
For example, if you want your file and thumbnail files to match without copy-pasting a large
format string, you can define something like:
.. code-block:: yaml
presets:
my_example_preset:
overrides:
output_directory: "/path/to/media"
custom_file_name: "{upload_date_standardized}.{title_sanitized}"
# Then use the override variables in the output options
output_options:
output_directory: "{output_directory}"
file_name: "{custom_file_name}.{ext}"
thumbnail_name: "{custom_file_name}.{thumbnail_ext}"
Override variables can contain explicit values and other variables, including both override
and source variables.
In addition, any override variable defined will automatically create a ``sanitized`` variable
for use. In the example above, ``output_directory_sanitized`` will exist and perform
sanitization on the value when used.
"""
@classmethod
def partial_validate(cls, name: str, value: Any) -> None:
dict_formatter = DictFormatterValidator(name=name, value=value)
_ = [parse(format_string) for format_string in dict_formatter.dict_with_format_strings]
def __init__(self, name, value):
DictFormatterValidator.__init__(self, name, value)
Scriptable.__init__(self)
for key in self._keys:
self.ensure_variable_name_valid(key)
self.unresolvable.add(VARIABLES.entry_metadata.variable_name)
def ensure_added_plugin_variable_valid(self, added_variable: str) -> bool:
"""
Returns False if the variable exists as a non-override.
Raises
------
ValidationException
If the variable is already added as an override variable.
"""
try:
self.ensure_variable_name_valid(added_variable)
except ValidationException:
return False
if added_variable in self.keys:
raise self._validation_exception(
f"Override variable with name {added_variable} cannot be used since it is"
" added by a plugin."
)
return True
def ensure_variable_name_valid(self, name: str) -> None:
"""
Ensures the variable name does not collide with any entry variables or built-in functions.
"""
if OverrideVariables.is_entry_variable_name(name):
raise self._validation_exception(
f"Override variable with name {name} cannot be used since it is a"
" built-in ytdl-sub entry variable name."
)
if OverrideVariables.is_function_name(name):
raise self._validation_exception(
f"Override function definition with name {name} cannot be used since it is"
" a built-in ytdl-sub function name."
)
def initial_variables(
self, unresolved_variables: Optional[Dict[str, str]] = None
) -> Dict[str, str]:
"""
Returns
-------
Variables and format strings for all Override variables + additional variables (Optional)
"""
initial_variables: Dict[str, str] = {}
mergedeep.merge(
initial_variables,
self.dict_with_format_strings,
unresolved_variables if unresolved_variables else {},
{SUBSCRIPTION_NAME: self.subscription_name},
)
return ScriptUtils.add_sanitized_variables(initial_variables)
def initialize_script(self, unresolved_variables: Set[str]) -> "Overrides":
"""
Initialize the override script with override variables + any unresolved variables
"""
self.script.add(
self.initial_variables(
unresolved_variables={
var_name: f"{{%throw('Plugin variable {var_name} has not been created yet')}}"
for var_name in unresolved_variables
}
)
)
self.unresolvable.update(unresolved_variables)
self.update_script()
return self
@property
def subscription_name(self) -> str:
"""
Returns
-------
Name of the subscription
"""
return self._root_name
def apply_formatter(
self,
formatter: StringFormatterValidator,
entry: Optional[Entry] = None,
function_overrides: Dict[str, str] = None,
) -> str:
"""
Parameters
----------
formatter
Formatter to apply
entry
Optional. Entry to add source variables to the formatter
function_overrides
Optional. Explicit values to override the overrides themselves and source variables
Returns
-------
The format_string after .format has been called
"""
script: Script = self.script
unresolvable: Set[str] = self.unresolvable
if entry:
script = entry.script
unresolvable = entry.unresolvable
return formatter.post_process(
str(
script.resolve_once(
dict({"tmp_var": formatter.format_string}, **(function_overrides or {})),
unresolvable=unresolvable,
)["tmp_var"]
)
)

0
src/ytdl_sub/config/plugin/__init__.py Normal file
View file

36
src/ytdl_sub/config/plugin.py → src/ytdl_sub/config/plugin/plugin.py
View file

@ -7,44 +7,13 @@ from typing import Optional
from typing import Tuple
from typing import Type
from ytdl_sub.config.preset_options import Overrides
from ytdl_sub.config.preset_options import TOptionsValidator
from ytdl_sub.config.overrides import Overrides
from ytdl_sub.config.validators.options import TOptionsValidator
from ytdl_sub.entries.entry import Entry
from ytdl_sub.utils.file_handler import FileMetadata
from ytdl_sub.ytdl_additions.enhanced_download_archive import DownloadArchiver
from ytdl_sub.ytdl_additions.enhanced_download_archive import EnhancedDownloadArchive
class PluginPriority:
"""
Defines priority for plugins, 0 is highest priority
"""
# If modify_entry priority is >= to this value, run after split
MODIFY_ENTRY_AFTER_SPLIT = 10
# if post_process is >= to this value, run after file_convert
POST_PROCESS_AFTER_FILE_CONVERT = 10
MODIFY_ENTRY_FIRST = 0
def __init__(
self, modify_entry_metadata: int = 5, modify_entry: int = 5, post_process: int = 5
):
self.modify_entry_metadata = modify_entry_metadata
self.modify_entry = modify_entry
self.post_process = post_process
@property
def modify_entry_after_split(self) -> bool:
"""
Returns
-------
True if the plugin should modify an entry after a potential split. False otherwise.
"""
return self.modify_entry >= PluginPriority.MODIFY_ENTRY_AFTER_SPLIT
# pylint: disable=no-self-use,unused-argument
@ -53,7 +22,6 @@ class BasePlugin(DownloadArchiver, Generic[TOptionsValidator], ABC):
Shared code amongst all SourcePlugins (downloaders) and Plugins (post-download modification)
"""
priority: PluginPriority = PluginPriority()
plugin_options_type: Type[TOptionsValidator]
def __init__(

200
src/ytdl_sub/config/plugin/plugin_mapping.py Normal file
View file

@ -0,0 +1,200 @@
from typing import Dict
from typing import List
from typing import Optional
from typing import Tuple
from typing import Type
from ytdl_sub.config.plugin.plugin import Plugin
from ytdl_sub.config.plugin.plugin import SplitPlugin
from ytdl_sub.config.plugin.plugin_operation import PluginOperation
from ytdl_sub.config.validators.options import OptionsValidator
from ytdl_sub.downloaders.url.downloader import UrlDownloaderCollectionVariablePlugin
from ytdl_sub.downloaders.url.downloader import UrlDownloaderThumbnailPlugin
from ytdl_sub.plugins.audio_extract import AudioExtractPlugin
from ytdl_sub.plugins.chapters import ChaptersPlugin
from ytdl_sub.plugins.date_range import DateRangePlugin
from ytdl_sub.plugins.embed_thumbnail import EmbedThumbnailPlugin
from ytdl_sub.plugins.file_convert import FileConvertPlugin
from ytdl_sub.plugins.format import FormatPlugin
from ytdl_sub.plugins.internal.view import ViewPlugin
from ytdl_sub.plugins.match_filters import MatchFiltersPlugin
from ytdl_sub.plugins.music_tags import MusicTagsPlugin
from ytdl_sub.plugins.nfo_tags import NfoTagsPlugin
from ytdl_sub.plugins.output_directory_nfo_tags import OutputDirectoryNfoTagsPlugin
from ytdl_sub.plugins.regex import RegexPlugin
from ytdl_sub.plugins.split_by_chapters import SplitByChaptersPlugin
from ytdl_sub.plugins.subtitles import SubtitlesPlugin
from ytdl_sub.plugins.throttle_protection import ThrottleProtectionPlugin
from ytdl_sub.plugins.video_tags import VideoTagsPlugin
class PluginMapping:
"""
Maps plugins defined in the preset to its respective plugin class
"""
_MAPPING: Dict[str, Type[Plugin]] = {
"_view": ViewPlugin,
"audio_extract": AudioExtractPlugin,
"date_range": DateRangePlugin,
"embed_thumbnail": EmbedThumbnailPlugin,
"file_convert": FileConvertPlugin,
"format": FormatPlugin,
"match_filters": MatchFiltersPlugin,
"music_tags": MusicTagsPlugin,
"video_tags": VideoTagsPlugin,
"nfo_tags": NfoTagsPlugin,
"output_directory_nfo_tags": OutputDirectoryNfoTagsPlugin,
"regex": RegexPlugin,
"subtitles": SubtitlesPlugin,
"chapters": ChaptersPlugin,
"split_by_chapters": SplitByChaptersPlugin,
"throttle_protection": ThrottleProtectionPlugin,
}
# All other plugins are added after the defined ordered ones
_ORDER_MODIFY_ENTRY_METADATA: List[Type[Plugin]] = [
ThrottleProtectionPlugin,
UrlDownloaderCollectionVariablePlugin,
SubtitlesPlugin,
# add all others
]
_ORDER_MODIFY_ENTRY: List[Type[Plugin]] = [
UrlDownloaderThumbnailPlugin,
AudioExtractPlugin,
FileConvertPlugin,
ChaptersPlugin,
SplitByChaptersPlugin,
RegexPlugin,
# add all others
]
_ORDER_POST_PROCESS: List[Type[Plugin]] = [
AudioExtractPlugin,
FileConvertPlugin,
ChaptersPlugin,
SubtitlesPlugin,
MusicTagsPlugin,
VideoTagsPlugin,
NfoTagsPlugin,
EmbedThumbnailPlugin,
]
@classmethod
def _order_by(
cls, plugin_types: List[Type[Plugin]], operation: PluginOperation
) -> List[Type[Plugin]]:
if operation == PluginOperation.MODIFY_ENTRY_METADATA:
ordering = cls._ORDER_MODIFY_ENTRY_METADATA
elif operation == PluginOperation.MODIFY_ENTRY:
ordering = cls._ORDER_MODIFY_ENTRY
elif operation == PluginOperation.POST_PROCESS:
ordering = cls._ORDER_POST_PROCESS
else:
raise ValueError("PluginOperation does not support ordering")
ordered_plugin_operations: List[Type[Plugin]] = []
for pl_type in reversed(ordering):
for plugin_type in plugin_types:
if plugin_type == pl_type:
ordered_plugin_operations.insert(0, plugin_type)
else:
ordered_plugin_operations.append(plugin_type)
return ordered_plugin_operations
@classmethod
def order_options_by(
cls, zipped: List[Tuple[Type[Plugin], OptionsValidator]], operation: PluginOperation
) -> List[OptionsValidator]:
"""
Returns
-------
Ordered plugin options with respect to the PluginOperation.
"""
ordered_types: List[Type[Plugin]] = cls._order_by(
plugin_types=[val[0] for val in zipped], operation=operation
)
ordered_options: List[OptionsValidator] = []
for plugin_type, plugin_options in zipped:
sorted_idx = ordered_types.index(plugin_type)
ordered_options.insert(sorted_idx, plugin_options)
return ordered_options
@classmethod
def _is_modified_after_split(cls, plugin: Plugin) -> bool:
if type(plugin) not in cls._ORDER_MODIFY_ENTRY:
return True
return cls._ORDER_MODIFY_ENTRY.index(type(plugin)) > cls._ORDER_MODIFY_ENTRY.index(
SplitByChaptersPlugin
)
@classmethod
def order_plugins_by(
cls, plugins: List[Plugin], operation: PluginOperation, before_split: Optional[bool] = None
) -> List[Plugin]:
"""
Returns
-------
Ordered plugins with respect to the PluginOperation. Optionally only return plugins
before/after a split plugin.
"""
ordered_types: List[Type[Plugin]] = cls._order_by(
plugin_types=[type(plugin) for plugin in plugins], operation=operation
)
ordered_plugins: List[Plugin] = []
for plugin in plugins:
sorted_idx = ordered_types.index(type(plugin))
ordered_plugins.insert(sorted_idx, plugin)
if before_split is None:
return ordered_plugins
# Remove the split plugin if differentiating
ordered_plugins = [
plugin for plugin in ordered_plugins if not isinstance(plugin, SplitPlugin)
]
if before_split:
return [
plugin for plugin in ordered_plugins if not cls._is_modified_after_split(plugin)
]
# before_split is False
return [plugin for plugin in ordered_plugins if cls._is_modified_after_split(plugin)]
@classmethod
def plugins(cls) -> List[str]:
"""
Returns
-------
Available download sources
"""
return sorted(list(cls._MAPPING.keys()))
@classmethod
def get(cls, plugin: str) -> Type[Plugin]:
"""
Parameters
----------
plugin
Name of the plugin
Returns
-------
The plugin class
Raises
------
ValueError
Raised if the plugin does not exist
"""
if plugin not in cls.plugins():
raise ValueError(
f"Tried to use plugin '{plugin}' that does not exist. Available plugins: "
f"{', '.join(cls.plugins())}"
)
return cls._MAPPING[plugin]

9
src/ytdl_sub/config/plugin/plugin_operation.py Normal file
View file

@ -0,0 +1,9 @@
from enum import Enum
class PluginOperation(Enum):
ANY = -2
DOWNLOADER = -1
MODIFY_ENTRY_METADATA = 0
MODIFY_ENTRY = 1
POST_PROCESS = 2

46
src/ytdl_sub/config/plugin/preset_plugins.py Normal file
View file

@ -0,0 +1,46 @@
from typing import List
from typing import Optional
from typing import Tuple
from typing import Type
from ytdl_sub.config.plugin.plugin import Plugin
from ytdl_sub.config.validators.options import OptionsValidator
from ytdl_sub.config.validators.options import TOptionsValidator
class PresetPlugins:
def __init__(self):
self.plugin_types: List[Type[Plugin]] = []
self.plugin_options: List[OptionsValidator] = []
def add(self, plugin_type: Type[Plugin], plugin_options: OptionsValidator) -> "PresetPlugins":
"""
Add a pair of plugin type and options to the list
"""
self.plugin_types.append(plugin_type)
self.plugin_options.append(plugin_options)
return self
def zipped(self) -> List[Tuple[Type[Plugin], OptionsValidator]]:
"""
Returns
-------
Plugin and PluginOptions zipped
"""
return list(zip(self.plugin_types, self.plugin_options))
def get(self, plugin_type: Type[TOptionsValidator]) -> Optional[TOptionsValidator]:
"""
Parameters
----------
plugin_type
Fetch the plugin options for this type
Returns
-------
Options of this plugin if they exit. Otherwise, return None.
"""
plugin_option_types = [type(plugin_options) for plugin_options in self.plugin_options]
if plugin_type in plugin_option_types:
return self.plugin_options[plugin_option_types.index(plugin_type)]
return None

79
src/ytdl_sub/config/plugin_mapping.py
View file

@ -1,79 +0,0 @@
from typing import Dict
from typing import List
from typing import Type
from ytdl_sub.config.plugin import Plugin
from ytdl_sub.plugins.audio_extract import AudioExtractPlugin
from ytdl_sub.plugins.chapters import ChaptersPlugin
from ytdl_sub.plugins.date_range import DateRangePlugin
from ytdl_sub.plugins.embed_thumbnail import EmbedThumbnailPlugin
from ytdl_sub.plugins.file_convert import FileConvertPlugin
from ytdl_sub.plugins.format import FormatPlugin
from ytdl_sub.plugins.internal.view import ViewPlugin
from ytdl_sub.plugins.match_filters import MatchFiltersPlugin
from ytdl_sub.plugins.music_tags import MusicTagsPlugin
from ytdl_sub.plugins.nfo_tags import NfoTagsPlugin
from ytdl_sub.plugins.output_directory_nfo_tags import OutputDirectoryNfoTagsPlugin
from ytdl_sub.plugins.regex import RegexPlugin
from ytdl_sub.plugins.split_by_chapters import SplitByChaptersPlugin
from ytdl_sub.plugins.subtitles import SubtitlesPlugin
from ytdl_sub.plugins.throttle_protection import ThrottleProtectionPlugin
from ytdl_sub.plugins.video_tags import VideoTagsPlugin
class PluginMapping:
"""
Maps plugins defined in the preset to its respective plugin class
"""
_MAPPING: Dict[str, Type[Plugin]] = {
"_view": ViewPlugin,
"audio_extract": AudioExtractPlugin,
"date_range": DateRangePlugin,
"embed_thumbnail": EmbedThumbnailPlugin,
"file_convert": FileConvertPlugin,
"format": FormatPlugin,
"match_filters": MatchFiltersPlugin,
"music_tags": MusicTagsPlugin,
"video_tags": VideoTagsPlugin,
"nfo_tags": NfoTagsPlugin,
"output_directory_nfo_tags": OutputDirectoryNfoTagsPlugin,
"regex": RegexPlugin,
"subtitles": SubtitlesPlugin,
"chapters": ChaptersPlugin,
"split_by_chapters": SplitByChaptersPlugin,
"throttle_protection": ThrottleProtectionPlugin,
}
@classmethod
def plugins(cls) -> List[str]:
"""
Returns
-------
Available download sources
"""
return sorted(list(cls._MAPPING.keys()))
@classmethod
def get(cls, plugin: str) -> Type[Plugin]:
"""
Parameters
----------
plugin
Name of the plugin
Returns
-------
The plugin class
Raises
------
ValueError
Raised if the plugin does not exist
"""
if plugin not in cls.plugins():
raise ValueError(
f"Tried to use plugin '{plugin}' that does not exist. Available plugins: "
f"{', '.join(cls.plugins())}"
)
return cls._MAPPING[plugin]

163
src/ytdl_sub/config/preset.py
View file

@ -1,39 +1,25 @@
import copy
from typing import Any
from typing import Dict
from typing import Iterable
from typing import List
from typing import Optional
from typing import Tuple
from typing import Type
from typing import Union
from mergedeep import mergedeep
from ytdl_sub.config.config_validator import ConfigValidator
from ytdl_sub.config.plugin import Plugin
from ytdl_sub.config.plugin_mapping import PluginMapping
from ytdl_sub.config.preset_options import OptionsValidator
from ytdl_sub.config.overrides import Overrides
from ytdl_sub.config.plugin.plugin_mapping import PluginMapping
from ytdl_sub.config.plugin.preset_plugins import PresetPlugins
from ytdl_sub.config.preset_options import OutputOptions
from ytdl_sub.config.preset_options import Overrides
from ytdl_sub.config.preset_options import TOptionsValidator
from ytdl_sub.config.preset_options import YTDLOptions
from ytdl_sub.config.validators.variable_validation import VariableValidation
from ytdl_sub.downloaders.url.validators import MultiUrlValidator
from ytdl_sub.entries.entry import Entry
from ytdl_sub.prebuilt_presets import PREBUILT_PRESET_NAMES
from ytdl_sub.prebuilt_presets import PUBLISHED_PRESET_NAMES
from ytdl_sub.utils.exceptions import ValidationException
from ytdl_sub.utils.logger import Logger
from ytdl_sub.utils.yaml import dump_yaml
from ytdl_sub.validators.strict_dict_validator import StrictDictValidator
from ytdl_sub.validators.string_formatter_validators import DictFormatterValidator
from ytdl_sub.validators.string_formatter_validators import OverridesDictFormatterValidator
from ytdl_sub.validators.string_formatter_validators import OverridesStringFormatterValidator
from ytdl_sub.validators.string_formatter_validators import StringFormatterValidator
from ytdl_sub.validators.validators import DictValidator
from ytdl_sub.validators.validators import ListValidator
from ytdl_sub.validators.validators import StringListValidator
from ytdl_sub.validators.validators import Validator
from ytdl_sub.validators.validators import validation_exception
PRESET_KEYS = {
@ -61,44 +47,6 @@ def _parent_preset_error_message(
)
class PresetPlugins:
def __init__(self):
self.plugin_types: List[Type[Plugin]] = []
self.plugin_options: List[OptionsValidator] = []
def add(self, plugin_type: Type[Plugin], plugin_options: OptionsValidator) -> "PresetPlugins":
"""
Add a pair of plugin type and options to the list
"""
self.plugin_types.append(plugin_type)
self.plugin_options.append(plugin_options)
return self
def zipped(self) -> Iterable[Tuple[Type[Plugin], OptionsValidator]]:
"""
Returns
-------
Plugin and PluginOptions zipped
"""
return zip(self.plugin_types, self.plugin_options)
def get(self, plugin_type: Type[TOptionsValidator]) -> Optional[TOptionsValidator]:
"""
Parameters
----------
plugin_type
Fetch the plugin options for this type
Returns
-------
Options of this plugin if they exit. Otherwise, return None.
"""
plugin_option_types = [type(plugin_options) for plugin_options in self.plugin_options]
if plugin_type in plugin_option_types:
return self.plugin_options[plugin_option_types.index(plugin_type)]
return None
class _PresetShell(StrictDictValidator):
# Have all present keys optional since parent presets could not have all the
# required keys. They will get validated in the init after the mergedeep of dicts
@ -154,11 +102,7 @@ class Preset(_PresetShell):
validator=PluginMapping.get(plugin_name).plugin_options_type,
)
@property
def _source_variables(self) -> List[str]:
return Entry.source_variables()
def __validate_and_get_plugins(self) -> PresetPlugins:
def _validate_and_get_plugins(self) -> PresetPlugins:
preset_plugins = PresetPlugins()
for key in self._keys:
@ -172,88 +116,6 @@ class Preset(_PresetShell):
return preset_plugins
def __validate_added_variables(self):
source_variables = copy.deepcopy(self._source_variables)
# Validate added download option variables here since plugins could subsequently use them
self.downloader_options.validate_with_variables(
source_variables=source_variables,
override_variables=self.overrides.dict_with_format_strings,
)
source_variables.extend(self.downloader_options.added_source_variables())
for _, plugin_options in sorted(
self.plugins.zipped(), key=lambda pl: pl[0].priority.modify_entry
):
# Validate current plugin using source + added plugin variables
plugin_options.validate_with_variables(
source_variables=source_variables,
override_variables=self.overrides.dict_with_format_strings,
)
# Extend existing source variables with ones created from this plugin
source_variables.extend(plugin_options.added_source_variables())
def __validate_override_string_formatter_validator(
self,
formatter_validator: Union[StringFormatterValidator, OverridesStringFormatterValidator],
):
# Set the formatter variables to be the overrides
variable_dict = copy.deepcopy(self.overrides.dict_with_format_strings)
# If the formatter supports source variables, set the formatter variables to include
# both source and override variables
if not isinstance(formatter_validator, OverridesStringFormatterValidator):
source_variables = {
source_var: "dummy_string"
for source_var in self._source_variables
+ self.downloader_options.added_source_variables()
}
variable_dict = dict(source_variables, **variable_dict)
# For all plugins, add in any extra added source variables
for plugin_options in self.plugins.plugin_options:
added_plugin_variables = {
source_var: "dummy_string"
for source_var in plugin_options.added_source_variables()
}
# sanity check plugin variables do not override source variables
expected_len = len(variable_dict) + len(added_plugin_variables)
variable_dict = dict(variable_dict, **added_plugin_variables)
assert (
len(variable_dict) == expected_len
), "plugin variables overwrote source variables"
_ = formatter_validator.apply_formatter(variable_dict=variable_dict)
def __recursive_preset_validate(
self,
validator: Optional[Validator] = None,
) -> None:
"""
Ensure all OverridesStringFormatterValidator's only contain variables from the overrides
and resolve.
"""
if validator is None:
validator = self
if isinstance(validator, DictValidator):
# pylint: disable=protected-access
# Usage of protected variables in other validators is fine. The reason to keep
# them protected is for readability when using them in subscriptions.
for validator_value in validator._validator_dict.values():
self.__recursive_preset_validate(validator_value)
# pylint: enable=protected-access
elif isinstance(validator, ListValidator):
for list_value in validator.list:
self.__recursive_preset_validate(list_value)
elif isinstance(validator, (StringFormatterValidator, OverridesStringFormatterValidator)):
self.__validate_override_string_formatter_validator(validator)
elif isinstance(validator, (DictFormatterValidator, OverridesDictFormatterValidator)):
for validator_value in validator.dict.values():
self.__validate_override_string_formatter_validator(validator_value)
def _get_presets_to_merge(
self, parent_presets: str | List[str], seen_presets: List[str], config: ConfigValidator
) -> List[Dict]:
@ -291,7 +153,7 @@ class Preset(_PresetShell):
return presets_to_merge
def __merge_parent_preset_dicts_if_present(self, config: ConfigValidator):
def _merge_parent_preset_dicts_if_present(self, config: ConfigValidator):
parent_preset_validator = self._validate_key_if_present(
key="preset", validator=StringListValidator
)
@ -314,7 +176,7 @@ class Preset(_PresetShell):
super().__init__(name=name, value=value)
# Perform the merge of parent presets before validating any keys
self.__merge_parent_preset_dicts_if_present(config=config)
self._merge_parent_preset_dicts_if_present(config=config)
self.downloader_options: MultiUrlValidator = self._validate_key(
key="download", validator=MultiUrlValidator
@ -329,13 +191,14 @@ class Preset(_PresetShell):
key="ytdl_options", validator=YTDLOptions, default={}
)
self.plugins: PresetPlugins = self._validate_and_get_plugins()
self.overrides = self._validate_key(key="overrides", validator=Overrides, default={})
self.plugins: PresetPlugins = self.__validate_and_get_plugins()
self.__validate_added_variables()
# After all options are initialized, perform a recursive post-validate that requires
# values from multiple validators
self.__recursive_preset_validate()
VariableValidation(
downloader_options=self.downloader_options,
output_options=self.output_options,
plugins=self.plugins,
).initialize_overrides(overrides=self.overrides).ensure_proper_usage()
@property
def name(self) -> str:

167
src/ytdl_sub/config/preset_options.py
View file

@ -1,88 +1,16 @@
from abc import ABC
from typing import Any
from typing import Dict
from typing import List
from typing import Optional
from typing import TypeVar
from yt_dlp.utils import sanitize_filename
from ytdl_sub.config.defaults import DEFAULT_DOWNLOAD_ARCHIVE_NAME
from ytdl_sub.entries.entry import Entry
from ytdl_sub.entries.variables.override_variables import SUBSCRIPTION_NAME
from ytdl_sub.utils.exceptions import ValidationException
from ytdl_sub.validators.file_path_validators import OverridesStringFormatterFilePathValidator
from ytdl_sub.validators.file_path_validators import StringFormatterFileNameValidator
from ytdl_sub.validators.strict_dict_validator import StrictDictValidator
from ytdl_sub.validators.string_datetime import StringDatetimeValidator
from ytdl_sub.validators.string_formatter_validators import DictFormatterValidator
from ytdl_sub.validators.string_formatter_validators import OverridesIntegerFormatterValidator
from ytdl_sub.validators.string_formatter_validators import OverridesStringFormatterValidator
from ytdl_sub.validators.string_formatter_validators import StringFormatterValidator
from ytdl_sub.validators.validators import BoolValidator
from ytdl_sub.validators.validators import LiteralDictValidator
from ytdl_sub.validators.validators import Validator
# pylint: disable=no-self-use
# pylint: disable=unused-argument
class OptionsValidator(Validator, ABC):
"""
Abstract class that validates options for preset sections (plugins, downloaders)
"""
def validation_exception(
self,
error_message: str | Exception,
) -> ValidationException:
"""
Parameters
----------
error_message
Error message to include in the validation exception
Returns
-------
Validation exception that points to the location in the config. To be used to throw good
validation exceptions at runtime from code outside this class.
"""
return self._validation_exception(error_message=error_message)
def added_source_variables(self) -> List[str]:
"""
If the plugin adds source variables, list them here.
Returns
-------
List of added source variables this plugin creates
"""
return []
def validate_with_variables(
self, source_variables: List[str], override_variables: Dict[str, str]
) -> None:
"""
Optional validation after init with the session's source and override variables.
Parameters
----------
source_variables
Available source variables when running the plugin
override_variables
Available override variables when running the plugin
"""
return None
TOptionsValidator = TypeVar("TOptionsValidator", bound=OptionsValidator)
class OptionsDictValidator(StrictDictValidator, OptionsValidator, ABC):
pass
# pylint: enable=no-self-use
# pylint: enable=unused-argument
class YTDLOptions(LiteralDictValidator):
@ -124,101 +52,6 @@ class YTDLOptions(LiteralDictValidator):
# Disable for proper docstring formatting
# pylint: disable=line-too-long
class Overrides(DictFormatterValidator):
"""
Optional. This section allows you to define variables that can be used in any string formatter.
For example, if you want your file and thumbnail files to match without copy-pasting a large
format string, you can define something like:
.. code-block:: yaml
presets:
my_example_preset:
overrides:
output_directory: "/path/to/media"
custom_file_name: "{upload_year}.{upload_month_padded}.{upload_day_padded}.{title_sanitized}"
# Then use the override variables in the output options
output_options:
output_directory: "{output_directory}"
file_name: "{custom_file_name}.{ext}"
thumbnail_name: "{custom_file_name}.{thumbnail_ext}"
Override variables can contain explicit values and other variables, including both override
and source variables.
In addition, any override variable defined will automatically create a ``sanitized`` variable
for use. In the example above, ``output_directory_sanitized`` will exist and perform
sanitization on the value when used.
"""
# pylint: enable=line-too-long
def _add_override_variable(self, key_name: str, format_string: str, sanitize: bool = False):
if sanitize:
key_name = f"{key_name}_sanitized"
format_string = sanitize_filename(format_string)
self._value[key_name] = StringFormatterValidator(
name="__should_never_fail__",
value=format_string,
)
def __init__(self, name, value):
super().__init__(name, value)
# Add sanitized overrides
for key in self._keys:
self._add_override_variable(
key_name=key,
format_string=self._value[key].format_string,
sanitize=True,
)
if SUBSCRIPTION_NAME not in self._value:
for sanitized in [True, False]:
self._add_override_variable(
key_name=SUBSCRIPTION_NAME,
format_string=self.subscription_name,
sanitize=sanitized,
)
@property
def subscription_name(self) -> str:
"""
Returns
-------
Name of the subscription
"""
return self._root_name
def apply_formatter(
self,
formatter: StringFormatterValidator,
entry: Optional[Entry] = None,
function_overrides: Dict[str, str] = None,
) -> str:
"""
Parameters
----------
formatter
Formatter to apply
entry
Optional. Entry to add source variables to the formatter
function_overrides
Optional. Explicit values to override the overrides themselves and source variables
Returns
-------
The format_string after .format has been called
"""
variable_dict = self.dict_with_format_strings
if entry:
variable_dict = dict(entry.to_dict(), **variable_dict)
if function_overrides:
variable_dict = dict(variable_dict, **function_overrides)
return formatter.apply_formatter(variable_dict)
class OutputOptions(StrictDictValidator):

0
src/ytdl_sub/config/validators/__init__.py Normal file
View file

59
src/ytdl_sub/config/validators/options.py Normal file
View file

@ -0,0 +1,59 @@
from abc import ABC
from typing import Dict
from typing import Set
from typing import TypeVar
from ytdl_sub.config.plugin.plugin_operation import PluginOperation
from ytdl_sub.utils.exceptions import ValidationException
from ytdl_sub.validators.strict_dict_validator import StrictDictValidator
from ytdl_sub.validators.validators import Validator
# pylint: disable=no-self-use
# pylint: disable=unused-argument
class OptionsValidator(Validator, ABC):
"""
Abstract class that validates options for preset sections (plugins, downloaders)
"""
def validation_exception(
self,
error_message: str | Exception,
) -> ValidationException:
"""
Parameters
----------
error_message
Error message to include in the validation exception
Returns
-------
Validation exception that points to the location in the config. To be used to throw good
validation exceptions at runtime from code outside this class.
"""
return self._validation_exception(error_message=error_message)
def modified_variables(self) -> Dict[PluginOperation, Set[str]]:
"""
If the plugin modifies existing variables, define them here
"""
return {}
def added_variables(
self,
resolved_variables: Set[str],
unresolved_variables: Set[str],
plugin_op: PluginOperation,
) -> Dict[PluginOperation, Set[str]]:
"""
If the plugin adds source variables, list them here.
"""
return {}
TOptionsValidator = TypeVar("TOptionsValidator", bound=OptionsValidator)
class OptionsDictValidator(StrictDictValidator, OptionsValidator, ABC):
pass

178
src/ytdl_sub/config/validators/variable_validation.py Normal file
View file

@ -0,0 +1,178 @@
import copy
from typing import Dict
from typing import Iterable
from typing import List
from typing import Optional
from typing import Set
from typing import Tuple
from ytdl_sub.config.overrides import Overrides
from ytdl_sub.config.plugin.plugin_mapping import PluginMapping
from ytdl_sub.config.plugin.plugin_operation import PluginOperation
from ytdl_sub.config.plugin.preset_plugins import PresetPlugins
from ytdl_sub.config.preset_options import OutputOptions
from ytdl_sub.config.validators.options import OptionsValidator
from ytdl_sub.downloaders.url.validators import MultiUrlValidator
from ytdl_sub.entries.script.variable_scripts import VARIABLE_SCRIPTS
from ytdl_sub.script.script import Script
from ytdl_sub.validators.string_formatter_validators import validate_formatters
def _add_dummy_variables(variables: Iterable[str]) -> Dict[str, str]:
dummy_variables: Dict[str, str] = {}
for var in variables:
dummy_variables[var] = ""
dummy_variables[f"{var}_sanitized"] = ""
return dummy_variables
def _get_added_and_modified_variables(
plugins: PresetPlugins, downloader_options: MultiUrlValidator
) -> Iterable[Tuple[OptionsValidator, Set[str], Set[str]]]:
"""
Iterates and returns the plugin options, added variables, modified variables
"""
options: List[OptionsValidator] = plugins.plugin_options
options.append(downloader_options)
for plugin_options in options:
added_variables: Set[str] = set()
modified_variables: Set[str] = set()
for plugin_added_variables in plugin_options.added_variables(
resolved_variables=set(),
unresolved_variables=set(),
plugin_op=PluginOperation.ANY,
).values():
added_variables |= set(plugin_added_variables)
for plugin_modified_variables in plugin_options.modified_variables().values():
modified_variables = plugin_modified_variables
yield plugin_options, added_variables, modified_variables
def _override_variables(overrides: Overrides) -> Set[str]:
return set(list(overrides.initial_variables().keys()))
def _entry_variables() -> Set[str]:
return set(list(VARIABLE_SCRIPTS.keys()))
class VariableValidation:
def __init__(
self,
downloader_options: MultiUrlValidator,
output_options: OutputOptions,
plugins: PresetPlugins,
):
self.downloader_options = downloader_options
self.output_options = output_options
self.plugins = plugins
self.script: Optional[Script] = None
self.resolved_variables: Set[str] = set()
self.unresolved_variables: Set[str] = set()
def initialize_overrides(self, overrides: Overrides) -> "VariableValidation":
"""
Do some gymnastics to initialize the Overrides script.
"""
entry_variables = _entry_variables()
override_variables = _override_variables(overrides)
# Set resolved variables as all entry + override variables
# at this point to generate every possible added/modified variable
self.resolved_variables = entry_variables | override_variables
for (
plugin_options,
added_variables,
modified_variables,
) in _get_added_and_modified_variables(
plugins=self.plugins,
downloader_options=self.downloader_options,
):
for added_variable in added_variables:
if not overrides.ensure_added_plugin_variable_valid(added_variable=added_variable):
# pylint: disable=protected-access
raise plugin_options._validation_exception(
f"Cannot use the variable name {added_variable} because it exists as a"
" built-in ytdl-sub variable name."
)
# pylint: enable=protected-access
# Set unresolved as variables that are added but do not exist as
# entry/override variables since they are created at run-time
self.unresolved_variables |= added_variables | modified_variables
# Then update resolved variables to reflect that
self.resolved_variables -= self.unresolved_variables
# Initialize overrides with unresolved variables + modified variables to throw an error.
# For modified variables, this is to prevent a resolve(update=True) to setting any
# dependencies until it has been explicitly added
overrides = overrides.initialize_script(unresolved_variables=self.unresolved_variables)
# copy the script and mock entry variables
self.script = copy.deepcopy(overrides.script).add(_add_dummy_variables(entry_variables))
return self
def _update_script(self) -> None:
_ = self.script.resolve(unresolvable=self.unresolved_variables, update=True)
def _add_variables(self, plugin_op: PluginOperation, options: OptionsValidator) -> Set[str]:
added_variables = options.added_variables(
resolved_variables=self.resolved_variables,
unresolved_variables=self.unresolved_variables,
plugin_op=plugin_op,
).get(plugin_op, set())
modified_variables = options.modified_variables().get(plugin_op, set())
resolved_variables = added_variables | modified_variables
self.script.add(_add_dummy_variables(resolved_variables))
self.resolved_variables |= resolved_variables
self.unresolved_variables -= resolved_variables
return added_variables
def ensure_proper_usage(self) -> None:
"""
Validate variables resolve as plugins are executed, and return
a mock script which contains actualized added variables from the plugins
"""
self._add_variables(PluginOperation.DOWNLOADER, options=self.downloader_options)
# Metadata variables to be added
for plugin_options in PluginMapping.order_options_by(
self.plugins.zipped(), PluginOperation.MODIFY_ENTRY_METADATA
):
self._add_variables(PluginOperation.MODIFY_ENTRY_METADATA, options=plugin_options)
self._update_script()
for plugin_options in PluginMapping.order_options_by(
self.plugins.zipped(), PluginOperation.MODIFY_ENTRY
):
added = self._add_variables(PluginOperation.MODIFY_ENTRY, options=plugin_options)
if added:
self._update_script()
# Validate that any formatter in the plugin options can resolve
validate_formatters(
script=self.script,
unresolved_variables=self.unresolved_variables,
validator=plugin_options,
)
validate_formatters(
script=self.script,
unresolved_variables=self.unresolved_variables,
validator=self.output_options,
)
assert not self.unresolved_variables

32
src/ytdl_sub/downloaders/info_json/info_json_downloader.py
View file

@ -6,17 +6,23 @@ from typing import Iterable
from typing import List
from typing import Optional
from ytdl_sub.config.preset_options import OptionsDictValidator
from ytdl_sub.config.preset_options import Overrides
from ytdl_sub.config.overrides import Overrides
from ytdl_sub.config.validators.options import OptionsDictValidator
from ytdl_sub.downloaders.source_plugin import SourcePlugin
from ytdl_sub.downloaders.ytdl_options_builder import YTDLOptionsBuilder
from ytdl_sub.entries.entry import Entry
from ytdl_sub.entries.script.variable_definitions import VARIABLES
from ytdl_sub.entries.script.variable_definitions import VariableDefinitions
from ytdl_sub.entries.script.variable_scripts import DOWNLOADER_INJECTED_VARIABLES
from ytdl_sub.entries.script.variable_scripts import VARIABLE_SCRIPTS
from ytdl_sub.utils.exceptions import ValidationException
from ytdl_sub.utils.file_handler import FileHandler
from ytdl_sub.utils.file_handler import get_file_extension
from ytdl_sub.ytdl_additions.enhanced_download_archive import DownloadMapping
from ytdl_sub.ytdl_additions.enhanced_download_archive import EnhancedDownloadArchive
v: VariableDefinitions = VARIABLES
class InfoJsonDownloaderOptions(OptionsDictValidator):
_optional_keys = {"no-op"}
@ -97,14 +103,26 @@ class InfoJsonDownloader(SourcePlugin[InfoJsonDownloaderOptions]):
for download_mapping in self._enhanced_download_archive.mapping.entry_mappings.values():
entry = self._get_entry_from_download_mapping(download_mapping)
# See if prior variables exist. If so, delete them from metadata
# to avoid saving them recursively on multiple updates
prior_variables = entry.maybe_get_prior_variables()
entry.initialize_script(self.overrides).add(
{
inj: prior_variables.get(
inj.variable_name,
VARIABLE_SCRIPTS[inj.variable_name],
)
for inj in DOWNLOADER_INJECTED_VARIABLES
}
)
entries.append(entry)
# Remove each entry from the live download archive since it will get re-added
# unless it is filtered
for entry in entries:
for entry in sorted(entries, key=lambda ent: ent.get(v.download_index, int)):
# Remove each entry from the live download archive since it will get re-added
# unless it is filtered
self._enhanced_download_archive.mapping.remove_entry(entry.uid)
for entry in sorted(entries, key=lambda ent: ent.download_index):
yield entry
# If the original entry file_path is no longer maintained in the new mapping, then

8
src/ytdl_sub/downloaders/source_plugin.py
View file

@ -8,10 +8,10 @@ from typing import Optional
from typing import Type
from typing import final
from ytdl_sub.config.plugin import BasePlugin
from ytdl_sub.config.plugin import Plugin
from ytdl_sub.config.preset_options import Overrides
from ytdl_sub.config.preset_options import TOptionsValidator
from ytdl_sub.config.overrides import Overrides
from ytdl_sub.config.plugin.plugin import BasePlugin
from ytdl_sub.config.plugin.plugin import Plugin
from ytdl_sub.config.validators.options import TOptionsValidator
from ytdl_sub.downloaders.ytdl_options_builder import YTDLOptionsBuilder
from ytdl_sub.entries.entry import Entry
from ytdl_sub.ytdl_additions.enhanced_download_archive import EnhancedDownloadArchive

73
src/ytdl_sub/downloaders/url/downloader.py
View file

@ -11,8 +11,7 @@ from typing import Tuple
from yt_dlp.utils import RejectedVideoReached
from ytdl_sub.config.plugin import PluginPriority
from ytdl_sub.config.preset_options import Overrides
from ytdl_sub.config.overrides import Overrides
from ytdl_sub.downloaders.source_plugin import SourcePlugin
from ytdl_sub.downloaders.source_plugin import SourcePluginExtension
from ytdl_sub.downloaders.url.validators import MultiUrlValidator
@ -22,14 +21,8 @@ from ytdl_sub.downloaders.ytdl_options_builder import YTDLOptionsBuilder
from ytdl_sub.downloaders.ytdlp import YTDLP
from ytdl_sub.entries.entry import Entry
from ytdl_sub.entries.entry_parent import EntryParent
from ytdl_sub.entries.variables.kwargs import COLLECTION_URL
from ytdl_sub.entries.variables.kwargs import COMMENTS
from ytdl_sub.entries.variables.kwargs import DOWNLOAD_INDEX
from ytdl_sub.entries.variables.kwargs import PLAYLIST_ENTRY
from ytdl_sub.entries.variables.kwargs import REQUESTED_SUBTITLES
from ytdl_sub.entries.variables.kwargs import SOURCE_ENTRY
from ytdl_sub.entries.variables.kwargs import SPONSORBLOCK_CHAPTERS
from ytdl_sub.entries.variables.kwargs import UPLOAD_DATE_INDEX
from ytdl_sub.entries.script.variable_definitions import VARIABLES
from ytdl_sub.entries.script.variable_definitions import VariableDefinitions
from ytdl_sub.utils.file_handler import FileHandler
from ytdl_sub.utils.logger import Logger
from ytdl_sub.utils.thumbnail import ThumbnailTypes
@ -37,6 +30,8 @@ from ytdl_sub.utils.thumbnail import download_and_convert_url_thumbnail
from ytdl_sub.utils.thumbnail import try_convert_download_thumbnail
from ytdl_sub.ytdl_additions.enhanced_download_archive import EnhancedDownloadArchive
v: VariableDefinitions = VARIABLES
download_logger = Logger.get(name="downloader")
@ -47,8 +42,6 @@ class URLDownloadState:
class UrlDownloaderThumbnailPlugin(SourcePluginExtension):
priority = PluginPriority(modify_entry=0)
def __init__(
self,
options: MultiUrlValidator,
@ -119,22 +112,18 @@ class UrlDownloaderThumbnailPlugin(SourcePluginExtension):
directory, run this function. This lets the downloader add any extra files directly to the
output directory, for things like YT channel image, banner.
"""
if entry.kwargs_contains(PLAYLIST_ENTRY):
if playlist_metadata := entry.get(v.playlist_metadata, dict):
self._download_parent_thumbnails(
thumbnail_list_info=collection_url.playlist_thumbnails,
entry=entry,
parent=EntryParent(
entry.kwargs(PLAYLIST_ENTRY), working_directory=self.working_directory
),
parent=EntryParent(playlist_metadata, working_directory=self.working_directory),
)
if entry.kwargs_contains(SOURCE_ENTRY):
if source_metadata := entry.get(v.source_metadata, dict):
self._download_parent_thumbnails(
thumbnail_list_info=collection_url.source_thumbnails,
entry=entry,
parent=EntryParent(
entry.kwargs(SOURCE_ENTRY), working_directory=self.working_directory
),
parent=EntryParent(source_metadata, working_directory=self.working_directory),
)
def modify_entry(self, entry: Entry) -> Optional[Entry]:
@ -147,17 +136,15 @@ class UrlDownloaderThumbnailPlugin(SourcePluginExtension):
if not self.is_dry_run:
try_convert_download_thumbnail(entry=entry)
if entry.kwargs_get(COLLECTION_URL) in self._collection_url_mapping:
if (input_url := entry.get(v.ytdl_sub_input_url, str)) in self._collection_url_mapping:
self._download_url_thumbnails(
collection_url=self._collection_url_mapping[entry.kwargs(COLLECTION_URL)],
collection_url=self._collection_url_mapping[input_url],
entry=entry,
)
return entry
class UrlDownloaderCollectionVariablePlugin(SourcePluginExtension):
priority = PluginPriority(modify_entry_metadata=0)
def __init__(
self,
options: MultiUrlValidator,
@ -181,7 +168,7 @@ class UrlDownloaderCollectionVariablePlugin(SourcePluginExtension):
"""
# COLLECTION_URL is a recent variable that may not exist for old entries when updating.
# Try to use source_webpage_url if it does not exist
entry_collection_url = entry.kwargs_get(COLLECTION_URL, entry.source_webpage_url)
entry_collection_url = entry.get(v.ytdl_sub_input_url, str)
# If the collection URL cannot find its mapping, use the last URL
collection_url = (
@ -189,7 +176,7 @@ class UrlDownloaderCollectionVariablePlugin(SourcePluginExtension):
or list(self._collection_url_mapping.values())[-1]
)
entry.add_variables(variables_to_add=collection_url.variables.dict_with_format_strings)
entry.add(collection_url.variables.dict_with_format_strings)
return entry
@ -356,7 +343,10 @@ class MultiUrlDownloader(SourcePlugin[MultiUrlValidator]):
else entry.is_thumbnail_downloaded_via_ytdlp,
url=entry.webpage_url,
)
return Entry(download_entry_dict, working_directory=self.working_directory)
return Entry(
download_entry_dict,
working_directory=self.working_directory,
)
def _iterate_child_entries(
self, url_validator: UrlValidator, entries: List[Entry]
@ -413,7 +403,9 @@ class MultiUrlDownloader(SourcePlugin[MultiUrlValidator]):
working_directory=self.working_directory,
)
orphans = EntryParent.from_entry_dicts_with_no_parents(
parents=parents, entry_dicts=entry_dicts, working_directory=self.working_directory
parents=parents,
entry_dicts=entry_dicts,
working_directory=self.working_directory,
)
return parents, orphans
@ -459,9 +451,8 @@ class MultiUrlDownloader(SourcePlugin[MultiUrlValidator]):
for entry in self._iterate_entries(
url_validator=collection_url, parents=parents, orphans=orphan_entries
):
# Add the collection URL to the info_dict to trace where it came from
entry.add_kwargs(
{COLLECTION_URL: self.overrides.apply_formatter(collection_url.url)}
entry.initialize_script(self.overrides).add(
{v.ytdl_sub_input_url: self.overrides.apply_formatter(collection_url.url)}
)
yield entry
@ -497,22 +488,12 @@ class MultiUrlDownloader(SourcePlugin[MultiUrlValidator]):
return None
upload_date_idx = self._enhanced_download_archive.mapping.get_num_entries_with_upload_date(
upload_date_standardized=entry.upload_date_standardized
upload_date_standardized=entry.get(v.upload_date_standardized, str)
)
download_idx = self._enhanced_download_archive.num_entries
entry.add_kwargs(
{
# Subtitles are not downloaded in metadata run, only here, so move over
REQUESTED_SUBTITLES: download_entry.kwargs_get(REQUESTED_SUBTITLES),
# Same with sponsorblock chapters
SPONSORBLOCK_CHAPTERS: download_entry.kwargs_get(SPONSORBLOCK_CHAPTERS),
COMMENTS: download_entry.kwargs_get(COMMENTS),
# Tracks number of entries downloaded
DOWNLOAD_INDEX: download_idx,
# Tracks number of entries with the same upload date to make them unique
UPLOAD_DATE_INDEX: upload_date_idx,
}
return entry.add_injected_variables(
download_entry=download_entry,
download_idx=download_idx,
upload_date_idx=upload_date_idx,
)
return entry

59
src/ytdl_sub/downloaders/url/validators.py
View file

@ -1,10 +1,12 @@
import copy
from typing import Any
from typing import Dict
from typing import List
from typing import Optional
from typing import Set
from ytdl_sub.config.preset_options import OptionsValidator
from ytdl_sub.config.plugin.plugin_operation import PluginOperation
from ytdl_sub.config.validators.options import OptionsValidator
from ytdl_sub.script.parser import parse
from ytdl_sub.validators.strict_dict_validator import StrictDictValidator
from ytdl_sub.validators.string_formatter_validators import DictFormatterValidator
from ytdl_sub.validators.string_formatter_validators import OverridesStringFormatterValidator
@ -243,45 +245,26 @@ class MultiUrlValidator(OptionsValidator):
# keep for readthedocs documentation
return self._urls.list[0].variables
def added_source_variables(self) -> List[str]:
def added_variables(
self,
resolved_variables: Set[str],
unresolved_variables: Set[str],
plugin_op: PluginOperation,
) -> Dict[PluginOperation, Set[str]]:
"""
Returns
-------
List of variables added. The first collection url always contains all the variables.
"""
return list(self._urls.list[0].variables.keys)
if plugin_op != PluginOperation.ANY:
for url in self._urls.list:
for variable_name, definition in url.variables.dict_with_format_strings.items():
used_variables = set(var.name for var in parse(definition).variables)
if unresolved := used_variables & unresolved_variables:
raise self._validation_exception(
f"variable {variable_name} cannot use the variables "
f"{', '.join(sorted(list(unresolved)))} because it depends on other"
" variables that are computed later in execution"
)
def validate_with_variables(
self, source_variables: List[str], override_variables: Dict[str, str]
) -> None:
"""
Ensures new variables added are not existing variables
"""
for source_var_name in self.added_source_variables():
if source_var_name in source_variables:
raise self._validation_exception(
f"'{source_var_name}' cannot be used as a variable name because it "
f"is an existing source variable"
)
base_variables = dict(
override_variables, **{source_var: "dummy_string" for source_var in source_variables}
)
# Apply formatting to each new source variable, ensure it resolves
for collection_url in self.urls.list:
for (
source_var_name,
source_var_formatter_str,
) in collection_url.variables.dict_with_format_strings.items():
_ = StringFormatterValidator(
name=f"{self._name}.{source_var_name}", value=source_var_formatter_str
).apply_formatter(base_variables)
# Ensure at least URL is non-empty
has_non_empty_url = False
for url_validator in self.urls.list:
has_non_empty_url |= bool(url_validator.url.apply_formatter(override_variables))
if not has_non_empty_url:
raise self._validation_exception("Must contain at least one url that is non-empty")
return {PluginOperation.DOWNLOADER: set(self._urls.list[0].variables.keys)}

2
src/ytdl_sub/downloaders/ytdl_options_builder.py
View file

@ -17,7 +17,7 @@ class YTDLOptionsBuilder:
self,
*ytdl_option_dicts: Optional[Dict],
before: bool = False,
strategy: mergedeep.Strategy = mergedeep.Strategy.TYPESAFE_ADDITIVE
strategy: mergedeep.Strategy = mergedeep.Strategy.TYPESAFE_ADDITIVE,
) -> "YTDLOptionsBuilder":
"""
Parameters

354
src/ytdl_sub/entries/base_entry.py
View file

@ -1,240 +1,22 @@
# pylint: disable=protected-access
from abc import ABC
from datetime import datetime
from pathlib import Path
from typing import Any
from typing import Dict
from typing import List
from typing import Optional
from typing import Type
from typing import TypeVar
from typing import final
from yt_dlp.utils import sanitize_filename
from ytdl_sub.entries.variables.kwargs import DESCRIPTION
from ytdl_sub.entries.variables.kwargs import EPOCH
from ytdl_sub.entries.variables.kwargs import IE_KEY
from ytdl_sub.entries.variables.kwargs import TITLE
from ytdl_sub.entries.variables.kwargs import UID
from ytdl_sub.entries.variables.kwargs import UPLOADER
from ytdl_sub.entries.variables.kwargs import UPLOADER_ID
from ytdl_sub.entries.variables.kwargs import UPLOADER_URL
from ytdl_sub.entries.variables.kwargs import WEBPAGE_URL
# pylint: disable=no-member
def _sanitize_plex(string: str) -> str:
out = ""
for char in string:
match char:
case "0":
out += "０"
case "1":
out += "１"
case "2":
out += "２"
case "3":
out += "３"
case "4":
out += "４"
case "5":
out += "５"
case "6":
out += "６"
case "7":
out += "７"
case "8":
out += "８"
case "9":
out += "９"
case _:
out += char
return out
class BaseEntryVariables:
"""
Source variables are ``{variables}`` that contain metadata from downloaded media.
These variables can be used with fields that expect
:class:`~ytdl_sub.validators.string_formatter_validators.StringFormatterValidator`,
but not
:class:`~ytdl_sub.validators.string_formatter_validators.OverridesStringFormatterValidator`.
"""
@property
def uid(self: "BaseEntry") -> str:
"""
Returns
-------
str
The entry's unique ID
"""
return str(self.kwargs(UID))
@property
def uid_sanitized(self: "BaseEntry") -> str:
"""
Returns
-------
str
The sanitized uid of the entry, which is safe to use for Unix and Windows file names.
"""
return sanitize_filename(self.uid)
@property
def uid_sanitized_plex(self: "BaseEntry") -> str:
"""
Returns
-------
str
The sanitized uid with additional sanitizing for Plex. Replaces numbers with
fixed-width numbers so Plex does not recognize them as season or episode numbers.
"""
return _sanitize_plex(self.uid_sanitized)
@property
def extractor(self: "BaseEntry") -> str:
"""
Returns
-------
str
The ytdl extractor name used in the download archive
"""
# pylint: disable=line-too-long
# Taken from https://github.com/yt-dlp/yt-dlp/blob/e6ab678e36c40ded0aae305bbb866cdab554d417/yt_dlp/YoutubeDL.py#L3514
# pylint: enable=line-too-long
return str(self.kwargs_get("extractor_key") or self.kwargs(IE_KEY)).lower()
@property
def epoch(self: "BaseEntry") -> int:
"""
Returns
-------
int
The unix epoch of when the metadata was scraped by yt-dlp.
"""
return self.kwargs(EPOCH)
@property
def epoch_date(self: "BaseEntry") -> str:
"""
Returns
-------
str
The epoch's date, in YYYYMMDD format.
"""
return datetime.utcfromtimestamp(self.epoch).strftime("%Y%m%d")
@property
def epoch_hour(self: "BaseEntry") -> str:
"""
Returns
-------
str
The epoch's hour, padded
"""
return datetime.utcfromtimestamp(self.epoch).strftime("%H")
@property
def title(self: "BaseEntry") -> str:
"""
Returns
-------
str
The title of the entry. If a title does not exist, returns its unique ID.
"""
return self.kwargs_get(TITLE, self.uid)
@property
def title_sanitized(self) -> str:
"""
Returns
-------
str
The sanitized title of the entry, which is safe to use for Unix and Windows file names.
"""
return sanitize_filename(self.title)
@property
def title_sanitized_plex(self) -> str:
"""
Returns
-------
str
The sanitized title with additional sanitizing for Plex. Replaces numbers with
fixed-width numbers so Plex does not recognize them as season or episode numbers.
"""
return _sanitize_plex(self.title_sanitized)
@property
def webpage_url(self: "BaseEntry") -> str:
"""
Returns
-------
str
The url to the webpage.
"""
return self.kwargs(WEBPAGE_URL)
@property
def info_json_ext(self) -> str:
"""
Returns
-------
str
The "info.json" extension
"""
return "info.json"
@property
def description(self: "BaseEntry") -> str:
"""
Returns
-------
str
The description if it exists. Otherwise, returns an emtpy string.
"""
return self.kwargs_get(DESCRIPTION, "")
@property
def uploader_id(self: "BaseEntry") -> str:
"""
Returns
-------
str
The uploader id if it exists, otherwise return the unique ID.
"""
return self.kwargs_get(UPLOADER_ID, self.uid)
@property
def uploader(self: "BaseEntry") -> str:
"""
Returns
-------
str
The uploader if it exists, otherwise return the uploader ID.
"""
return self.kwargs_get(UPLOADER, self.uploader_id)
@property
def uploader_url(self: "BaseEntry") -> str:
"""
Returns
-------
str
The uploader url if it exists, otherwise returns the webpage_url.
"""
return self.kwargs_get(UPLOADER_URL, self.webpage_url)
# pylint: enable=no-member
from ytdl_sub.entries.script.variable_definitions import VARIABLES
from ytdl_sub.entries.script.variable_definitions import VariableDefinitions
v: VariableDefinitions = VARIABLES
TBaseEntry = TypeVar("TBaseEntry", bound="BaseEntry")
class BaseEntry(BaseEntryVariables, ABC):
class BaseEntry(ABC):
"""
Abstract entry object to represent anything download from ytdl (playlist metadata, media, etc).
"""
@ -253,30 +35,63 @@ class BaseEntry(BaseEntryVariables, ABC):
self._working_directory = working_directory
self._kwargs = entry_dict
self._additional_variables: Dict[str, str | int] = {}
@property
def uid(self) -> str:
"""
Returns
-------
str
The entry's unique ID
"""
return str(self._kwargs[v.uid.metadata_key])
def kwargs_contains(self, key: str) -> bool:
"""Returns whether internal kwargs contains the specified key"""
return key in self._kwargs
@property
def download_archive_extractor(self) -> str:
"""
The extractor name used in yt-dlp download archives
"""
# pylint: disable=line-too-long
# Taken from https://github.com/yt-dlp/yt-dlp/blob/e6ab678e36c40ded0aae305bbb866cdab554d417/yt_dlp/YoutubeDL.py#L3514
# pylint: enable=line-too-long
return str(
self._kwargs_get(v.extractor_key.metadata_key)
or self._kwargs_get(v.ie_key.metadata_key)
or "NO_EXTRACTOR"
).lower()
def kwargs(self, key) -> Any:
"""Returns an internal kwarg value supplied from ytdl"""
if not self.kwargs_contains(key):
raise KeyError(f"Expected '{key}' in {self.__class__.__name__} but does not exist.")
output = self._kwargs[key]
@property
def title(self) -> str:
"""
The title of the entry. If a title does not exist, returns its unique ID.
"""
return self._kwargs_get(v.title.metadata_key, self.uid)
# Replace curly braces with unicode version to avoid variable shenanigans
if isinstance(output, str):
return output.replace("{", "｛").replace("}", "｝")
return output
@property
def webpage_url(self) -> str:
"""
The url to the webpage.
"""
return self._kwargs[v.webpage_url.metadata_key]
def kwargs_get(self, key: str, default: Optional[Any] = None) -> Any:
@property
def info_json_ext(self) -> str:
"""The "info.json" extension"""
return "info.json"
@property
def uploader_id(self) -> str:
"""
The uploader id if it exists, otherwise return the unique ID.
"""
return self._kwargs_get(v.uploader_id.metadata_key, self.uid)
def _kwargs_get(self, key: str, default: Optional[Any] = None) -> Any:
"""
Dict get on kwargs
"""
if not self.kwargs_contains(key) or self.kwargs(key) is None:
if (out := self._kwargs.get(key)) is None:
return default
return self.kwargs(key)
return out
def working_directory(self) -> str:
"""
@ -303,31 +118,6 @@ class BaseEntry(BaseEntryVariables, ABC):
self._kwargs = dict(self._kwargs, **variables_to_add)
return self
def add_variables(self, variables_to_add: Dict[str, str]) -> "BaseEntry":
"""
Parameters
----------
variables_to_add
Variables to add to this entry
Returns
-------
self
Raises
------
ValueError
If a variable trying to be added already exists as a source variable
"""
for variable_name in variables_to_add.keys():
if self.kwargs_contains(variable_name):
raise ValueError(
f"Cannot add variable '{variable_name}': already exists in the kwargs"
)
self._additional_variables = dict(self._additional_variables, **variables_to_add)
return self
def get_download_info_json_name(self) -> str:
"""
Returns
@ -344,36 +134,6 @@ class BaseEntry(BaseEntryVariables, ABC):
"""
return str(Path(self.working_directory()) / self.get_download_info_json_name())
def _added_variables(self) -> Dict[str, str]:
"""
Returns
-------
Dict of variables added to this entry
"""
return self._additional_variables
@classmethod
def source_variables(cls) -> List[str]:
"""
Returns
-------
List of all source variables
"""
property_names = [prop for prop in dir(cls) if isinstance(getattr(cls, prop), property)]
return property_names
@final
def to_dict(self) -> Dict[str, str]:
"""
Returns
-------
Dictionary containing all variables
"""
source_variable_dict = {
source_var: getattr(self, source_var) for source_var in self.source_variables()
}
return dict(source_variable_dict, **self._added_variables())
@final
def to_type(self, entry_type: Type[TBaseEntry]) -> TBaseEntry:
"""
@ -392,7 +152,7 @@ class BaseEntry(BaseEntryVariables, ABC):
"""
entry_type: Optional[str] = None
if isinstance(entry_dict, cls):
entry_type = entry_dict.kwargs_get("_type")
entry_type = entry_dict._kwargs_get("_type")
if isinstance(entry_dict, dict):
entry_type = entry_dict.get("_type")
@ -407,7 +167,7 @@ class BaseEntry(BaseEntryVariables, ABC):
"""
entry_ext: Optional[str] = None
if isinstance(entry_dict, cls):
entry_ext = entry_dict.kwargs_get("ext")
entry_ext = entry_dict._kwargs_get("ext")
if isinstance(entry_dict, dict):
entry_ext = entry_dict.get("ext")
@ -419,4 +179,4 @@ class BaseEntry(BaseEntryVariables, ABC):
-------
extractor + uid, making this a unique hash for any entry
"""
return self.extractor + self.uid
return self.download_archive_extractor + self.uid

133
src/ytdl_sub/entries/entry.py
View file

@ -1,21 +1,103 @@
# pylint: disable=protected-access
import copy
import json
import os
from pathlib import Path
from typing import Any
from typing import Dict
from typing import Optional
from typing import Type
from typing import TypeVar
from typing import final
from ytdl_sub.entries.base_entry import BaseEntry
from ytdl_sub.entries.variables.entry_variables import EntryVariables
from ytdl_sub.entries.script.variable_definitions import VARIABLES
from ytdl_sub.entries.script.variable_definitions import Variable
from ytdl_sub.entries.script.variable_definitions import VariableDefinitions
from ytdl_sub.script.utils.exceptions import ScriptVariableNotResolved
from ytdl_sub.utils.script import ScriptUtils
from ytdl_sub.utils.scriptable import Scriptable
from ytdl_sub.validators.audo_codec_validator import AUDIO_CODEC_EXTS
from ytdl_sub.validators.audo_codec_validator import VIDEO_CODEC_EXTS
v: VariableDefinitions = VARIABLES
class Entry(EntryVariables, BaseEntry):
_YTDL_SUB_ENTRY_VARIABLES_KWARG_KEY: str = "ytdl_sub_entry_variables"
ytdl_sub_chapters_from_comments = Variable("ytdl_sub_chapters_from_comments")
ytdl_sub_split_by_chapters_parent_uid = Variable("ytdl_sub_split_by_chapters_parent_uid")
TypeT = TypeVar("TypeT")
class Entry(BaseEntry, Scriptable):
"""
Entry object to represent a single media object returned from yt-dlp.
"""
def __init__(self, entry_dict: Dict, working_directory: str):
BaseEntry.__init__(self, entry_dict=entry_dict, working_directory=working_directory)
Scriptable.__init__(self)
def _add_entry_kwargs_to_script(self) -> None:
# Add entry metadata, but avoid the `.add()` helper since it also adds sanitized
self.unresolvable.remove(v.entry_metadata.variable_name)
self.script.add({v.entry_metadata.variable_name: ScriptUtils.to_script(self._kwargs)})
self.update_script()
def initialize_script(self, other: Optional[Scriptable] = None) -> "Entry":
"""
Initializes the entry script using the Overrides script, then adding
its kwargs to the entry metadata variable
"""
# Overrides contains added variables that are unresolvable, add them here
if other:
self.script = copy.deepcopy(other.script)
self.unresolvable = copy.deepcopy(other.unresolvable)
self._add_entry_kwargs_to_script()
return self
def get(self, variable: Variable, expected_type: Type[TypeT]) -> TypeT:
"""
Gets a variable of an expected type. Will error if it does not exist or is not resolved.
"""
out = self.script.resolve(unresolvable=self.unresolvable).get_native(variable.variable_name)
return expected_type(out)
def try_get(self, variable: Variable, expected_type: Type[TypeT]) -> Optional[TypeT]:
"""
Gets a variable of an expected type. Returns None if it does not exist or is not resolved.
"""
try:
return self.get(variable=variable, expected_type=expected_type)
except ScriptVariableNotResolved:
return None
def add_injected_variables(
self, download_entry: "Entry", download_idx: int, upload_date_idx: int
) -> "Entry":
"""
Adds variables that get injected into the Entry script that aren't available at
metadata scrape time (only after the actual download).
"""
self.add(
{
# Tracks number of entries downloaded
v.download_index: download_idx + 1,
# Tracks number of entries with the same upload date to make them unique
v.upload_date_index: upload_date_idx + 1,
v.requested_subtitles: download_entry._kwargs_get(
v.requested_subtitles.metadata_key, []
),
v.chapters: download_entry._kwargs_get(v.chapters.metadata_key, []),
v.sponsorblock_chapters: download_entry._kwargs_get(
v.sponsorblock_chapters.metadata_key, []
),
v.comments: download_entry._kwargs_get(v.comments.metadata_key, []),
}
)
return self
@property
def ext(self) -> str:
"""
@ -23,12 +105,13 @@ class Entry(EntryVariables, BaseEntry):
This is not reflected in the entry. See if the mkv file exists and return "mkv" if so,
otherwise, return the original extension.
"""
for possible_ext in [super().ext, "mkv"]:
ext = self.try_get(v.ext, str) or self._kwargs[v.ext.metadata_key]
for possible_ext in [ext, "mkv"]:
file_path = str(Path(self.working_directory()) / f"{self.uid}.{possible_ext}")
if os.path.isfile(file_path):
return possible_ext
return super().ext
return ext
def get_download_file_name(self) -> str:
"""
@ -48,7 +131,7 @@ class Entry(EntryVariables, BaseEntry):
-------
The download thumbnail's file name
"""
return f"{self.uid}.{self.thumbnail_ext}"
return f"{self.uid}.{self.get(v.thumbnail_ext, str)}"
def get_download_thumbnail_path(self) -> str:
"""Returns the entry's thumbnail's file path to where it was downloaded"""
@ -59,7 +142,7 @@ class Entry(EntryVariables, BaseEntry):
The source `thumbnail` value and the actual downloaded thumbnail extension sometimes do
not match. Return the actual downloaded thumbnail path.
"""
thumbnails = self.kwargs_get("thumbnails", [])
thumbnails = self._kwargs_get("thumbnails", [])
possible_thumbnail_exts = {"jpg", "webp"} # Always check for jpg and webp thumbs
for thumbnail in thumbnails:
@ -77,7 +160,7 @@ class Entry(EntryVariables, BaseEntry):
Write the entry's _kwargs back into the info.json file as well as its source variables
"""
kwargs_dict = copy.deepcopy(self._kwargs)
kwargs_dict["ytdl_sub_entry_variables"] = self.to_dict()
kwargs_dict[_YTDL_SUB_ENTRY_VARIABLES_KWARG_KEY] = self.to_dict()
kwargs_json = json.dumps(kwargs_dict, ensure_ascii=False, sort_keys=True, indent=2)
with open(self.get_download_info_json_path(), "w", encoding="utf-8") as file:
@ -119,3 +202,39 @@ class Entry(EntryVariables, BaseEntry):
break
return file_exists
def maybe_get_prior_variables(self) -> Dict[str, Any]:
"""
If variables exist in the .info.json from a prior run, delete them
from kwargs (to prevent nested writes) and return them
"""
maybe_prior_variables: Dict[str, Any] = {}
if _YTDL_SUB_ENTRY_VARIABLES_KWARG_KEY in self._kwargs:
maybe_prior_variables = self._kwargs[_YTDL_SUB_ENTRY_VARIABLES_KWARG_KEY]
del self._kwargs[_YTDL_SUB_ENTRY_VARIABLES_KWARG_KEY]
return maybe_prior_variables
@final
def to_dict(self) -> Dict[str, Any]:
"""
Returns
-------
Dictionary containing all variables
"""
return self.script.resolve().as_native()
@classmethod
def create_split_entry(cls, entry: "Entry", new_uid: str) -> "Entry":
"""
Creates a copy of an entry with a new uid to use as the starting point for a split entry
"""
new_entry = copy.deepcopy(entry)
new_entry._kwargs[v.uid.metadata_key] = new_uid
new_entry.add(
{
v.uid.variable_name: new_uid,
ytdl_sub_split_by_chapters_parent_uid.variable_name: entry.uid,
}
)
return new_entry

162
src/ytdl_sub/entries/entry_parent.py
View file

@ -1,55 +1,33 @@
import math
from typing import Any
from typing import Dict
from typing import List
from typing import Optional
import mergedeep
from ytdl_sub.entries.base_entry import BaseEntry
from ytdl_sub.entries.base_entry import TBaseEntry
from ytdl_sub.entries.entry import Entry
from ytdl_sub.entries.variables.kwargs import DESCRIPTION
from ytdl_sub.entries.variables.kwargs import PLAYLIST_COUNT
from ytdl_sub.entries.variables.kwargs import PLAYLIST_DESCRIPTION
from ytdl_sub.entries.variables.kwargs import PLAYLIST_ENTRY
from ytdl_sub.entries.variables.kwargs import PLAYLIST_INDEX
from ytdl_sub.entries.variables.kwargs import PLAYLIST_MAX_UPLOAD_YEAR
from ytdl_sub.entries.variables.kwargs import PLAYLIST_MAX_UPLOAD_YEAR_TRUNCATED
from ytdl_sub.entries.variables.kwargs import PLAYLIST_TITLE
from ytdl_sub.entries.variables.kwargs import PLAYLIST_UID
from ytdl_sub.entries.variables.kwargs import PLAYLIST_UPLOADER
from ytdl_sub.entries.variables.kwargs import PLAYLIST_UPLOADER_ID
from ytdl_sub.entries.variables.kwargs import PLAYLIST_UPLOADER_URL
from ytdl_sub.entries.variables.kwargs import PLAYLIST_WEBPAGE_URL
from ytdl_sub.entries.variables.kwargs import SOURCE_COUNT
from ytdl_sub.entries.variables.kwargs import SOURCE_DESCRIPTION
from ytdl_sub.entries.variables.kwargs import SOURCE_ENTRY
from ytdl_sub.entries.variables.kwargs import SOURCE_INDEX
from ytdl_sub.entries.variables.kwargs import SOURCE_TITLE
from ytdl_sub.entries.variables.kwargs import SOURCE_UID
from ytdl_sub.entries.variables.kwargs import SOURCE_UPLOADER
from ytdl_sub.entries.variables.kwargs import SOURCE_UPLOADER_ID
from ytdl_sub.entries.variables.kwargs import SOURCE_UPLOADER_URL
from ytdl_sub.entries.variables.kwargs import SOURCE_WEBPAGE_URL
from ytdl_sub.entries.variables.kwargs import TITLE
from ytdl_sub.entries.variables.kwargs import UID
from ytdl_sub.entries.variables.kwargs import UPLOADER
from ytdl_sub.entries.variables.kwargs import UPLOADER_ID
from ytdl_sub.entries.variables.kwargs import UPLOADER_URL
from ytdl_sub.entries.variables.kwargs import WEBPAGE_URL
from ytdl_sub.entries.script.variable_definitions import VARIABLES
from ytdl_sub.entries.script.variable_definitions import MetadataVariable
from ytdl_sub.entries.script.variable_definitions import VariableDefinitions
from ytdl_sub.entries.script.variable_scripts import ENTRY_DEFAULT_VARIABLES
from ytdl_sub.entries.script.variable_scripts import ENTRY_REQUIRED_VARIABLES
v: VariableDefinitions = VARIABLES
class ParentType:
PLAYLIST = "playlist"
SOURCE = "source"
def _sort_entries(entries: List[TBaseEntry]) -> List[TBaseEntry]:
"""Try sorting by playlist_id first, then fall back to uid"""
return sorted(entries, key=lambda ent: (ent.kwargs_get(PLAYLIST_INDEX, math.inf), ent.uid))
# pylint: disable=protected-access
class EntryParent(BaseEntry):
@classmethod
def _sort_entries(cls, entries: List[TBaseEntry]) -> List[TBaseEntry]:
"""Try sorting by playlist_id first, then fall back to uid"""
return sorted(
entries,
key=lambda ent: (ent._kwargs_get(v.playlist_index.metadata_key, math.inf), ent.uid),
)
def __init__(self, entry_dict: Dict, working_directory: str):
super().__init__(entry_dict=entry_dict, working_directory=working_directory)
self._parent_children: List["EntryParent"] = []
@ -73,86 +51,38 @@ class EntryParent(BaseEntry):
self.entry_children()
)
def _playlist_variables(self, idx: int, children: List[TBaseEntry], parent_type: str) -> Dict:
_count = self.kwargs_get(PLAYLIST_COUNT, len(children))
_index = children[idx].kwargs_get(PLAYLIST_INDEX, idx + 1)
if parent_type == ParentType.SOURCE:
return {SOURCE_INDEX: _index, SOURCE_COUNT: _count}
return {
SOURCE_INDEX: self.kwargs_get(SOURCE_INDEX, 1),
SOURCE_COUNT: self.kwargs_get(SOURCE_INDEX, 1),
PLAYLIST_INDEX: _index,
PLAYLIST_COUNT: _count,
}
def _parent_variables(self, parent_type: str) -> Dict:
def _(source_key: str, playlist_key: str) -> str:
return playlist_key if parent_type == ParentType.PLAYLIST else source_key
def __(key: str) -> Optional[str]:
return self.kwargs_get(key=key)
return {
_(SOURCE_ENTRY, PLAYLIST_ENTRY): self._kwargs,
_(SOURCE_TITLE, PLAYLIST_TITLE): __(TITLE),
_(SOURCE_WEBPAGE_URL, PLAYLIST_WEBPAGE_URL): __(WEBPAGE_URL),
_(SOURCE_UID, PLAYLIST_UID): __(UID),
_(SOURCE_DESCRIPTION, PLAYLIST_DESCRIPTION): __(DESCRIPTION),
_(SOURCE_UPLOADER, PLAYLIST_UPLOADER): __(UPLOADER),
_(SOURCE_UPLOADER_ID, PLAYLIST_UPLOADER_ID): __(UPLOADER_ID),
_(SOURCE_UPLOADER_URL, PLAYLIST_UPLOADER_URL): __(UPLOADER_URL),
}
def _get_entry_children_variable_list(self, variable_name: str) -> List[str | int]:
return [getattr(entry_child, variable_name) for entry_child in self.entry_children()]
def _entry_aggregate_variables(self) -> Dict:
if not self.entry_children():
return {}
return {
PLAYLIST_MAX_UPLOAD_YEAR: max(self._get_entry_children_variable_list("upload_year")),
PLAYLIST_MAX_UPLOAD_YEAR_TRUNCATED: max(
self._get_entry_children_variable_list("upload_year_truncated")
),
}
# pylint: disable=protected-access
def _sibling_entry_metadata(self) -> List[Dict[str, Any]]:
sibling_entry_metadata: List[Dict[str, Any]] = []
variable_filter: List[MetadataVariable] = list(ENTRY_REQUIRED_VARIABLES.keys()) + list(
ENTRY_DEFAULT_VARIABLES.keys()
)
for entry in self.entry_children():
sibling_entry_metadata.append(
{var.metadata_key: entry._kwargs_get(var.metadata_key) for var in variable_filter}
)
return sibling_entry_metadata
def _set_child_variables(self, parents: Optional[List["EntryParent"]] = None) -> "EntryParent":
if parents is None:
parents = [self]
self.add_kwargs(
self._playlist_variables(idx=0, children=parents, parent_type=ParentType.SOURCE)
)
kwargs_to_add: Dict = {}
kwargs_to_add: Dict[str, Any] = {
v.sibling_metadata.metadata_key: self._sibling_entry_metadata()
}
if len(parents) >= 1:
mergedeep.merge(kwargs_to_add, parents[-1]._parent_variables(ParentType.PLAYLIST))
kwargs_to_add[v.playlist_metadata.metadata_key] = parents[-1]._kwargs
if len(parents) >= 2:
mergedeep.merge(kwargs_to_add, parents[-2]._parent_variables(ParentType.SOURCE))
kwargs_to_add[v.source_metadata.metadata_key] = parents[-2]._kwargs
if len(parents) >= 3:
raise ValueError(
"ytdl-sub currently does support more than 3 layers of playlists/entries. "
"If you encounter this error, please file a ticket with the URLs used."
)
mergedeep.merge(kwargs_to_add, self._entry_aggregate_variables())
for idx, entry_child in enumerate(self.entry_children()):
entry_child.add_kwargs(
self._playlist_variables(
idx=idx, children=self.entry_children(), parent_type=ParentType.PLAYLIST
)
)
entry_child.add_kwargs(kwargs_to_add)
for entry_child in self.entry_children():
entry_child._kwargs = dict(entry_child._kwargs, **kwargs_to_add)
for idx, parent_child in enumerate(self.parent_children()):
parent_child.add_kwargs(
self._playlist_variables(
idx=idx, children=self.parent_children(), parent_type=ParentType.SOURCE
)
)
for parent_child in self.parent_children():
parent_child._set_child_variables(parents=parents + [parent_child])
return self
@ -170,8 +100,10 @@ class EntryParent(BaseEntry):
if entry_dict in self
]
self._parent_children = _sort_entries([ent for ent in entries if self.is_entry_parent(ent)])
self._entry_children = _sort_entries(
self._parent_children = self._sort_entries(
[ent for ent in entries if self.is_entry_parent(ent)]
)
self._entry_children = self._sort_entries(
[ent.to_type(Entry) for ent in entries if self.is_entry(ent)]
)
@ -190,7 +122,7 @@ class EntryParent(BaseEntry):
-------
Desired thumbnail url if it exists. None if it does not.
"""
for thumbnail in self.kwargs_get("thumbnails", []):
for thumbnail in self._kwargs_get("thumbnails", []):
if thumbnail["id"] == thumbnail_id:
return thumbnail["url"]
return None
@ -200,7 +132,7 @@ class EntryParent(BaseEntry):
if isinstance(item, dict):
playlist_id = item.get("playlist_id")
elif isinstance(item, BaseEntry):
playlist_id = item.kwargs_get("playlist_id")
playlist_id = item._kwargs_get("playlist_id")
if not playlist_id:
return False
@ -275,11 +207,12 @@ class EntryParent(BaseEntry):
return parents
# pylint: enable=protected-access
@classmethod
def from_entry_dicts_with_no_parents(
cls, parents: List["EntryParent"], entry_dicts: List[Dict], working_directory: str
cls,
parents: List["EntryParent"],
entry_dicts: List[Dict],
working_directory: str,
) -> List[Entry]:
"""
Reads all entries that do not have any parents
@ -289,7 +222,10 @@ class EntryParent(BaseEntry):
return any(entry_dict in parent for parent in parents)
return [
Entry(entry_dict=entry_dict, working_directory=working_directory)
Entry(
entry_dict=entry_dict,
working_directory=working_directory,
)
for entry_dict in entry_dicts
if cls.is_entry(entry_dict) and not _in_any_parents(entry_dict)
]

0
src/ytdl_sub/entries/script/__init__.py Normal file
View file

178
src/ytdl_sub/entries/script/custom_functions.py Normal file
View file

@ -0,0 +1,178 @@
import os
import posixpath
from yt_dlp.utils import sanitize_filename
from ytdl_sub.script.functions import Functions
from ytdl_sub.script.types.map import Map
from ytdl_sub.script.types.resolvable import AnyArgument
from ytdl_sub.script.types.resolvable import Integer
from ytdl_sub.script.types.resolvable import ReturnableArgument
from ytdl_sub.script.types.resolvable import String
from ytdl_sub.script.utils.exceptions import RuntimeException
from ytdl_sub.utils.file_path import FilePathTruncater
def _pad(num: int, width: int):
return str(num).zfill(width)
_days_in_month = [0, 31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31]
class CustomFunctions:
@staticmethod
def legacy_bracket_safety(value: ReturnableArgument) -> ReturnableArgument:
"""
ytdl-sub used to replace brackets ('{', '}') with unicode brackets ('｛', '｝') to not
interfere with its legacy variable scripting system. This function replicates that
behavior.
"""
if isinstance(value, String):
value = String(value.value.replace("{", "｛").replace("}", "｝"))
return value
@staticmethod
def to_native_filepath(filepath: String) -> String:
"""
Convert any unix-based path separators ('/') with the OS's native
separator.
"""
return String(filepath.value.replace(posixpath.sep, os.sep))
@staticmethod
def truncate_filepath_if_too_long(filepath: String) -> String:
"""
If a file-path is too long for the OS, this function will truncate it while preserving
the extension.
"""
return String(FilePathTruncater.maybe_truncate_file_path(filepath.value))
@staticmethod
def sanitize(value: AnyArgument) -> String:
"""
Sanitize a string using yt-dlp's ``sanitize_filename`` method to ensure it's safe to use
for file/directory names on any OS.
"""
return String(sanitize_filename(str(value)))
@staticmethod
def sanitize_plex_episode(string: String) -> String:
"""
Sanitize a string using ``sanitize`` and replace numerics with their respective fixed-width
numbers. This is used to have Plex avoid scraping numbers like ``4x4`` as the
season and/or episode.
"""
sanitized_string = CustomFunctions.sanitize(string).value
out = ""
for char in sanitized_string:
match char:
case "0":
out += "０"
case "1":
out += "１"
case "2":
out += "２"
case "3":
out += "３"
case "4":
out += "４"
case "5":
out += "５"
case "6":
out += "６"
case "7":
out += "７"
case "8":
out += "８"
case "9":
out += "９"
case _:
out += char
return String(out)
@staticmethod
def to_date_metadata(yyyymmdd: String) -> Map:
"""
Takes a date in the form of YYYYMMDD and returns a Map containing:
- date (String, YYYYMMDD)
- date_standardized (String, YYYY-MM-DD)
- year (Integer)
- month (Integer)
- day (Integer)
- year_truncated (String, YY from YY[YY])
- month_padded (String)
- day_padded (String)
- year_truncated_reversed (Integer, 100 - year_truncated)
- month_reversed (Integer, 13 - month)
- month_reversed_padded (String)
- day_reversed (Integer, total_days_in_month + 1 - day)
- day_reversed_padded (String)
- day_of_year (Integer)
- day_of_year_padded (String, padded 3)
- day_of_year_reversed (Integer, total_days_in_year + 1 - day_of_year)
- day_of_year_reversed_padded (String, padded 3)
"""
date_str = yyyymmdd.value
if not (date_str.isnumeric() and len(date_str) == 8):
raise RuntimeException(
f"Expected input of date_metadata to be YYYYMMDD, but received {date_str}"
)
year: int = int(date_str[:4])
month_padded: str = date_str[4:6]
day_padded: str = date_str[6:8]
month: int = int(month_padded)
day: int = int(day_padded)
year_truncated: int = int(str(year)[-2:])
day_of_year: int = sum(_days_in_month[:month]) + day
total_days_in_month: int = _days_in_month[month]
total_days_in_year: int = 365
if year % 4 == 0:
total_days_in_year += 1
if month == 2:
total_days_in_month += 1
if month > 2:
day_of_year += 1
day_of_year_reversed: int = total_days_in_year + 1 - day_of_year
month_reversed: int = 13 - month
day_reversed: int = total_days_in_month + 1 - day
return Map(
{
String("date"): yyyymmdd,
String("date_standardized"): String(f"{year}-{month_padded}-{day_padded}"),
String("year"): Integer(year),
String("month"): Integer(month),
String("day"): Integer(day),
String("year_truncated"): String(year_truncated),
String("month_padded"): String(month_padded),
String("day_padded"): String(day_padded),
String("year_truncated_reversed"): Integer(100 - year_truncated),
String("month_reversed"): Integer(month_reversed),
String("month_reversed_padded"): String(_pad(month_reversed, width=2)),
String("day_reversed"): Integer(day_reversed),
String("day_reversed_padded"): String(_pad(day_reversed, width=2)),
String("day_of_year"): Integer(day_of_year),
String("day_of_year_padded"): String(_pad(day_of_year, width=3)),
String("day_of_year_reversed"): Integer(day_of_year_reversed),
String("day_of_year_reversed_padded"): String(_pad(day_of_year_reversed, width=3)),
}
)
@staticmethod
def register():
"""
Register Custom functions once and only once
"""
if not Functions.is_built_in("sanitize"):
Functions.register_function(CustomFunctions.legacy_bracket_safety)
Functions.register_function(CustomFunctions.truncate_filepath_if_too_long)
Functions.register_function(CustomFunctions.to_native_filepath)
Functions.register_function(CustomFunctions.sanitize)
Functions.register_function(CustomFunctions.sanitize_plex_episode)
Functions.register_function(CustomFunctions.to_date_metadata)

37
src/ytdl_sub/entries/script/function_scripts.py Normal file
View file

@ -0,0 +1,37 @@
from typing import Dict
from ytdl_sub.entries.script.variable_definitions import VARIABLES
from ytdl_sub.entries.script.variable_definitions import VariableDefinitions
v: VariableDefinitions = VARIABLES
CUSTOM_FUNCTION_SCRIPTS: Dict[str, str] = {
"%extract_field_from_metadata_array_getter": """{
%map_get( %map(%array_at($0, 0)), %array_at($0, 1) )
}""",
"%extract_field_from_metadata_array": """{
%if(
%bool($0),
%array_extend(
%array_apply(
%array_product(
%array($0),
[ %string($1) ]
),
%extract_field_from_metadata_array_getter
)
),
[]
)
}""",
"%extract_field_from_siblings": f"""{{
%if(
%bool({v.sibling_metadata.variable_name}),
%extract_field_from_metadata_array(
{v.sibling_metadata.variable_name},
$0
),
[]
)
}}""",
}

991
src/ytdl_sub/entries/script/variable_definitions.py Normal file
View file

@ -0,0 +1,991 @@
from dataclasses import dataclass
# This file contains mixins to a BaseEntry subclass. Ignore pylint's "no kwargs member" suggestion
# pylint: disable=no-member
# pylint: disable=too-many-public-methods
@dataclass(frozen=True)
class Variable:
variable_name: str
@dataclass(frozen=True)
class InternalVariable(Variable):
pass
@dataclass(frozen=True)
class Metadata(Variable):
pass
@dataclass(frozen=True)
class MetadataVariable(Variable):
metadata_key: str
@dataclass(frozen=True)
class RelativeMetadata(MetadataVariable, Metadata):
pass
@dataclass(frozen=True)
class SiblingMetadata(MetadataVariable):
pass
class VariableDefinitions:
@property
def entry_metadata(self) -> Metadata:
"""
The entry's info.json
"""
return Metadata("entry_metadata")
@property
def playlist_metadata(self) -> RelativeMetadata:
"""
Metadata from the playlist (i.e. the parent metadata, like playlist -> entry)
"""
return RelativeMetadata("playlist_metadata", metadata_key="playlist_metadata")
@property
def source_metadata(self) -> RelativeMetadata:
"""
Metadata from the source (i.e. the grandparent metadata, like channel -> playlist -> entry)
"""
return RelativeMetadata("source_metadata", metadata_key="source_metadata")
@property
def sibling_metadata(self) -> SiblingMetadata:
"""
Metadata from any sibling entries that reside in the same playlist as this entry.
"""
return SiblingMetadata("sibling_metadata", metadata_key="sibling_metadata")
@property
def uid(self) -> MetadataVariable:
"""
The entry's unique ID
"""
return MetadataVariable(metadata_key="id", variable_name="uid")
@property
def duration(self) -> MetadataVariable:
"""
The duration of the entry in seconds
"""
return MetadataVariable("duration", metadata_key="duration")
@property
def uid_sanitized_plex(self) -> Variable:
"""
The sanitized uid with additional sanitizing for Plex. Replaces numbers with
fixed-width numbers so Plex does not recognize them as season or episode numbers.
"""
return Variable("uid_sanitized_plex")
@property
def ie_key(self) -> MetadataVariable:
"""
The ie_key, used in legacy yt-dlp things as the 'info-extractor key'
"""
return MetadataVariable(metadata_key="ie_key", variable_name="ie_key")
@property
def extractor_key(self) -> MetadataVariable:
"""
The yt-dlp extractor key
"""
return MetadataVariable(metadata_key="extractor_key", variable_name="extractor_key")
@property
def extractor(self) -> MetadataVariable:
"""
The yt-dlp extractor name
"""
return MetadataVariable(variable_name="extractor", metadata_key="extractor")
@property
def epoch(self) -> MetadataVariable:
"""
The unix epoch of when the metadata was scraped by yt-dlp.
"""
return MetadataVariable(metadata_key="epoch", variable_name="epoch")
@property
def epoch_date(self) -> Variable:
"""
The epoch's date, in YYYYMMDD format.
"""
return Variable("epoch_date")
@property
def epoch_hour(self) -> Variable:
"""
The epoch's hour
"""
return Variable("epoch_hour")
@property
def title(self) -> MetadataVariable:
"""
The title of the entry. If a title does not exist, returns its unique ID.
"""
return MetadataVariable(variable_name="title", metadata_key="title")
@property
def title_sanitized_plex(self) -> Variable:
"""
The sanitized title with additional sanitizing for Plex. It replaces numbers with
fixed-width numbers so Plex does not recognize them as season or episode numbers.
"""
return Variable("title_sanitized_plex")
@property
def webpage_url(self) -> MetadataVariable:
"""
Returns
-------
str
The url to the webpage.
"""
return MetadataVariable(metadata_key="webpage_url", variable_name="webpage_url")
@property
def info_json_ext(self) -> Variable:
"""
Returns
-------
str
The "info.json" extension
"""
return Variable("info_json_ext")
@property
def description(self) -> MetadataVariable:
"""
Returns
-------
str
The description if it exists. Otherwise, returns an emtpy string.
"""
return MetadataVariable(variable_name="description", metadata_key="description")
@property
def uploader_id(self) -> MetadataVariable:
"""
Returns
-------
str
The uploader id if it exists, otherwise return the unique ID.
"""
return MetadataVariable(variable_name="uploader_id", metadata_key="uploader_id")
@property
def uploader(self) -> MetadataVariable:
"""
Returns
-------
str
The uploader if it exists, otherwise return the uploader ID.
"""
return MetadataVariable(variable_name="uploader", metadata_key="uploader")
@property
def uploader_url(self) -> MetadataVariable:
"""
Returns
-------
str
The uploader url if it exists, otherwise returns the webpage_url.
"""
return MetadataVariable("uploader_url", metadata_key="uploader_url")
@property
def source_title(self) -> MetadataVariable:
"""
Returns
-------
str
Name of the source (i.e. channel with multiple playlists) if it exists, otherwise
returns its playlist_title.
"""
return MetadataVariable("source_title", metadata_key=self.title.metadata_key)
@property
def source_uid(self) -> MetadataVariable:
"""
Returns
-------
str
The source unique id if it exists, otherwise returns the playlist unique ID.
"""
return MetadataVariable("source_uid", metadata_key=self.uid.metadata_key)
@property
def source_index(self) -> MetadataVariable:
"""
Returns
-------
int
Source index if it exists, otherwise returns ``1``.
It is recommended to not use this unless you know the source will never add new content
(it is easy for this value to change).
"""
return MetadataVariable("source_index", metadata_key=self.playlist_index.metadata_key)
@property
def source_index_padded(self) -> Variable:
"""
Returns
-------
int
The source index, padded.
"""
return Variable("source_index_padded")
@property
def source_count(self) -> MetadataVariable:
"""
Returns
-------
int
The source count if it exists, otherwise returns the playlist count.
"""
return MetadataVariable("source_count", metadata_key=self.playlist_count.metadata_key)
@property
def source_webpage_url(self) -> MetadataVariable:
"""
Returns
-------
str
The source webpage url if it exists, otherwise returns the playlist webpage url.
"""
return MetadataVariable("source_webpage_url", metadata_key=self.webpage_url.metadata_key)
@property
def source_description(self) -> MetadataVariable:
"""
Returns
-------
str
The source description if it exists, otherwise returns the playlist description.
"""
return MetadataVariable("source_description", metadata_key=self.description.metadata_key)
@property
def playlist_uid(self) -> MetadataVariable:
"""
Returns
-------
str
The playlist unique ID if it exists, otherwise return the entry unique ID.
"""
return MetadataVariable(variable_name="playlist_uid", metadata_key="playlist_id")
@property
def playlist_title(self) -> MetadataVariable:
"""
Returns
-------
str
Name of its parent playlist/channel if it exists, otherwise returns its title.
"""
return MetadataVariable(variable_name="playlist_title", metadata_key="playlist_title")
@property
def playlist_index(self) -> MetadataVariable:
"""
Returns
-------
int
Playlist index if it exists, otherwise returns ``1``.
Note that for channels/playlists, any change (i.e. adding or removing a video) will make
this value change. Use with caution.
"""
return MetadataVariable(metadata_key="playlist_index", variable_name="playlist_index")
@property
def playlist_index_reversed(self) -> Variable:
"""
Returns
-------
int
Playlist index reversed via ``playlist_count - playlist_index + 1``
"""
return Variable("playlist_index_reversed")
@property
def playlist_index_padded(self) -> Variable:
"""
Returns
-------
str
playlist_index padded two digits
"""
return Variable("playlist_index_padded")
@property
def playlist_index_reversed_padded(self) -> Variable:
"""
Returns
-------
str
playlist_index_reversed padded two digits
"""
return Variable("playlist_index_reversed_padded")
@property
def playlist_index_padded6(self) -> Variable:
"""
Returns
-------
str
playlist_index padded six digits.
"""
return Variable("playlist_index_padded6")
@property
def playlist_index_reversed_padded6(self) -> Variable:
"""
Returns
-------
str
playlist_index_reversed padded six digits.
"""
return Variable("playlist_index_reversed_padded6")
@property
def playlist_count(self) -> MetadataVariable:
"""
Returns
-------
int
Playlist count if it exists, otherwise returns ``1``.
Note that for channels/playlists, any change (i.e. adding or removing a video) will make
this value change. Use with caution.
"""
return MetadataVariable(variable_name="playlist_count", metadata_key="playlist_count")
@property
def playlist_description(self) -> MetadataVariable:
"""
Returns
-------
str
The playlist description if it exists, otherwise returns the entry's description.
"""
return MetadataVariable(
variable_name="playlist_description", metadata_key=self.description.metadata_key
)
@property
def playlist_webpage_url(self) -> MetadataVariable:
"""
Returns
-------
str
The playlist webpage url if it exists. Otherwise, returns the entry webpage url.
"""
return MetadataVariable(
variable_name="playlist_webpage_url", metadata_key=self.webpage_url.metadata_key
)
@property
def playlist_max_upload_date(self) -> Variable:
"""
Returns
-------
Max upload_date for all entries in this entry's playlist if it exists, otherwise returns
``upload_date``
"""
return Variable("playlist_max_upload_date")
@property
def playlist_max_upload_year(self) -> Variable:
"""
Returns
-------
int
Max upload_year for all entries in this entry's playlist if it exists, otherwise returns
``upload_year``
"""
# override in EntryParent
return Variable("playlist_max_upload_year")
@property
def playlist_max_upload_year_truncated(self) -> Variable:
"""
Returns
-------
int
The max playlist truncated upload year for all entries in this entry's playlist if it
exists, otherwise returns ``upload_year_truncated``.
"""
return Variable("playlist_max_upload_year_truncated")
@property
def playlist_uploader_id(self) -> MetadataVariable:
"""
Returns
-------
str
The playlist uploader id if it exists, otherwise returns the entry uploader ID.
"""
return MetadataVariable("playlist_uploader_id", metadata_key="playlist_uploader_id")
@property
def playlist_uploader(self) -> MetadataVariable:
"""
Returns
-------
str
The playlist uploader if it exists, otherwise return the entry uploader.
"""
return MetadataVariable("playlist_uploader", metadata_key=self.uploader.metadata_key)
@property
def playlist_uploader_url(self) -> MetadataVariable:
"""
Returns
-------
str
The playlist uploader url if it exists, otherwise returns the playlist webpage_url.
"""
return MetadataVariable(
"playlist_uploader_url", metadata_key=self.uploader_url.metadata_key
)
@property
def source_uploader_id(self) -> MetadataVariable:
"""
Returns
-------
str
The source uploader id if it exists, otherwise returns the playlist_uploader_id
"""
return MetadataVariable("source_uploader_id", metadata_key=self.uploader_id.metadata_key)
@property
def source_uploader(self) -> MetadataVariable:
"""
Returns
-------
str
The source uploader if it exists, otherwise return the playlist_uploader
"""
return MetadataVariable("source_uploader", metadata_key=self.uploader.metadata_key)
@property
def source_uploader_url(self) -> MetadataVariable:
"""
Returns
-------
str
The source uploader url if it exists, otherwise returns the source webpage_url.
"""
return MetadataVariable("source_uploader_url", metadata_key=self.uploader_url.metadata_key)
@property
def creator(self) -> MetadataVariable:
"""
Returns
-------
str
The creator name if it exists, otherwise returns the channel.
"""
return MetadataVariable(variable_name="creator", metadata_key="creator")
@property
def channel(self) -> MetadataVariable:
"""
Returns
-------
str
The channel name if it exists, otherwise returns the uploader.
"""
return MetadataVariable(variable_name="channel", metadata_key="channel")
@property
def channel_id(self) -> MetadataVariable:
"""
Returns
-------
str
The channel id if it exists, otherwise returns the entry uploader ID.
"""
return MetadataVariable(variable_name="channel_id", metadata_key="channel_id")
@property
def ext(self) -> MetadataVariable:
"""
Returns
-------
str
The downloaded entry's file extension
"""
return MetadataVariable(variable_name="ext", metadata_key="ext")
@property
def thumbnail_ext(self) -> Variable:
"""
Returns
-------
str
The download entry's thumbnail extension. Will always return 'jpg'. Until there is a
need to support other image types, we always convert to jpg.
"""
return Variable("thumbnail_ext")
@property
def comments(self) -> MetadataVariable:
"""
Comments if they are requested
"""
return MetadataVariable("comments", "comments")
@property
def chapters(self) -> MetadataVariable:
"""
Chapters if they exist
"""
return MetadataVariable("chapters", "chapters")
@property
def sponsorblock_chapters(self) -> MetadataVariable:
"""
Sponsorblock Chapters if they are requested and exist
"""
return MetadataVariable("sponsorblock_chapters", "sponsorblock_chapters")
@property
def requested_subtitles(self) -> MetadataVariable:
"""
Subtitles if they are requested and exist
"""
return MetadataVariable("requested_subtitles", "requested_subtitles")
@property
def ytdl_sub_input_url(self) -> Variable:
"""
The input URL used in ytdl-sub to create this entry.
"""
return Variable("ytdl_sub_input_url")
@property
def download_index(self) -> Variable:
"""
Returns
-------
int
The i'th entry downloaded. NOTE that this is fetched dynamically from the download
archive.
"""
return Variable(variable_name="download_index")
@property
def download_index_padded6(self) -> Variable:
"""
Returns
-------
str
The download_index padded six digits
"""
return Variable("download_index_padded6")
@property
def upload_date_index(self) -> Variable:
"""
Returns
-------
int
The i'th entry downloaded with this upload date.
"""
return Variable(variable_name="upload_date_index")
@property
def upload_date_index_padded(self) -> Variable:
"""
Returns
-------
int
The upload_date_index padded two digits
"""
return Variable("upload_date_index_padded")
@property
def upload_date_index_reversed(self) -> Variable:
"""
Returns
-------
int
100 - upload_date_index
"""
return Variable("upload_date_index_reversed")
@property
def upload_date_index_reversed_padded(self) -> Variable:
"""
Returns
-------
int
The upload_date_index padded two digits
"""
return Variable("upload_date_index_reversed_padded")
@property
def upload_date(self) -> MetadataVariable:
"""
Returns
-------
str
The entry’s uploaded date, in YYYYMMDD format. If not present, return today’s date.
"""
return MetadataVariable(variable_name="upload_date", metadata_key="upload_date")
@property
def upload_year(self) -> Variable:
"""
Returns
-------
int
The entry's upload year
"""
return Variable("upload_year")
@property
def upload_year_truncated(self) -> Variable:
"""
Returns
-------
int
The last two digits of the upload year, i.e. 22 in 2022
"""
return Variable("upload_year_truncated")
@property
def upload_year_truncated_reversed(self) -> Variable:
"""
Returns
-------
int
The upload year truncated, but reversed using ``100 - {upload_year_truncated}``, i.e.
2022 returns ``100 - 22`` = ``78``
"""
return Variable("upload_year_truncated_reversed")
@property
def upload_month_reversed(self) -> Variable:
"""
Returns
-------
int
The upload month, but reversed using ``13 - {upload_month}``, i.e. March returns ``10``
"""
return Variable("upload_month_reversed")
@property
def upload_month_reversed_padded(self) -> Variable:
"""
Returns
-------
str
The reversed upload month, but padded. i.e. November returns "02"
"""
return Variable("upload_month_reversed_padded")
@property
def upload_month_padded(self) -> Variable:
"""
Returns
-------
str
The entry's upload month padded to two digits, i.e. March returns "03"
"""
return Variable("upload_month_padded")
@property
def upload_day_padded(self) -> Variable:
"""
Returns
-------
str
The entry's upload day padded to two digits, i.e. the fifth returns "05"
"""
return Variable("upload_day_padded")
@property
def upload_month(self) -> Variable:
"""
Returns
-------
int
The upload month as an integer (no padding).
"""
return Variable("upload_month")
@property
def upload_day(self) -> Variable:
"""
Returns
-------
int
The upload day as an integer (no padding).
"""
return Variable("upload_day")
@property
def upload_day_reversed(self) -> Variable:
"""
Returns
-------
int
The upload day, but reversed using ``{total_days_in_month} + 1 - {upload_day}``,
i.e. August 8th would have upload_day_reversed of ``31 + 1 - 8`` = ``24``
"""
return Variable("upload_day_reversed")
@property
def upload_day_reversed_padded(self) -> Variable:
"""
Returns
-------
str
The reversed upload day, but padded. i.e. August 30th returns "02".
"""
return Variable("upload_day_reversed_padded")
@property
def upload_day_of_year(self) -> Variable:
"""
Returns
-------
int
The day of the year, i.e. February 1st returns ``32``
"""
return Variable("upload_day_of_year")
@property
def upload_day_of_year_padded(self) -> Variable:
"""
Returns
-------
str
The upload day of year, but padded i.e. February 1st returns "032"
"""
return Variable("upload_day_of_year_padded")
@property
def upload_day_of_year_reversed(self) -> Variable:
"""
Returns
-------
int
The upload day, but reversed using ``{total_days_in_year} + 1 - {upload_day}``,
i.e. February 2nd would have upload_day_of_year_reversed of ``365 + 1 - 32`` = ``334``
"""
return Variable("upload_day_of_year_reversed")
@property
def upload_day_of_year_reversed_padded(self) -> Variable:
"""
Returns
-------
str
The reversed upload day of year, but padded i.e. December 31st returns "001"
"""
return Variable("upload_day_of_year_reversed_padded")
@property
def upload_date_standardized(self) -> Variable:
"""
Returns
-------
str
The uploaded date formatted as YYYY-MM-DD
"""
return Variable("upload_date_standardized")
@property
def release_date(self) -> MetadataVariable:
"""
Returns
-------
str
The entry’s release date, in YYYYMMDD format. If not present, return the upload date.
"""
return MetadataVariable(variable_name="release_date", metadata_key="release_date")
@property
def release_year(self) -> Variable:
"""
Returns
-------
int
The entry's release year
"""
return Variable("release_year")
@property
def release_year_truncated(self) -> Variable:
"""
Returns
-------
int
The last two digits of the release year, i.e. 22 in 2022
"""
return Variable("release_year_truncated")
@property
def release_year_truncated_reversed(self) -> Variable:
"""
Returns
-------
int
The release year truncated, but reversed using ``100 - {release_year_truncated}``, i.e.
2022 returns ``100 - 22`` = ``78``
"""
return Variable("release_year_truncated_reversed")
@property
def release_month_reversed(self) -> Variable:
"""
Returns
-------
int
The release month, but reversed
using ``13 - {release_month}``, i.e. March returns ``10``
"""
return Variable("release_month_reversed")
@property
def release_month_reversed_padded(self) -> Variable:
"""
Returns
-------
str
The reversed release month, but padded. i.e. November returns "02"
"""
return Variable("release_month_reversed_padded")
@property
def release_month_padded(self) -> Variable:
"""
Returns
-------
str
The entry's release month padded to two digits, i.e. March returns "03"
"""
return Variable("release_month_padded")
@property
def release_day_padded(self) -> Variable:
"""
Returns
-------
str
The entry's release day padded to two digits, i.e. the fifth returns "05"
"""
return Variable("release_day_padded")
@property
def release_month(self) -> Variable:
"""
Returns
-------
int
The release month as an integer (no padding).
"""
return Variable("release_month")
@property
def release_day(self) -> Variable:
"""
Returns
-------
int
The release day as an integer (no padding).
"""
return Variable("release_day")
@property
def release_day_reversed(self) -> Variable:
"""
Returns
-------
int
The release day, but reversed using ``{total_days_in_month} + 1 - {release_day}``,
i.e. August 8th would have release_day_reversed of ``31 + 1 - 8`` = ``24``
"""
return Variable("release_day_reversed")
@property
def release_day_reversed_padded(self) -> Variable:
"""
Returns
-------
str
The reversed release day, but padded. i.e. August 30th returns "02".
"""
return Variable("release_day_reversed_padded")
@property
def release_day_of_year(self) -> Variable:
"""
Returns
-------
int
The day of the year, i.e. February 1st returns ``32``
"""
return Variable("release_day_of_year")
@property
def release_day_of_year_padded(self) -> Variable:
"""
Returns
-------
str
The release day of year, but padded i.e. February 1st returns "032"
"""
return Variable("release_day_of_year_padded")
@property
def release_day_of_year_reversed(self) -> Variable:
"""
Returns
-------
int
The release day, but reversed using ``{total_days_in_year} + 1 - {release_day}``,
i.e. February 2nd would have release_day_of_year_reversed of ``365 + 1 - 32`` = ``334``
"""
return Variable("release_day_of_year_reversed")
@property
def release_day_of_year_reversed_padded(self) -> Variable:
"""
Returns
-------
str
The reversed release day of year, but padded i.e. December 31st returns "001"
"""
return Variable("release_day_of_year_reversed_padded")
@property
def release_date_standardized(self) -> Variable:
"""
Returns
-------
str
The release date formatted as YYYY-MM-DD
"""
return Variable("release_date_standardized")
# Singleton to use externally
VARIABLES: VariableDefinitions = VariableDefinitions()

296
src/ytdl_sub/entries/script/variable_scripts.py Normal file
View file

@ -0,0 +1,296 @@
from typing import Dict
from typing import List
from typing import Optional
from typing import Set
import mergedeep
from ytdl_sub.entries.script.custom_functions import CustomFunctions
from ytdl_sub.entries.script.variable_definitions import VARIABLES
from ytdl_sub.entries.script.variable_definitions import Metadata
from ytdl_sub.entries.script.variable_definitions import MetadataVariable
from ytdl_sub.entries.script.variable_definitions import Variable
from ytdl_sub.entries.script.variable_definitions import VariableDefinitions
###############################################################################################
# Helpers
v: VariableDefinitions = VARIABLES
def _pad_int(key: Variable, pad: int) -> str:
return f"{{%pad_zero({key.variable_name}, {pad})}}"
def _sanitized_plex(key: Variable) -> str:
return f"{{%sanitize_plex_episode({key.variable_name})}}"
def _date_metadata(date_key: Variable, metadata_key: str) -> str:
return f"{{%map_get(%to_date_metadata({date_key.variable_name}), '{metadata_key}')}}"
###############################################################################################
# Metadata Getters
def _get(
cast: str,
metadata: Metadata,
key: MetadataVariable,
default: Optional[Variable | str | int | Dict | List],
) -> str:
if default is None:
# TODO: assert with good error message if key DNE
out = f"%map_get({metadata.variable_name}, '{key.metadata_key}')"
elif isinstance(default, Variable):
args = f"{metadata.variable_name}, '{key.metadata_key}', {default.variable_name}"
out = f"%map_get_non_empty({args})"
elif isinstance(default, str):
out = f"%map_get_non_empty({metadata.variable_name}, '{key.metadata_key}', '{default}')"
elif isinstance(default, dict):
out = f"%map_get_non_empty({metadata.variable_name}, '{key.metadata_key}', {{}})"
elif isinstance(default, list):
out = f"%map_get_non_empty({metadata.variable_name}, '{key.metadata_key}', [])"
else:
out = f"%map_get_non_empty({metadata.variable_name}, '{key.metadata_key}', {default})"
return f"{{ %legacy_bracket_safety(%{cast}({out})) }}"
###############################################################################################
# Entry Getters
def _entry_get_str(key: MetadataVariable, default: Optional[Variable | str] = None) -> str:
return _get("string", metadata=v.entry_metadata, key=key, default=default)
def _entry_get_int(key: MetadataVariable, default: Optional[Variable | int] = None) -> str:
return _get("int", metadata=v.entry_metadata, key=key, default=default)
def _entry_get_map(key: MetadataVariable, default: Optional[Variable | Dict] = None):
return _get("map", metadata=v.entry_metadata, key=key, default=default)
def _entry_get_array(key: MetadataVariable, default: Optional[Variable | List] = None):
return _get("array", metadata=v.entry_metadata, key=key, default=default)
###############################################################################################
# Playlist Getters
def _playlist_get_str(key: MetadataVariable, default: Optional[Variable | str] = None) -> str:
return _get("string", metadata=v.playlist_metadata, key=key, default=default)
def _playlist_get_int(key: MetadataVariable, default: Optional[Variable | int] = None) -> str:
return _get("int", metadata=v.playlist_metadata, key=key, default=default)
###############################################################################################
# Source Getters
def _source_get_str(key: MetadataVariable, default: Optional[Variable | str] = None) -> str:
return _get("string", metadata=v.source_metadata, key=key, default=default)
###############################################################################################
# Scripts
ENTRY_EMPTY_METADATA: Dict[Variable, str] = {v.entry_metadata: "{ {} }"}
ENTRY_HARDCODED_VARIABLES: Dict[Variable, str] = {
v.info_json_ext: "info.json",
v.thumbnail_ext: "jpg",
}
ENTRY_RELATIVE_VARIABLES: Dict[MetadataVariable, str] = {
v.playlist_metadata: _entry_get_map(v.playlist_metadata, {}),
v.source_metadata: _entry_get_map(v.source_metadata, {}),
v.sibling_metadata: _entry_get_array(v.sibling_metadata, []),
}
ENTRY_REQUIRED_VARIABLES: Dict[MetadataVariable, str] = {
v.uid: _entry_get_str(v.uid),
v.extractor_key: _entry_get_str(v.extractor_key),
v.epoch: _entry_get_int(v.epoch),
v.webpage_url: _entry_get_str(v.webpage_url),
v.ext: _entry_get_str(v.ext),
}
ENTRY_DEFAULT_VARIABLES: Dict[MetadataVariable, str] = {
v.title: _entry_get_str(v.title, v.uid),
v.extractor: _entry_get_str(v.extractor, v.extractor_key),
v.description: _entry_get_str(v.description, ""),
v.ie_key: _entry_get_str(v.ie_key, v.extractor_key),
v.uploader_id: _entry_get_str(v.uploader_id, v.uid),
v.uploader: _entry_get_str(v.uploader, v.uploader_id),
v.uploader_url: _entry_get_str(v.uploader_url, v.webpage_url),
v.upload_date: _entry_get_str(v.upload_date, v.epoch_date),
v.release_date: _entry_get_str(v.release_date, v.upload_date),
v.channel: _entry_get_str(v.channel, v.uploader),
v.creator: _entry_get_str(v.creator, v.channel),
v.channel_id: _entry_get_str(v.channel_id, v.uploader_id),
v.duration: _entry_get_int(v.duration, 0),
v.playlist_index: _entry_get_int(v.playlist_index, 1),
v.playlist_count: _entry_get_int(v.playlist_count, 1),
v.playlist_uid: _entry_get_str(v.playlist_uid, v.uid),
v.playlist_title: _entry_get_str(v.playlist_title, v.title),
v.playlist_uploader_id: _entry_get_str(v.playlist_uploader_id, v.uploader_id),
}
# MARK AS UNRESOLVABLE UNTIL THEY ARE ADDED IN THE DOWNLOADER
DOWNLOADER_INJECTED_VARIABLES: Dict[Variable, str] = {
v.download_index: "{%int(1)}",
v.upload_date_index: "{%int(1)}",
v.comments: "{ [] }",
v.requested_subtitles: "{ {} }",
v.chapters: "{ [] }",
v.sponsorblock_chapters: "{ [] }",
v.ytdl_sub_input_url: f"{{{v.source_webpage_url.variable_name}}}",
}
ENTRY_DERIVED_VARIABLES: Dict[Variable, str] = {
v.uid_sanitized_plex: _sanitized_plex(v.uid),
v.title_sanitized_plex: _sanitized_plex(v.title),
v.epoch_date: f"{{%datetime_strftime({v.epoch.variable_name}, '%Y%m%d')}}",
v.epoch_hour: f"{{%datetime_strftime({v.epoch.variable_name}, '%H')}}",
v.download_index_padded6: _pad_int(v.download_index, 6),
v.upload_date_index_padded: _pad_int(v.upload_date_index, 2),
v.upload_date_index_reversed: f"{{%sub(100, {v.upload_date_index.variable_name})}}",
v.upload_date_index_reversed_padded: _pad_int(v.upload_date_index_reversed, 2),
v.playlist_index_reversed: (
f"{{%sub({v.playlist_count.variable_name}, {v.playlist_index.variable_name}, -1)}}"
),
v.playlist_index_padded: _pad_int(v.playlist_index, 2),
v.playlist_index_reversed_padded: _pad_int(v.playlist_index_reversed, 2),
v.playlist_index_padded6: _pad_int(v.playlist_index, 6),
v.playlist_index_reversed_padded6: _pad_int(v.playlist_index_reversed, 6),
}
ENTRY_UPLOAD_DATE_VARIABLES: Dict[Variable, str] = {
v.upload_year: _date_metadata(v.upload_date, "year"),
v.upload_year_truncated: _date_metadata(v.upload_date, "year_truncated"),
v.upload_year_truncated_reversed: _date_metadata(v.upload_date, "year_truncated_reversed"),
v.upload_month_reversed: _date_metadata(v.upload_date, "month_reversed"),
v.upload_month_reversed_padded: _date_metadata(v.upload_date, "month_reversed_padded"),
v.upload_month_padded: _date_metadata(v.upload_date, "month_padded"),
v.upload_day_padded: _date_metadata(v.upload_date, "day_padded"),
v.upload_month: _date_metadata(v.upload_date, "month"),
v.upload_day: _date_metadata(v.upload_date, "day"),
v.upload_day_reversed: _date_metadata(v.upload_date, "day_reversed"),
v.upload_day_reversed_padded: _date_metadata(v.upload_date, "day_reversed_padded"),
v.upload_day_of_year: _date_metadata(v.upload_date, "day_of_year"),
v.upload_day_of_year_padded: _date_metadata(v.upload_date, "day_of_year_padded"),
v.upload_day_of_year_reversed: _date_metadata(v.upload_date, "day_of_year_reversed"),
v.upload_day_of_year_reversed_padded: _date_metadata(
v.upload_date, "day_of_year_reversed_padded"
),
v.upload_date_standardized: _date_metadata(v.upload_date, "date_standardized"),
}
ENTRY_RELEASE_DATE_VARIABLES: Dict[Variable, str] = {
v.release_year: _date_metadata(v.release_date, "year"),
v.release_year_truncated: _date_metadata(v.release_date, "year_truncated"),
v.release_year_truncated_reversed: _date_metadata(v.release_date, "year_truncated_reversed"),
v.release_month_reversed: _date_metadata(v.release_date, "month_reversed"),
v.release_month_reversed_padded: _date_metadata(v.release_date, "month_reversed_padded"),
v.release_month_padded: _date_metadata(v.release_date, "month_padded"),
v.release_day_padded: _date_metadata(v.release_date, "day_padded"),
v.release_month: _date_metadata(v.release_date, "month"),
v.release_day: _date_metadata(v.release_date, "day"),
v.release_day_reversed: _date_metadata(v.release_date, "day_reversed"),
v.release_day_reversed_padded: _date_metadata(v.release_date, "day_reversed_padded"),
v.release_day_of_year: _date_metadata(v.release_date, "day_of_year"),
v.release_day_of_year_padded: _date_metadata(v.release_date, "day_of_year_padded"),
v.release_day_of_year_reversed: _date_metadata(v.release_date, "day_of_year_reversed"),
v.release_day_of_year_reversed_padded: _date_metadata(
v.release_date, "day_of_year_reversed_padded"
),
v.release_date_standardized: _date_metadata(v.release_date, "date_standardized"),
}
PLAYLIST_VARIABLES: Dict[Variable, str] = {
v.playlist_webpage_url: _playlist_get_str(v.playlist_webpage_url, v.webpage_url),
v.playlist_description: _playlist_get_str(v.playlist_description, v.description),
v.playlist_uploader: _playlist_get_str(v.playlist_uploader, v.uploader),
v.playlist_uploader_url: _playlist_get_str(v.playlist_uploader_url, v.playlist_webpage_url),
v.source_index: _playlist_get_int(v.source_index, 1),
v.source_count: _playlist_get_int(v.source_count, 1),
}
SOURCE_VARIABLES: Dict[Variable, str] = {
v.source_uid: _source_get_str(v.source_uid, v.playlist_uid),
v.source_title: _source_get_str(v.source_title, v.playlist_title),
v.source_webpage_url: _source_get_str(v.source_webpage_url, v.playlist_webpage_url),
v.source_description: _source_get_str(v.source_description, v.playlist_description),
v.source_uploader_id: _source_get_str(v.source_uploader_id, v.playlist_uploader_id),
v.source_uploader: _source_get_str(v.source_uploader, v.playlist_uploader),
v.source_uploader_url: _source_get_str(v.source_uploader_url, v.source_webpage_url),
}
SOURCE_DERIVED_VARIABLES: Dict[Variable, str] = {
v.source_index_padded: _pad_int(v.source_index, 2),
}
SIBLING_VARIABLES: Dict[Variable, str] = {
v.playlist_max_upload_date: f"""{{
%array_reduce(
%if_passthrough(
%extract_field_from_siblings('{v.upload_date.variable_name}'),
[{v.upload_date.variable_name}]
),
%max
)
}}"""
}
SIBLING_DERIVED_VARIABLES: Dict[Variable, str] = {
v.playlist_max_upload_year: _date_metadata(v.playlist_max_upload_date, "year"),
v.playlist_max_upload_year_truncated: _date_metadata(
v.playlist_max_upload_date, "year_truncated"
),
}
_VARIABLE_SCRIPTS: Dict[Variable, str] = {}
mergedeep.merge(
_VARIABLE_SCRIPTS,
ENTRY_EMPTY_METADATA,
ENTRY_HARDCODED_VARIABLES,
ENTRY_RELATIVE_VARIABLES,
ENTRY_REQUIRED_VARIABLES,
ENTRY_DEFAULT_VARIABLES,
DOWNLOADER_INJECTED_VARIABLES,
ENTRY_DERIVED_VARIABLES,
ENTRY_UPLOAD_DATE_VARIABLES,
ENTRY_RELEASE_DATE_VARIABLES,
SIBLING_VARIABLES,
SIBLING_DERIVED_VARIABLES,
PLAYLIST_VARIABLES,
SOURCE_VARIABLES,
SOURCE_DERIVED_VARIABLES,
)
VARIABLE_SCRIPTS: Dict[str, str] = {
var.variable_name: script for var, script in _VARIABLE_SCRIPTS.items()
}
def _keys(*variables: Dict[Variable, str]) -> Set[str]:
keys: Set[str] = set()
for variable_set in variables:
keys.update(set(var.variable_name for var in variable_set.keys()))
return keys
UNRESOLVED_VARIABLES: Set[str] = _keys(
ENTRY_EMPTY_METADATA,
DOWNLOADER_INJECTED_VARIABLES,
)
CustomFunctions.register()

857
src/ytdl_sub/entries/variables/entry_variables.py
View file

@ -1,857 +0,0 @@
from datetime import datetime
from typing import Union
from yt_dlp.utils import sanitize_filename
from ytdl_sub.entries.base_entry import BaseEntry
from ytdl_sub.entries.base_entry import BaseEntryVariables
from ytdl_sub.entries.variables.kwargs import CHANNEL
from ytdl_sub.entries.variables.kwargs import CHANNEL_ID
from ytdl_sub.entries.variables.kwargs import CREATOR
from ytdl_sub.entries.variables.kwargs import DOWNLOAD_INDEX
from ytdl_sub.entries.variables.kwargs import EXT
from ytdl_sub.entries.variables.kwargs import PLAYLIST_COUNT
from ytdl_sub.entries.variables.kwargs import PLAYLIST_DESCRIPTION
from ytdl_sub.entries.variables.kwargs import PLAYLIST_INDEX
from ytdl_sub.entries.variables.kwargs import PLAYLIST_MAX_UPLOAD_YEAR
from ytdl_sub.entries.variables.kwargs import PLAYLIST_MAX_UPLOAD_YEAR_TRUNCATED
from ytdl_sub.entries.variables.kwargs import PLAYLIST_TITLE
from ytdl_sub.entries.variables.kwargs import PLAYLIST_UID
from ytdl_sub.entries.variables.kwargs import PLAYLIST_UPLOADER
from ytdl_sub.entries.variables.kwargs import PLAYLIST_UPLOADER_ID
from ytdl_sub.entries.variables.kwargs import PLAYLIST_UPLOADER_URL
from ytdl_sub.entries.variables.kwargs import PLAYLIST_WEBPAGE_URL
from ytdl_sub.entries.variables.kwargs import RELEASE_DATE
from ytdl_sub.entries.variables.kwargs import SOURCE_COUNT
from ytdl_sub.entries.variables.kwargs import SOURCE_DESCRIPTION
from ytdl_sub.entries.variables.kwargs import SOURCE_INDEX
from ytdl_sub.entries.variables.kwargs import SOURCE_TITLE
from ytdl_sub.entries.variables.kwargs import SOURCE_UID
from ytdl_sub.entries.variables.kwargs import SOURCE_UPLOADER
from ytdl_sub.entries.variables.kwargs import SOURCE_UPLOADER_ID
from ytdl_sub.entries.variables.kwargs import SOURCE_UPLOADER_URL
from ytdl_sub.entries.variables.kwargs import SOURCE_WEBPAGE_URL
from ytdl_sub.entries.variables.kwargs import UPLOAD_DATE
from ytdl_sub.entries.variables.kwargs import UPLOAD_DATE_INDEX
# This file contains mixins to a BaseEntry subclass. Ignore pylint's "no kwargs member" suggestion
# pylint: disable=no-member
# pylint: disable=too-many-public-methods
def pad(num: int, width: int = 2):
"""Pad integers"""
return str(num).zfill(width)
_days_in_month = [0, 31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31]
Self = Union[BaseEntry, "EntryVariables"]
class EntryVariables(BaseEntryVariables):
@property
def source_title(self: Self) -> str:
"""
Returns
-------
str
Name of the source (i.e. channel with multiple playlists) if it exists, otherwise
returns its playlist_title.
"""
return self.kwargs_get(SOURCE_TITLE, self.playlist_title)
@property
def source_title_sanitized(self: Self) -> str:
"""
Returns
-------
str
The source title, sanitized
"""
return sanitize_filename(self.source_title)
@property
def source_uid(self: Self) -> str:
"""
Returns
-------
str
The source unique id if it exists, otherwise returns the playlist unique ID.
"""
return self.kwargs_get(SOURCE_UID, self.playlist_uid)
@property
def source_index(self: Self) -> int:
"""
Returns
-------
int
Source index if it exists, otherwise returns ``1``.
It is recommended to not use this unless you know the source will never add new content
(it is easy for this value to change).
"""
return self.kwargs_get(SOURCE_INDEX, self.playlist_index)
@property
def source_index_padded(self: Self) -> str:
"""
Returns
-------
int
The source index, padded.
"""
return pad(self.source_index, 2)
@property
def source_count(self: Self) -> int:
"""
Returns
-------
int
The source count if it exists, otherwise returns the playlist count.
"""
return self.kwargs_get(SOURCE_COUNT, self.playlist_count)
@property
def source_webpage_url(self: Self) -> str:
"""
Returns
-------
str
The source webpage url if it exists, otherwise returns the playlist webpage url.
"""
return self.kwargs_get(SOURCE_WEBPAGE_URL, self.playlist_webpage_url)
@property
def source_description(self: Self) -> str:
"""
Returns
-------
str
The source description if it exists, otherwise returns the playlist description.
"""
return self.kwargs_get(SOURCE_DESCRIPTION, self.playlist_description)
@property
def playlist_uid(self: Self) -> str:
"""
Returns
-------
str
The playlist unique ID if it exists, otherwise return the entry unique ID.
"""
return self.kwargs_get(PLAYLIST_UID, self.uid)
@property
def playlist_title(self: Self) -> str:
"""
Returns
-------
str
Name of its parent playlist/channel if it exists, otherwise returns its title.
"""
return self.kwargs_get(PLAYLIST_TITLE, self.title)
@property
def playlist_title_sanitized(self: Self) -> str:
"""
Returns
-------
str
The playlist name, sanitized
"""
return sanitize_filename(self.playlist_title)
@property
def playlist_index(self: Self) -> int:
"""
Returns
-------
int
Playlist index if it exists, otherwise returns ``1``.
Note that for channels/playlists, any change (i.e. adding or removing a video) will make
this value change. Use with caution.
"""
return self.kwargs_get(PLAYLIST_INDEX, 1)
@property
def playlist_index_reversed(self: Self) -> int:
"""
Returns
-------
int
Playlist index reversed via ``playlist_count - playlist_index + 1``
"""
return self.playlist_count - self.playlist_index + 1
@property
def playlist_index_padded(self: Self) -> str:
"""
Returns
-------
str
playlist_index padded two digits
"""
return pad(self.playlist_index, width=2)
@property
def playlist_index_reversed_padded(self: Self) -> str:
"""
Returns
-------
str
playlist_index_reversed padded two digits
"""
return pad(self.playlist_index_reversed, width=2)
@property
def playlist_index_padded6(self: Self) -> str:
"""
Returns
-------
str
playlist_index padded six digits.
"""
return pad(self.playlist_index, width=6)
@property
def playlist_index_reversed_padded6(self: Self) -> str:
"""
Returns
-------
str
playlist_index_reversed padded six digits.
"""
return pad(self.playlist_index_reversed, width=6)
@property
def playlist_count(self: Self) -> int:
"""
Returns
-------
int
Playlist count if it exists, otherwise returns ``1``.
Note that for channels/playlists, any change (i.e. adding or removing a video) will make
this value change. Use with caution.
"""
return self.kwargs_get(PLAYLIST_COUNT, 1)
@property
def playlist_description(self: Self) -> str:
"""
Returns
-------
str
The playlist description if it exists, otherwise returns the entry's description.
"""
return self.kwargs_get(PLAYLIST_DESCRIPTION, self.description)
@property
def playlist_webpage_url(self: Self) -> str:
"""
Returns
-------
str
The playlist webpage url if it exists. Otherwise, returns the entry webpage url.
"""
return self.kwargs_get(PLAYLIST_WEBPAGE_URL, self.webpage_url)
@property
def playlist_max_upload_year(self: Self) -> int:
"""
Returns
-------
int
Max upload_year for all entries in this entry's playlist if it exists, otherwise returns
``upload_year``
"""
# override in EntryParent
return self.kwargs_get(PLAYLIST_MAX_UPLOAD_YEAR, self.upload_year)
@property
def playlist_max_upload_year_truncated(self: Self) -> int:
"""
Returns
-------
int
The max playlist truncated upload year for all entries in this entry's playlist if it
exists, otherwise returns ``upload_year_truncated``.
"""
return self.kwargs_get(PLAYLIST_MAX_UPLOAD_YEAR_TRUNCATED, self.upload_year_truncated)
@property
def playlist_uploader_id(self: Self) -> str:
"""
Returns
-------
str
The playlist uploader id if it exists, otherwise returns the entry uploader ID.
"""
return self.kwargs_get(PLAYLIST_UPLOADER_ID, self.uploader_id)
@property
def playlist_uploader(self: Self) -> str:
"""
Returns
-------
str
The playlist uploader if it exists, otherwise return the entry uploader.
"""
return self.kwargs_get(PLAYLIST_UPLOADER, self.uploader)
@property
def playlist_uploader_sanitized(self: Self) -> str:
"""
Returns
-------
str
The playlist uploader, sanitized.
"""
return sanitize_filename(self.playlist_uploader)
@property
def playlist_uploader_url(self: Self) -> str:
"""
Returns
-------
str
The playlist uploader url if it exists, otherwise returns the playlist webpage_url.
"""
return self.kwargs_get(PLAYLIST_UPLOADER_URL, self.playlist_webpage_url)
@property
def source_uploader_id(self: Self) -> str:
"""
Returns
-------
str
The source uploader id if it exists, otherwise returns the playlist_uploader_id
"""
return self.kwargs_get(SOURCE_UPLOADER_ID, self.playlist_uploader_id)
@property
def source_uploader(self: Self) -> str:
"""
Returns
-------
str
The source uploader if it exists, otherwise return the playlist_uploader
"""
return self.kwargs_get(SOURCE_UPLOADER, self.playlist_uploader)
@property
def source_uploader_url(self: Self) -> str:
"""
Returns
-------
str
The source uploader url if it exists, otherwise returns the source webpage_url.
"""
return self.kwargs_get(SOURCE_UPLOADER_URL, self.source_webpage_url)
@property
def creator(self: Self) -> str:
"""
Returns
-------
str
The creator name if it exists, otherwise returns the channel.
"""
return self.kwargs_get(CREATOR, self.channel)
@property
def creator_sanitized(self: Self) -> str:
"""
Returns
-------
str
The creator name, sanitized
"""
return sanitize_filename(self.creator)
@property
def channel(self: Self) -> str:
"""
Returns
-------
str
The channel name if it exists, otherwise returns the uploader.
"""
return self.kwargs_get(CHANNEL, self.uploader)
@property
def channel_sanitized(self: Self) -> str:
"""
Returns
-------
str
The channel name, sanitized.
"""
return sanitize_filename(self.channel)
@property
def channel_id(self: Self) -> str:
"""
Returns
-------
str
The channel id if it exists, otherwise returns the entry uploader ID.
"""
return self.kwargs_get(CHANNEL_ID, self.uploader_id)
@property
def ext(self: Self) -> str:
"""
Returns
-------
str
The downloaded entry's file extension
"""
return self.kwargs(EXT)
@property
def thumbnail_ext(self: Self) -> str:
"""
Returns
-------
str
The download entry's thumbnail extension. Will always return 'jpg'. Until there is a
need to support other image types, we always convert to jpg.
"""
return "jpg"
@property
def download_index(self: Self) -> int:
"""
Returns
-------
int
The i'th entry downloaded. NOTE that this is fetched dynamically from the download
archive.
"""
return self.kwargs_get(DOWNLOAD_INDEX, 0) + 1
@property
def download_index_padded6(self: Self) -> str:
"""
Returns
-------
str
The download_index padded six digits
"""
return pad(self.download_index, 6)
@property
def upload_date_index(self: Self) -> int:
"""
Returns
-------
int
The i'th entry downloaded with this upload date.
"""
return self.kwargs_get(UPLOAD_DATE_INDEX, 0) + 1
@property
def upload_date_index_padded(self: Self) -> str:
"""
Returns
-------
int
The upload_date_index padded two digits
"""
return pad(self.upload_date_index, 2)
@property
def upload_date_index_reversed(self: Self) -> int:
"""
Returns
-------
int
100 - upload_date_index
"""
return 100 - self.upload_date_index
@property
def upload_date_index_reversed_padded(self: Self) -> str:
"""
Returns
-------
int
The upload_date_index padded two digits
"""
return pad(self.upload_date_index_reversed, 2)
@property
def upload_date(self: Self) -> str:
"""
Returns
-------
str
The entry's uploaded date, in YYYYMMDD format. If not present, return today's date.
"""
return self.kwargs_get(UPLOAD_DATE, datetime.now().strftime("%Y%m%d"))
@property
def upload_year(self: Self) -> int:
"""
Returns
-------
int
The entry's upload year
"""
return int(self.upload_date[:4])
@property
def upload_year_truncated(self: Self) -> int:
"""
Returns
-------
int
The last two digits of the upload year, i.e. 22 in 2022
"""
return int(str(self.upload_year)[-2:])
@property
def upload_year_truncated_reversed(self: Self) -> int:
"""
Returns
-------
int
The upload year truncated, but reversed using ``100 - {upload_year_truncated}``, i.e.
2022 returns ``100 - 22`` = ``78``
"""
return 100 - self.upload_year_truncated
@property
def upload_month_reversed(self: Self) -> int:
"""
Returns
-------
int
The upload month, but reversed using ``13 - {upload_month}``, i.e. March returns ``10``
"""
return 13 - self.upload_month
@property
def upload_month_reversed_padded(self: Self) -> str:
"""
Returns
-------
str
The reversed upload month, but padded. i.e. November returns "02"
"""
return pad(self.upload_month_reversed)
@property
def upload_month_padded(self: Self) -> str:
"""
Returns
-------
str
The entry's upload month padded to two digits, i.e. March returns "03"
"""
return self.upload_date[4:6]
@property
def upload_day_padded(self: Self) -> str:
"""
Returns
-------
str
The entry's upload day padded to two digits, i.e. the fifth returns "05"
"""
return self.upload_date[6:8]
@property
def upload_month(self: Self) -> int:
"""
Returns
-------
int
The upload month as an integer (no padding).
"""
return int(self.upload_month_padded.lstrip("0"))
@property
def upload_day(self: Self) -> int:
"""
Returns
-------
int
The upload day as an integer (no padding).
"""
return int(self.upload_day_padded.lstrip("0"))
@property
def upload_day_reversed(self: Self) -> int:
"""
Returns
-------
int
The upload day, but reversed using ``{total_days_in_month} + 1 - {upload_day}``,
i.e. August 8th would have upload_day_reversed of ``31 + 1 - 8`` = ``24``
"""
total_days_in_month = _days_in_month[self.upload_month]
if self.upload_month == 2 and self.upload_year % 4 == 0: # leap year
total_days_in_month += 1
return total_days_in_month + 1 - self.upload_day
@property
def upload_day_reversed_padded(self: Self) -> str:
"""
Returns
-------
str
The reversed upload day, but padded. i.e. August 30th returns "02".
"""
return pad(self.upload_day_reversed)
@property
def upload_day_of_year(self: Self) -> int:
"""
Returns
-------
int
The day of the year, i.e. February 1st returns ``32``
"""
output = sum(_days_in_month[: self.upload_month]) + self.upload_day
if self.upload_month > 2 and self.upload_year % 4 == 0:
output += 1
return output
@property
def upload_day_of_year_padded(self: Self) -> str:
"""
Returns
-------
str
The upload day of year, but padded i.e. February 1st returns "032"
"""
return pad(self.upload_day_of_year, width=3)
@property
def upload_day_of_year_reversed(self: Self) -> int:
"""
Returns
-------
int
The upload day, but reversed using ``{total_days_in_year} + 1 - {upload_day}``,
i.e. February 2nd would have upload_day_of_year_reversed of ``365 + 1 - 32`` = ``334``
"""
total_days_in_year = 365
if self.upload_year % 4 == 0:
total_days_in_year += 1
return total_days_in_year + 1 - self.upload_day_of_year
@property
def upload_day_of_year_reversed_padded(self: Self) -> str:
"""
Returns
-------
str
The reversed upload day of year, but padded i.e. December 31st returns "001"
"""
return pad(self.upload_day_of_year_reversed, width=3)
@property
def upload_date_standardized(self: Self) -> str:
"""
Returns
-------
str
The uploaded date formatted as YYYY-MM-DD
"""
return f"{self.upload_year}-{self.upload_month_padded}-{self.upload_day_padded}"
@property
def release_date(self: Self) -> str:
"""
Returns
-------
str
The entry's release date, in YYYYMMDD format. If not present, return the upload date.
"""
return self.kwargs_get(RELEASE_DATE, self.upload_date)
@property
def release_year(self: Self) -> int:
"""
Returns
-------
int
The entry's release year
"""
return int(self.release_date[:4])
@property
def release_year_truncated(self: Self) -> int:
"""
Returns
-------
int
The last two digits of the release year, i.e. 22 in 2022
"""
return int(str(self.release_year)[-2:])
@property
def release_year_truncated_reversed(self: Self) -> int:
"""
Returns
-------
int
The release year truncated, but reversed using ``100 - {release_year_truncated}``, i.e.
2022 returns ``100 - 22`` = ``78``
"""
return 100 - self.release_year_truncated
@property
def release_month_reversed(self: Self) -> int:
"""
Returns
-------
int
The release month, but reversed
using ``13 - {release_month}``, i.e. March returns ``10``
"""
return 13 - self.release_month
@property
def release_month_reversed_padded(self: Self) -> str:
"""
Returns
-------
str
The reversed release month, but padded. i.e. November returns "02"
"""
return pad(self.release_month_reversed)
@property
def release_month_padded(self: Self) -> str:
"""
Returns
-------
str
The entry's release month padded to two digits, i.e. March returns "03"
"""
return self.release_date[4:6]
@property
def release_day_padded(self: Self) -> str:
"""
Returns
-------
str
The entry's release day padded to two digits, i.e. the fifth returns "05"
"""
return self.release_date[6:8]
@property
def release_month(self: Self) -> int:
"""
Returns
-------
int
The release month as an integer (no padding).
"""
return int(self.release_month_padded.lstrip("0"))
@property
def release_day(self: Self) -> int:
"""
Returns
-------
int
The release day as an integer (no padding).
"""
return int(self.release_day_padded.lstrip("0"))
@property
def release_day_reversed(self: Self) -> int:
"""
Returns
-------
int
The release day, but reversed using ``{total_days_in_month} + 1 - {release_day}``,
i.e. August 8th would have release_day_reversed of ``31 + 1 - 8`` = ``24``
"""
total_days_in_month = _days_in_month[self.release_month]
if self.release_month == 2 and self.release_year % 4 == 0: # leap year
total_days_in_month += 1
return total_days_in_month + 1 - self.release_day
@property
def release_day_reversed_padded(self: Self) -> str:
"""
Returns
-------
str
The reversed release day, but padded. i.e. August 30th returns "02".
"""
return pad(self.release_day_reversed)
@property
def release_day_of_year(self: Self) -> int:
"""
Returns
-------
int
The day of the year, i.e. February 1st returns ``32``
"""
output = sum(_days_in_month[: self.release_month]) + self.release_day
if self.release_month > 2 and self.release_year % 4 == 0:
output += 1
return output
@property
def release_day_of_year_padded(self: Self) -> str:
"""
Returns
-------
str
The release day of year, but padded i.e. February 1st returns "032"
"""
return pad(self.release_day_of_year, width=3)
@property
def release_day_of_year_reversed(self: Self) -> int:
"""
Returns
-------
int
The release day, but reversed using ``{total_days_in_year} + 1 - {release_day}``,
i.e. February 2nd would have release_day_of_year_reversed of ``365 + 1 - 32`` = ``334``
"""
total_days_in_year = 365
if self.release_year % 4 == 0:
total_days_in_year += 1
return total_days_in_year + 1 - self.release_day_of_year
@property
def release_day_of_year_reversed_padded(self: Self) -> str:
"""
Returns
-------
str
The reversed release day of year, but padded i.e. December 31st returns "001"
"""
return pad(self.release_day_of_year_reversed, width=3)
@property
def release_date_standardized(self: Self) -> str:
"""
Returns
-------
str
The release date formatted as YYYY-MM-DD
"""
return f"{self.release_year}-{self.release_month_padded}-{self.release_day_padded}"

68
src/ytdl_sub/entries/variables/kwargs.py
View file

@ -1,68 +0,0 @@
from typing import List
class KwargKeys:
keys: List[str] = []
backend_keys: List[str] = []
def _(key: str, backend: bool = False) -> str:
if backend:
assert key not in KwargKeys.backend_keys
KwargKeys.backend_keys.append(key)
else:
assert key not in KwargKeys.keys
KwargKeys.keys.append(key)
return key
SOURCE_ENTRY = _("source_entry", backend=True)
SOURCE_INDEX = _("source_index")
SOURCE_COUNT = _("source_count")
SOURCE_TITLE = _("source_title")
SOURCE_UID = _("source_uid")
SOURCE_DESCRIPTION = _("source_description")
SOURCE_WEBPAGE_URL = _("source_webpage_url")
SOURCE_UPLOADER = _("source_uploader")
SOURCE_UPLOADER_ID = _("source_uploader_id")
SOURCE_UPLOADER_URL = _("source_uploader_url")
PLAYLIST_ENTRY = _("playlist_entry", backend=True)
PLAYLIST_WEBPAGE_URL = _("playlist_webpage_url")
PLAYLIST_INDEX = _("playlist_index")
PLAYLIST_COUNT = _("playlist_count")
PLAYLIST_MAX_UPLOAD_YEAR = _("playlist_max_upload_year")
PLAYLIST_MAX_UPLOAD_YEAR_TRUNCATED = _("playlist_max_upload_year_truncated")
PLAYLIST_TITLE = _("playlist_title")
PLAYLIST_DESCRIPTION = _("playlist_description")
PLAYLIST_UID = _("playlist_uid")
PLAYLIST_UPLOADER = _("playlist_uploader")
PLAYLIST_UPLOADER_ID = _("playlist_uploader_id")
PLAYLIST_UPLOADER_URL = _("playlist_uploader_url")
COLLECTION_URL = _("collection_url", backend=True)
DOWNLOAD_INDEX = _("download_index", backend=True)
UPLOAD_DATE_INDEX = _("upload_date_index", backend=True)
REQUESTED_SUBTITLES = _("requested_subtitles", backend=True)
CHAPTERS = _("chapters", backend=True)
YTDL_SUB_CUSTOM_CHAPTERS = _("ytdl_sub_custom_chapters", backend=True)
YTDL_SUB_REGEX_SOURCE_VARS = _("ytdl_sub_regex_source_vars", backend=True)
SPONSORBLOCK_CHAPTERS = _("sponsorblock_chapters", backend=True)
SPLIT_BY_CHAPTERS_PARENT_ENTRY = _("split_by_chapters_parent_entry", backend=True)
COMMENTS = _("comments", backend=True)
UID = _("id")
EXTRACTOR = _("extractor")
IE_KEY = _("ie_key")
EPOCH = _("epoch")
CHANNEL = _("channel")
CHANNEL_ID = _("channel_id")
CREATOR = _("creator")
EXT = _("ext")
TITLE = _("title")
DESCRIPTION = _("description")
WEBPAGE_URL = _("webpage_url")
RELEASE_DATE = _("release_date")
UPLOAD_DATE = _("upload_date")
UPLOADER = _("uploader")
UPLOADER_ID = _("uploader_id")
UPLOADER_URL = _("uploader_url")

26
src/ytdl_sub/entries/variables/override_variables.py
View file

@ -1,5 +1,11 @@
from ytdl_sub.entries.script.function_scripts import CUSTOM_FUNCTION_SCRIPTS
from ytdl_sub.entries.script.variable_scripts import VARIABLE_SCRIPTS
from ytdl_sub.script.functions import Functions
SUBSCRIPTION_NAME = "subscription_name"
SUBSCRIPTION_VALUE = "subscription_value"
SUBSCRIPTION_MAP = "subscription_map"
SUBSCRIPTION_ARRAY = "subscription_array"
class OverrideVariables:
@ -55,3 +61,23 @@ class OverrideVariables:
``subscription_value``.
"""
return f"subscription_value_{index + 1}"
@classmethod
def is_entry_variable_name(cls, name: str) -> bool:
"""
Returns
-------
True if the name is an entry variable name. False otherwise.
"""
return name in VARIABLE_SCRIPTS
@classmethod
def is_function_name(cls, name: str) -> bool:
"""
Returns
-------
True if the name is a function name (either built-in or script). False otherwise.
"""
if name.startswith("%"):
return name in CUSTOM_FUNCTION_SCRIPTS or Functions.is_built_in(name[1:])
return False

18
src/ytdl_sub/plugins/audio_extract.py
View file

@ -2,11 +2,15 @@ import os.path
from typing import Any
from typing import Dict
from typing import Optional
from typing import Set
from ytdl_sub.config.plugin import Plugin
from ytdl_sub.config.preset_options import OptionsDictValidator
from ytdl_sub.config.plugin.plugin import Plugin
from ytdl_sub.config.plugin.plugin_operation import PluginOperation
from ytdl_sub.config.validators.options import OptionsDictValidator
from ytdl_sub.downloaders.ytdl_options_builder import YTDLOptionsBuilder
from ytdl_sub.entries.entry import Entry
from ytdl_sub.entries.script.variable_definitions import VARIABLES
from ytdl_sub.entries.script.variable_definitions import VariableDefinitions
from ytdl_sub.utils.exceptions import FileNotDownloadedException
from ytdl_sub.utils.file_handler import FileMetadata
from ytdl_sub.validators.audo_codec_validator import AUDIO_CODEC_EXTS
@ -14,6 +18,8 @@ from ytdl_sub.validators.audo_codec_validator import AUDIO_CODEC_TYPES_EXTENSION
from ytdl_sub.validators.audo_codec_validator import AudioTypeValidator
from ytdl_sub.validators.validators import FloatValidator
v: VariableDefinitions = VARIABLES
class AudioExtractOptions(OptionsDictValidator):
"""
@ -65,6 +71,12 @@ class AudioExtractOptions(OptionsDictValidator):
return self._quality.value
return None
def modified_variables(self) -> Dict[PluginOperation, Set[str]]:
"""
Possibly changes ``ext``, so do not resolve until this has run
"""
return {PluginOperation.MODIFY_ENTRY: {v.ext.variable_name}}
class AudioExtractPlugin(Plugin[AudioExtractOptions]):
plugin_options_type = AudioExtractOptions
@ -125,7 +137,7 @@ class AudioExtractPlugin(Plugin[AudioExtractOptions]):
new_ext = AUDIO_CODEC_TYPES_EXTENSION_MAPPING[self.plugin_options.codec]
extracted_audio_file = entry.get_download_file_path().removesuffix(entry.ext) + new_ext
entry.add_kwargs({"ext": new_ext})
entry.add({v.ext: new_ext})
if not self.is_dry_run:
if not os.path.isfile(extracted_audio_file):

61
src/ytdl_sub/plugins/chapters.py
View file

@ -5,12 +5,15 @@ from typing import List
from typing import Optional
from typing import Set
from ytdl_sub.config.plugin import Plugin
from ytdl_sub.config.preset_options import OptionsDictValidator
from ytdl_sub.config.plugin.plugin import Plugin
from ytdl_sub.config.plugin.plugin_operation import PluginOperation
from ytdl_sub.config.validators.options import OptionsDictValidator
from ytdl_sub.downloaders.ytdl_options_builder import YTDLOptionsBuilder
from ytdl_sub.entries.entry import Entry
from ytdl_sub.entries.variables.kwargs import COMMENTS
from ytdl_sub.entries.variables.kwargs import YTDL_SUB_CUSTOM_CHAPTERS
from ytdl_sub.entries.entry import ytdl_sub_chapters_from_comments
from ytdl_sub.entries.entry import ytdl_sub_split_by_chapters_parent_uid
from ytdl_sub.entries.script.variable_definitions import VARIABLES
from ytdl_sub.entries.script.variable_definitions import VariableDefinitions
from ytdl_sub.utils.chapters import Chapters
from ytdl_sub.utils.ffmpeg import set_ffmpeg_metadata_chapters
from ytdl_sub.utils.file_handler import FileMetadata
@ -19,6 +22,8 @@ from ytdl_sub.validators.string_select_validator import StringSelectValidator
from ytdl_sub.validators.validators import BoolValidator
from ytdl_sub.validators.validators import ListValidator
v: VariableDefinitions = VARIABLES
SPONSORBLOCK_HIGHLIGHT_CATEGORIES: Set[str] = {"poi_highlight"}
SPONSORBLOCK_CATEGORIES: Set[str] = SPONSORBLOCK_HIGHLIGHT_CATEGORIES | {
"sponsor",
@ -33,15 +38,11 @@ SPONSORBLOCK_CATEGORIES: Set[str] = SPONSORBLOCK_HIGHLIGHT_CATEGORIES | {
def _chapters(entry: Entry) -> List[Dict]:
if entry.kwargs_contains("chapters"):
return entry.kwargs("chapters") or []
return []
return entry.get(v.chapters, list)
def _sponsorblock_chapters(entry: Entry) -> List[Dict]:
if entry.kwargs_contains("sponsorblock_chapters"):
return entry.kwargs("sponsorblock_chapters") or []
return []
return entry.get(v.sponsorblock_chapters, list)
def _contains_any_chapters(entry: Entry) -> bool:
@ -195,6 +196,14 @@ class ChaptersOptions(OptionsDictValidator):
"""
return self._force_key_frames
def added_variables(
self,
resolved_variables: Set[str],
unresolved_variables: Set[str],
plugin_op: PluginOperation,
) -> Dict[PluginOperation, Set[str]]:
return {PluginOperation.MODIFY_ENTRY: {ytdl_sub_chapters_from_comments.variable_name}}
class ChaptersPlugin(Plugin[ChaptersOptions]):
plugin_options_type = ChaptersOptions
@ -300,27 +309,33 @@ class ChaptersPlugin(Plugin[ChaptersOptions]):
-------
entry
"""
chapters = Chapters.from_empty()
has_chapters_from_comments = False
# If there are no embedded chapters, and comment chapters are allowed...
if not _contains_any_chapters(entry) and self.plugin_options.allow_chapters_from_comments:
chapters = Chapters.from_empty()
# Try to get chapters from comments
for comment in entry.kwargs_get(COMMENTS, []):
for comment in entry.get(v.comments, list):
chapters = Chapters.from_string(comment.get("text", ""))
if chapters.contains_any_chapters():
break
# If some are actually found, add a special kwarg and embed them
if chapters.contains_any_chapters():
entry.add_kwargs({YTDL_SUB_CUSTOM_CHAPTERS: chapters.to_file_metadata_dict()})
has_chapters_from_comments = True
entry.add({ytdl_sub_chapters_from_comments: chapters.to_yt_dlp_chapter_metadata()})
if not self.is_dry_run:
set_ffmpeg_metadata_chapters(
file_path=entry.get_download_file_path(),
chapters=chapters,
file_duration_sec=entry.kwargs("duration"),
file_duration_sec=entry.get(v.duration, int),
)
if not has_chapters_from_comments:
entry.add({ytdl_sub_chapters_from_comments: []})
return entry
def post_process_entry(self, entry: Entry) -> Optional[FileMetadata]:
@ -334,12 +349,9 @@ class ChaptersPlugin(Plugin[ChaptersOptions]):
-------
FileMetadata outlining which chapters/SponsorBlock segments got removed
"""
if custom_chapters_metadata := entry.kwargs_get(YTDL_SUB_CUSTOM_CHAPTERS):
title: str = "Chapters from comments"
return FileMetadata.from_dict(
value_dict=custom_chapters_metadata,
title=title,
sort_dict=False, # timestamps + titles are already sorted
if custom_chapters := entry.get(ytdl_sub_chapters_from_comments, list):
return Chapters.from_yt_dlp_chapters(custom_chapters).to_file_metadata(
title="Chapters from comments"
)
if self.plugin_options.embed_chapters:
@ -356,9 +368,10 @@ class ChaptersPlugin(Plugin[ChaptersOptions]):
if removed_sponsorblock:
metadata_dict["Removed SponsorBlock Category Count(s)"] = removed_sponsorblock
# TODO: check if file actually has embedded chapters
return FileMetadata.from_dict(
value_dict=metadata_dict, title="Embedded Chapters", sort_dict=False
)
# If the entry wasn't split on embedded chapters, report it in the file metadata
if not entry.try_get(ytdl_sub_split_by_chapters_parent_uid, str):
return FileMetadata.from_dict(
value_dict=metadata_dict, title="Embedded Chapters", sort_dict=False
)
return None

4
src/ytdl_sub/plugins/date_range.py
View file

@ -2,8 +2,8 @@ from typing import List
from typing import Optional
from typing import Tuple
from ytdl_sub.config.plugin import Plugin
from ytdl_sub.config.preset_options import OptionsDictValidator
from ytdl_sub.config.plugin.plugin import Plugin
from ytdl_sub.config.validators.options import OptionsDictValidator
from ytdl_sub.utils.datetime import to_date_str
from ytdl_sub.validators.string_datetime import StringDatetimeValidator

6
src/ytdl_sub/plugins/embed_thumbnail.py
View file

@ -3,9 +3,8 @@ from typing import Optional
import mediafile
from ytdl_sub.config.plugin import Plugin
from ytdl_sub.config.plugin import PluginPriority
from ytdl_sub.config.preset_options import OptionsValidator
from ytdl_sub.config.plugin.plugin import Plugin
from ytdl_sub.config.validators.options import OptionsValidator
from ytdl_sub.entries.entry import Entry
from ytdl_sub.utils.ffmpeg import FFMPEG
from ytdl_sub.utils.file_handler import FileHandler
@ -33,7 +32,6 @@ class EmbedThumbnailOptions(BoolValidator, OptionsValidator):
class EmbedThumbnailPlugin(Plugin[EmbedThumbnailOptions]):
plugin_options_type = EmbedThumbnailOptions
priority = PluginPriority(post_process=PluginPriority.POST_PROCESS_AFTER_FILE_CONVERT)
@property
def _embed_thumbnail(self) -> bool:

45
src/ytdl_sub/plugins/file_convert.py
View file

@ -2,12 +2,15 @@ import os
from typing import Any
from typing import Dict
from typing import Optional
from typing import Set
from ytdl_sub.config.plugin import Plugin
from ytdl_sub.config.plugin import PluginPriority
from ytdl_sub.config.preset_options import OptionsDictValidator
from ytdl_sub.config.overrides import Overrides
from ytdl_sub.config.plugin.plugin import Plugin
from ytdl_sub.config.plugin.plugin_operation import PluginOperation
from ytdl_sub.config.validators.options import OptionsDictValidator
from ytdl_sub.entries.entry import Entry
from ytdl_sub.entries.variables.kwargs import EXT
from ytdl_sub.entries.script.variable_definitions import VARIABLES
from ytdl_sub.entries.script.variable_definitions import VariableDefinitions
from ytdl_sub.utils.exceptions import FileNotDownloadedException
from ytdl_sub.utils.exceptions import ValidationException
from ytdl_sub.utils.ffmpeg import FFMPEG
@ -16,6 +19,9 @@ from ytdl_sub.utils.file_handler import FileMetadata
from ytdl_sub.validators.audo_codec_validator import FileTypeValidator
from ytdl_sub.validators.string_formatter_validators import OverridesStringFormatterValidator
from ytdl_sub.validators.string_select_validator import StringSelectValidator
from ytdl_sub.ytdl_additions.enhanced_download_archive import EnhancedDownloadArchive
v: VariableDefinitions = VARIABLES
class FileConvertWithValidator(StringSelectValidator):
@ -115,13 +121,26 @@ class FileConvertOptions(OptionsDictValidator):
"""
return self._ffmpeg_post_process_args
def modified_variables(self) -> Dict[PluginOperation, Set[str]]:
return {PluginOperation.MODIFY_ENTRY: {v.ext.variable_name}}
class FileConvertPlugin(Plugin[FileConvertOptions]):
plugin_options_type = FileConvertOptions
# Perform this after regex
priority: PluginPriority = PluginPriority(
modify_entry=PluginPriority.MODIFY_ENTRY_AFTER_SPLIT + 1
)
def __init__(
self,
options: FileConvertOptions,
overrides: Overrides,
enhanced_download_archive: EnhancedDownloadArchive,
):
super().__init__(
options=options,
overrides=overrides,
enhanced_download_archive=enhanced_download_archive,
)
# Lookup of entry id to what it was converted from for logging
self._converted_from_lookup: Dict[str, str] = {}
def ytdl_options(self) -> Optional[Dict]:
"""
@ -198,13 +217,9 @@ class FileConvertPlugin(Plugin[FileConvertOptions]):
FileHandler.delete(tmp_output_file)
if original_ext != new_ext:
entry.add_kwargs(
{
"__converted_from": original_ext,
}
)
self._converted_from_lookup[entry.ytdl_uid()] = original_ext
entry.add_kwargs({EXT: new_ext})
entry.add({v.ext: new_ext})
return entry
@ -212,7 +227,7 @@ class FileConvertPlugin(Plugin[FileConvertOptions]):
"""
Add metadata about conversion if it happened
"""
if converted_from := entry.kwargs_get("__converted_from"):
if converted_from := self._converted_from_lookup.get(entry.ytdl_uid()):
return FileMetadata(f"Converted from {converted_from}")
return None

4
src/ytdl_sub/plugins/format.py
View file

@ -1,8 +1,8 @@
from typing import Dict
from typing import Optional
from ytdl_sub.config.plugin import Plugin
from ytdl_sub.config.preset_options import OptionsValidator
from ytdl_sub.config.plugin.plugin import Plugin
from ytdl_sub.config.validators.options import OptionsValidator
from ytdl_sub.validators.validators import StringValidator

8
src/ytdl_sub/plugins/internal/view.py
View file

@ -1,10 +1,9 @@
import copy
from typing import Optional
from ytdl_sub.config.plugin import Plugin
from ytdl_sub.config.plugin import PluginPriority
from ytdl_sub.config.preset_options import OptionsDictValidator
from ytdl_sub.config.preset_options import Overrides
from ytdl_sub.config.overrides import Overrides
from ytdl_sub.config.plugin.plugin import Plugin
from ytdl_sub.config.validators.options import OptionsDictValidator
from ytdl_sub.entries.entry import Entry
from ytdl_sub.utils.file_handler import FileMetadata
from ytdl_sub.ytdl_additions.enhanced_download_archive import EnhancedDownloadArchive
@ -20,7 +19,6 @@ class ViewOptions(OptionsDictValidator):
class ViewPlugin(Plugin[ViewOptions]):
plugin_options_type = ViewOptions
priority: PluginPriority = PluginPriority(modify_entry=PluginPriority.MODIFY_ENTRY_AFTER_SPLIT)
_MAX_LINE_WIDTH: int = 80

4
src/ytdl_sub/plugins/match_filters.py
View file

@ -3,8 +3,8 @@ from typing import Any
from typing import List
from typing import Tuple
from ytdl_sub.config.plugin import Plugin
from ytdl_sub.config.preset_options import OptionsDictValidator
from ytdl_sub.config.plugin.plugin import Plugin
from ytdl_sub.config.validators.options import OptionsDictValidator
from ytdl_sub.utils.logger import Logger
from ytdl_sub.validators.validators import StringListValidator

12
src/ytdl_sub/plugins/music_tags.py
View file

@ -6,9 +6,11 @@ from typing import List
import mediafile
from ytdl_sub.config.plugin import Plugin
from ytdl_sub.config.preset_options import OptionsDictValidator
from ytdl_sub.config.plugin.plugin import Plugin
from ytdl_sub.config.validators.options import OptionsDictValidator
from ytdl_sub.entries.entry import Entry
from ytdl_sub.entries.script.variable_definitions import VARIABLES
from ytdl_sub.entries.script.variable_definitions import VariableDefinitions
from ytdl_sub.utils.file_handler import FileMetadata
from ytdl_sub.utils.logger import Logger
from ytdl_sub.validators.audo_codec_validator import AUDIO_CODEC_EXTS
@ -17,6 +19,8 @@ from ytdl_sub.validators.string_formatter_validators import ListFormatterValidat
from ytdl_sub.validators.string_formatter_validators import StringFormatterValidator
from ytdl_sub.validators.validators import BoolValidator
v: VariableDefinitions = VARIABLES
logger = Logger.get("music_tags")
@ -132,9 +136,9 @@ class MusicTagsPlugin(Plugin[MusicTagsOptions]):
"""
Tags the entry's audio file using values defined in the metadata options
"""
if entry.ext not in AUDIO_CODEC_EXTS:
if (ext := entry.get(v.ext, str)) not in AUDIO_CODEC_EXTS:
raise self.plugin_options.validation_exception(
f"music_tags plugin received a video with the extension '{entry.ext}'. Only audio "
f"music_tags plugin received a video with the extension '{ext}'. Only audio "
f"files are supported for setting music tags. Ensure you are converting the video "
f"to audio using the audio_extract plugin."
)

4
src/ytdl_sub/plugins/nfo_tags.py
View file

@ -7,8 +7,8 @@ from typing import Dict
from typing import List
from typing import Optional
from ytdl_sub.config.plugin import Plugin
from ytdl_sub.config.preset_options import OptionsDictValidator
from ytdl_sub.config.plugin.plugin import Plugin
from ytdl_sub.config.validators.options import OptionsDictValidator
from ytdl_sub.entries.entry import Entry
from ytdl_sub.utils.file_handler import FileHandler
from ytdl_sub.utils.file_handler import FileMetadata

2
src/ytdl_sub/plugins/output_directory_nfo_tags.py
View file

@ -1,4 +1,4 @@
from ytdl_sub.config.preset_options import Overrides
from ytdl_sub.config.overrides import Overrides
from ytdl_sub.entries.entry import Entry
from ytdl_sub.plugins.nfo_tags import NfoTagsValidator
from ytdl_sub.plugins.nfo_tags import SharedNfoTagsOptions

239
src/ytdl_sub/plugins/regex.py
View file

@ -1,17 +1,18 @@
from collections import defaultdict
from typing import Any
from typing import Dict
from typing import List
from typing import Optional
from typing import Set
from yt_dlp.utils import sanitize_filename
from ytdl_sub.config.plugin import Plugin
from ytdl_sub.config.plugin import PluginPriority
from ytdl_sub.config.preset_options import OptionsDictValidator
from ytdl_sub.config.overrides import Overrides
from ytdl_sub.config.plugin.plugin import Plugin
from ytdl_sub.config.plugin.plugin_operation import PluginOperation
from ytdl_sub.config.validators.options import OptionsDictValidator
from ytdl_sub.entries.entry import Entry
from ytdl_sub.entries.variables.kwargs import YTDL_SUB_REGEX_SOURCE_VARS
from ytdl_sub.script.parser import parse
from ytdl_sub.script.utils.exceptions import RuntimeException
from ytdl_sub.utils.exceptions import RegexNoMatchException
from ytdl_sub.utils.exceptions import StringFormattingVariableNotFoundException
from ytdl_sub.utils.logger import Logger
from ytdl_sub.validators.regex_validator import RegexListValidator
from ytdl_sub.validators.source_variable_validator import SourceVariableNameListValidator
@ -19,6 +20,8 @@ from ytdl_sub.validators.strict_dict_validator import StrictDictValidator
from ytdl_sub.validators.string_formatter_validators import ListFormatterValidator
from ytdl_sub.validators.string_formatter_validators import StringFormatterValidator
from ytdl_sub.validators.validators import BoolValidator
from ytdl_sub.validators.validators import DictValidator
from ytdl_sub.ytdl_additions.enhanced_download_archive import EnhancedDownloadArchive
logger = Logger.get(name="regex")
@ -109,11 +112,7 @@ class VariableRegex(StrictDictValidator):
return self._capture_group_defaults is not None
class FromSourceVariablesRegex(StrictDictValidator):
_optional_keys = Entry.source_variables()
_allow_extra_keys = True
class FromSourceVariablesRegex(DictValidator):
def __init__(self, name, value):
super().__init__(name, value)
self.variable_capture_dict: Dict[str, VariableRegex] = {
@ -204,6 +203,9 @@ class RegexOptions(OptionsDictValidator):
key="skip_if_match_fails", validator=BoolValidator, default=True
).value
# Variables added by the regex plugin
self._added_variable_names: Set[str] = set()
@property
def skip_if_match_fails(self) -> Optional[bool]:
"""
@ -212,39 +214,6 @@ class RegexOptions(OptionsDictValidator):
"""
return self._skip_if_match_fails
def validate_with_variables(
self, source_variables: List[str], override_variables: Dict[str, str]
) -> None:
"""
Ensures each source variable capture group is valid
Parameters
----------
source_variables
Available source variables when running the plugin
override_variables
Available override variables when running the plugin
"""
for key, regex_options in self.source_variable_capture_dict.items():
# Ensure each variable getting captured is a source variable
if key not in source_variables and key not in override_variables:
raise self._validation_exception(
f"cannot regex capture '{key}' because it is not a source or override variable"
)
# Ensure the capture group names are not existing source/override variables
for capture_group_name in regex_options.capture_group_names:
if capture_group_name in source_variables:
raise self._validation_exception(
f"'{capture_group_name}' cannot be used as a capture group name because it "
f"is a source variable"
)
if capture_group_name in override_variables:
raise self._validation_exception(
f"'{capture_group_name}' cannot be used as a capture group name because it "
f"is an override variable"
)
@property
def source_variable_capture_dict(self) -> Dict[str, VariableRegex]:
"""
@ -254,39 +223,91 @@ class RegexOptions(OptionsDictValidator):
"""
return self._from.variable_capture_dict
def added_source_variables(self) -> List[str]:
@classmethod
def _can_resolve(
cls, unresolved_variables: Set[str], input_variable_name: str, regex_options: VariableRegex
) -> bool:
if input_variable_name in unresolved_variables:
return False
for capture_group_default in regex_options.capture_group_defaults or []:
parsed_default = parse(capture_group_default.format_string)
if parsed_default.variables and parsed_default.variables.issubset(unresolved_variables):
return False
return True
def added_variables(
self,
resolved_variables: Set[str],
unresolved_variables: Set[str],
plugin_op: PluginOperation,
) -> Dict[PluginOperation, Set[str]]:
"""
Returns
-------
List of new source variables created via regex capture
"""
added_source_vars: List[str] = []
for regex_options in self.source_variable_capture_dict.values():
added_source_vars.extend(regex_options.capture_group_names)
added_source_vars.extend(
f"{capture_group_name}_sanitized"
for capture_group_name in regex_options.capture_group_names
)
added_source_vars: Dict[PluginOperation, Set[str]] = {
PluginOperation.MODIFY_ENTRY_METADATA: set(),
PluginOperation.MODIFY_ENTRY: set(),
}
for input_variable_name, regex_options in self.source_variable_capture_dict.items():
variables_to_add = set(regex_options.capture_group_names)
if plugin_op != PluginOperation.ANY and input_variable_name not in (
resolved_variables | unresolved_variables
):
raise self._validation_exception(
f"cannot regex capture '{input_variable_name}' because it is not a"
" defined variable."
)
if (
plugin_op.value >= PluginOperation.MODIFY_ENTRY.value
and input_variable_name in unresolved_variables
):
raise self._validation_exception(
f"cannot regex capture '{input_variable_name}' because it is not "
f"computed until later in execution."
)
if plugin_op == PluginOperation.ANY:
added_source_vars[PluginOperation.MODIFY_ENTRY_METADATA] |= variables_to_add
self._added_variable_names |= variables_to_add
continue
if not self._can_resolve(
unresolved_variables=unresolved_variables,
input_variable_name=input_variable_name,
regex_options=regex_options,
):
continue
for capture_group_name in regex_options.capture_group_names:
if capture_group_name in (resolved_variables - self._added_variable_names):
raise self._validation_exception(
f"cannot use '{capture_group_name}' as a capture group name because it is "
f"an already defined variable."
)
added_source_vars[plugin_op] |= set(regex_options.capture_group_names)
return added_source_vars
class RegexPlugin(Plugin[RegexOptions]):
plugin_options_type = RegexOptions
priority = PluginPriority(
modify_entry=PluginPriority.MODIFY_ENTRY_AFTER_SPLIT + 0,
)
@classmethod
def _add_processed_regex_variable_name(cls, entry: Entry, source_var: str) -> None:
if not entry.kwargs_contains(YTDL_SUB_REGEX_SOURCE_VARS):
entry.add_kwargs({YTDL_SUB_REGEX_SOURCE_VARS: []})
entry.kwargs(YTDL_SUB_REGEX_SOURCE_VARS).append(source_var)
@classmethod
def _contains_processed_regex_variable(cls, entry: Entry, variable_name: str) -> bool:
return variable_name in entry.kwargs_get(YTDL_SUB_REGEX_SOURCE_VARS, [])
def __init__(
self,
options: RegexOptions,
overrides: Overrides,
enhanced_download_archive: EnhancedDownloadArchive,
):
super().__init__(
options=options,
overrides=overrides,
enhanced_download_archive=enhanced_download_archive,
)
# Lookup of entry id to processed regex variables
self._processed_regex_vars: Dict[str, Set[str]] = defaultdict(set)
def _try_skip_entry(self, entry: Entry, variable_name: str) -> None:
# Skip the entry if toggled
@ -301,34 +322,16 @@ class RegexPlugin(Plugin[RegexOptions]):
# Otherwise, error
raise RegexNoMatchException(f"Regex failed to match '{variable_name}' from '{entry.title}'")
def _can_process_at_metadata_stage(self, entry: Entry, variable_name: str) -> bool:
# If the variable is an override...
if variable_name in self.overrides.dict:
# Try to see if it can resolve
try:
self.overrides.apply_formatter(
formatter=self.overrides.dict[variable_name],
entry=entry,
)
# If it can not from missing variables (from post-metadata stage), return False
except StringFormattingVariableNotFoundException:
return False
# If it is a source variable and not present, return false
elif variable_name not in entry.to_dict():
@classmethod
def _can_process_at_metadata_stage(cls, entry: Entry, variable_name: str) -> bool:
# Try to see if it can resolve
try:
_ = entry.script.get(variable_name)
return True
# If it can not from missing variables (from post-metadata stage), return False
except RuntimeException:
return False
return True
def _get_regex_input_string(self, entry: Entry, variable_name: str) -> str:
# Apply override formatter if it's an override
if variable_name in self.overrides.dict:
return self.overrides.apply_formatter(
formatter=self.overrides.dict[variable_name],
entry=entry,
)
# Otherwise pluck from the entry's source variable
return entry.to_dict()[variable_name]
def _modify_entry_metadata(self, entry: Entry, is_metadata_stage: bool) -> Optional[Entry]:
"""
Parameters
@ -356,7 +359,7 @@ class RegexPlugin(Plugin[RegexOptions]):
# Record which regex source variables are processed, to
# process as many variables as possible in the metadata stage, then the rest
# after the media file has been downloaded.
if self._contains_processed_regex_variable(entry, variable_name):
if variable_name in self._processed_regex_vars[entry.ytdl_uid()]:
continue
# If it's the metadata stage, and it can't be processed, skip until post-metadata
@ -365,12 +368,8 @@ class RegexPlugin(Plugin[RegexOptions]):
):
continue
self._add_processed_regex_variable_name(entry, variable_name)
regex_input_str = self._get_regex_input_string(
entry=entry,
variable_name=variable_name,
)
self._processed_regex_vars[entry.ytdl_uid()].add(variable_name)
regex_input_str = str(entry.script.get(variable_name))
if (
regex_options.exclude is not None
@ -388,49 +387,23 @@ class RegexPlugin(Plugin[RegexOptions]):
if not regex_options.has_defaults:
return self._try_skip_entry(entry=entry, variable_name=variable_name)
# otherwise, use defaults (apply them using the original entry source dict)
source_variables_and_overrides_dict = dict(
entry.to_dict(), **self.overrides.dict_with_format_strings
)
# add both the default...
entry.add_variables(
variables_to_add={
regex_options.capture_group_names[i]: default.apply_formatter(
variable_dict=source_variables_and_overrides_dict
entry.add(
{
regex_options.capture_group_names[i]: self.overrides.apply_formatter(
formatter=default, entry=entry
)
for i, default in enumerate(regex_options.capture_group_defaults)
},
)
# and sanitized default
entry.add_variables(
variables_to_add={
f"{regex_options.capture_group_names[i]}_sanitized": sanitize_filename(
default.apply_formatter(
variable_dict=source_variables_and_overrides_dict
)
)
for i, default in enumerate(regex_options.capture_group_defaults)
},
}
)
# There is a capture, add the source variables to the entry as
# {source_var}_capture_1, {source_var}_capture_2, ...
else:
# Add the value...
entry.add_variables(
variables_to_add={
entry.add(
{
regex_options.capture_group_names[i]: capture
for i, capture in enumerate(maybe_capture)
},
)
# And the sanitized value
entry.add_variables(
variables_to_add={
f"{regex_options.capture_group_names[i]}_sanitized": sanitize_filename(
capture
)
for i, capture in enumerate(maybe_capture)
},
}
)
return entry

121
src/ytdl_sub/plugins/split_by_chapters.py
View file

@ -1,19 +1,17 @@
import copy
from pathlib import Path
from typing import Any
from typing import Dict
from typing import List
from typing import Optional
from typing import Set
from typing import Tuple
from yt_dlp.utils import sanitize_filename
from ytdl_sub.config.plugin import SplitPlugin
from ytdl_sub.config.preset_options import OptionsDictValidator
from ytdl_sub.config.plugin.plugin import SplitPlugin
from ytdl_sub.config.plugin.plugin_operation import PluginOperation
from ytdl_sub.config.validators.options import OptionsDictValidator
from ytdl_sub.entries.entry import Entry
from ytdl_sub.entries.variables.kwargs import CHAPTERS
from ytdl_sub.entries.variables.kwargs import SPLIT_BY_CHAPTERS_PARENT_ENTRY
from ytdl_sub.entries.variables.kwargs import SPONSORBLOCK_CHAPTERS
from ytdl_sub.entries.variables.kwargs import UID
from ytdl_sub.entries.entry import ytdl_sub_split_by_chapters_parent_uid
from ytdl_sub.entries.script.variable_definitions import VARIABLES
from ytdl_sub.entries.script.variable_definitions import VariableDefinitions
from ytdl_sub.utils.chapters import Chapters
from ytdl_sub.utils.chapters import Timestamp
from ytdl_sub.utils.exceptions import ValidationException
@ -22,6 +20,8 @@ from ytdl_sub.utils.file_handler import FileHandler
from ytdl_sub.utils.file_handler import FileMetadata
from ytdl_sub.validators.string_select_validator import StringSelectValidator
v: VariableDefinitions = VARIABLES
def _split_video_ffmpeg_cmd(
input_file: str, output_file: str, timestamps: List[Timestamp], idx: int
@ -81,14 +81,21 @@ class SplitByChaptersOptions(OptionsDictValidator):
key="when_no_chapters", validator=WhenNoChaptersValidator
).value
def added_source_variables(self) -> List[str]:
return [
"chapter_title",
"chapter_title_sanitized",
"chapter_index",
"chapter_index_padded",
"chapter_count",
]
def added_variables(
self,
resolved_variables: Set[str],
unresolved_variables: Set[str],
plugin_op: PluginOperation,
) -> Dict[PluginOperation, Set[str]]:
return {
PluginOperation.MODIFY_ENTRY: {
"chapter_title",
"chapter_title_sanitized",
"chapter_index",
"chapter_index_padded",
"chapter_count",
}
}
@property
def when_no_chapters(self) -> str:
@ -98,44 +105,49 @@ class SplitByChaptersOptions(OptionsDictValidator):
"""
return self._when_no_chapters
def modified_variables(self) -> Dict[PluginOperation, Set[str]]:
return {
PluginOperation.MODIFY_ENTRY: {
v.uid.variable_name,
ytdl_sub_split_by_chapters_parent_uid.variable_name,
}
}
class SplitByChaptersPlugin(SplitPlugin[SplitByChaptersOptions]):
plugin_options_type = SplitByChaptersOptions
@classmethod
def _non_split_entry(cls, entry: Entry) -> Entry:
entry.add(
{
"chapter_title": f"{{ {v.title.variable_name} }}",
"chapter_index": 1,
"chapter_index_padded": "01",
"chapter_count": 1,
v.uid: entry.uid,
ytdl_sub_split_by_chapters_parent_uid: entry.uid,
}
)
return entry
def _create_split_entry(
self, source_entry: Entry, title: str, idx: int, chapters: Chapters
self, new_entry: Entry, title: str, idx: int, chapters: Chapters
) -> Tuple[Entry, FileMetadata]:
"""
Runs ffmpeg to create the split video
"""
entry = copy.deepcopy(source_entry)
entry.add_variables(
new_entry.add(
{
"chapter_title": title,
"chapter_title_sanitized": sanitize_filename(title),
"chapter_index": idx + 1,
"chapter_index_padded": f"{(idx + 1):02d}",
"chapter_count": len(chapters.timestamps),
}
)
# pylint: disable=protected-access
entry.add_kwargs(
{
UID: _split_video_uid(source_uid=entry.uid, idx=idx),
SPLIT_BY_CHAPTERS_PARENT_ENTRY: source_entry._kwargs,
}
)
if entry.kwargs_contains(CHAPTERS):
del entry._kwargs[CHAPTERS]
if entry.kwargs_contains(SPONSORBLOCK_CHAPTERS):
del entry._kwargs[SPONSORBLOCK_CHAPTERS]
# pylint: enable=protected-access
timestamp_begin = chapters.timestamps[idx].readable_str
timestamp_end = Timestamp(entry.kwargs("duration")).readable_str
timestamp_end = Timestamp(new_entry.get(v.duration, int)).readable_str
if idx + 1 < len(chapters.timestamps):
timestamp_end = chapters.timestamps[idx + 1].readable_str
@ -145,7 +157,7 @@ class SplitByChaptersPlugin(SplitPlugin[SplitByChaptersOptions]):
"Warning"
] = "Dry-run assumes embedded chapters with no modifications"
metadata_value_dict["Source Title"] = entry.title
metadata_value_dict["Source Title"] = new_entry.title
metadata_value_dict["Segment"] = f"{timestamp_begin} - {timestamp_end}"
metadata = FileMetadata.from_dict(
@ -154,7 +166,7 @@ class SplitByChaptersPlugin(SplitPlugin[SplitByChaptersOptions]):
sort_dict=False,
)
return entry, metadata
return new_entry, metadata
def split(self, entry: Entry) -> Optional[List[Tuple[Entry, FileMetadata]]]:
"""
@ -166,15 +178,9 @@ class SplitByChaptersPlugin(SplitPlugin[SplitByChaptersOptions]):
# If no chapters, do not split anything
if not chapters.contains_any_chapters():
if self.plugin_options.when_no_chapters == "pass":
entry.add_variables(
{
"chapter_title": entry.title,
"chapter_index": 1,
"chapter_index_padded": "01",
"chapter_count": 1,
}
)
return [(entry, FileMetadata())]
# Modify the entry t
return [(self._non_split_entry(entry), FileMetadata())]
if self.plugin_options.when_no_chapters == "drop":
return []
@ -183,18 +189,16 @@ class SplitByChaptersPlugin(SplitPlugin[SplitByChaptersOptions]):
)
for idx, title in enumerate(chapters.titles):
new_uid = _split_video_uid(source_uid=entry.uid, idx=idx)
new_entry = Entry.create_split_entry(
entry=entry, new_uid=_split_video_uid(source_uid=entry.uid, idx=idx)
)
if not self.is_dry_run:
# Get the input/output file paths
input_file = entry.get_download_file_path()
output_file = str(Path(self.working_directory) / f"{new_uid}.{entry.ext}")
# Run ffmpeg to create the split the video
FFMPEG.run(
_split_video_ffmpeg_cmd(
input_file=input_file,
output_file=output_file,
input_file=entry.get_download_file_path(),
output_file=new_entry.get_download_file_path(),
timestamps=chapters.timestamps,
idx=idx,
)
@ -205,14 +209,13 @@ class SplitByChaptersPlugin(SplitPlugin[SplitByChaptersOptions]):
if entry.is_thumbnail_downloaded():
FileHandler.copy(
src_file_path=entry.get_download_thumbnail_path(),
dst_file_path=Path(self.working_directory)
/ f"{new_uid}.{entry.thumbnail_ext}",
dst_file_path=new_entry.get_download_thumbnail_path(),
)
# Format the split video
split_videos_and_metadata.append(
self._create_split_entry(
source_entry=entry,
new_entry=new_entry,
title=title,
idx=idx,
chapters=chapters,

39
src/ytdl_sub/plugins/subtitles.py
View file

@ -2,11 +2,15 @@ from pathlib import Path
from typing import Dict
from typing import List
from typing import Optional
from typing import Set
from ytdl_sub.config.plugin import Plugin
from ytdl_sub.config.preset_options import OptionsDictValidator
from ytdl_sub.config.plugin.plugin import Plugin
from ytdl_sub.config.plugin.plugin_operation import PluginOperation
from ytdl_sub.config.validators.options import OptionsDictValidator
from ytdl_sub.downloaders.ytdl_options_builder import YTDLOptionsBuilder
from ytdl_sub.entries.entry import Entry
from ytdl_sub.entries.script.variable_definitions import VARIABLES
from ytdl_sub.entries.script.variable_definitions import VariableDefinitions
from ytdl_sub.utils.file_handler import FileHandler
from ytdl_sub.utils.file_handler import FileMetadata
from ytdl_sub.utils.logger import Logger
@ -17,6 +21,8 @@ from ytdl_sub.validators.string_select_validator import StringSelectValidator
from ytdl_sub.validators.validators import BoolValidator
from ytdl_sub.validators.validators import StringListValidator
v: VariableDefinitions = VARIABLES
logger = Logger.get(name="subtitles")
@ -113,13 +119,18 @@ class SubtitleOptions(OptionsDictValidator):
"""
return self._allow_auto_generated_subtitles
def added_source_variables(self) -> List[str]:
def added_variables(
self,
resolved_variables: Set[str],
unresolved_variables: Set[str],
plugin_op: PluginOperation,
) -> Dict[PluginOperation, Set[str]]:
"""
Returns
-------
List of new source variables created by using the subtitles plugin
"""
return ["lang", "subtitles_ext"]
return {PluginOperation.MODIFY_ENTRY_METADATA: {"lang", "subtitles_ext"}}
class SubtitlesPlugin(Plugin[SubtitleOptions]):
@ -158,18 +169,8 @@ class SubtitlesPlugin(Plugin[SubtitleOptions]):
return builder.to_dict()
def modify_entry(self, entry: Entry) -> Optional[Entry]:
if not (requested_subtitles := entry.kwargs_get("requested_subtitles", None)):
return entry
languages = sorted(requested_subtitles.keys())
entry.add_variables(
variables_to_add={
"subtitles_ext": self.plugin_options.subtitles_type,
"lang": ",".join(languages),
}
)
def modify_entry_metadata(self, entry: Entry) -> Optional[Entry]:
entry.add({"subtitles_ext": self.plugin_options.subtitles_type, "lang": ""})
return entry
def post_process_entry(self, entry: Entry) -> Optional[FileMetadata]:
@ -181,7 +182,7 @@ class SubtitlesPlugin(Plugin[SubtitleOptions]):
entry:
Entry to create subtitles for
"""
requested_subtitles = entry.kwargs("requested_subtitles")
requested_subtitles = entry.get(v.requested_subtitles, expected_type=dict)
if not requested_subtitles:
logger.debug("subtitles not found for %s", entry.title)
return None
@ -189,6 +190,10 @@ class SubtitlesPlugin(Plugin[SubtitleOptions]):
file_metadata: Optional[FileMetadata] = None
langs = list(requested_subtitles.keys())
# HACK to maintain order of languages for fixtures
if len(langs) == len(self.plugin_options.languages):
langs = self.plugin_options.languages
if self.plugin_options.embed_subtitles:
file_metadata = FileMetadata(f"Embedded subtitles with lang(s) {', '.join(langs)}")
if self.plugin_options.subtitles_name:

6
src/ytdl_sub/plugins/throttle_protection.py
View file

@ -4,9 +4,9 @@ from typing import List
from typing import Optional
from typing import Tuple
from ytdl_sub.config.plugin import Plugin
from ytdl_sub.config.preset_options import OptionsDictValidator
from ytdl_sub.config.preset_options import Overrides
from ytdl_sub.config.overrides import Overrides
from ytdl_sub.config.plugin.plugin import Plugin
from ytdl_sub.config.validators.options import OptionsDictValidator
from ytdl_sub.entries.entry import Entry
from ytdl_sub.utils.file_handler import FileMetadata
from ytdl_sub.utils.logger import Logger

4
src/ytdl_sub/plugins/video_tags.py
View file

@ -2,8 +2,8 @@ import copy
from typing import Any
from typing import Dict
from ytdl_sub.config.plugin import Plugin
from ytdl_sub.config.preset_options import OptionsDictValidator
from ytdl_sub.config.plugin.plugin import Plugin
from ytdl_sub.config.validators.options import OptionsDictValidator
from ytdl_sub.entries.entry import Entry
from ytdl_sub.utils.ffmpeg import add_ffmpeg_metadata_key_values
from ytdl_sub.utils.file_handler import FileMetadata

0
src/ytdl_sub/script/__init__.py Normal file
View file

82
src/ytdl_sub/script/functions/__init__.py Normal file
View file

@ -0,0 +1,82 @@
from typing import Callable
from typing import Dict
from ytdl_sub.script.functions.array_functions import ArrayFunctions
from ytdl_sub.script.functions.boolean_functions import BooleanFunctions
from ytdl_sub.script.functions.conditional_functions import ConditionalFunctions
from ytdl_sub.script.functions.date_functions import DateFunctions
from ytdl_sub.script.functions.error_functions import ErrorFunctions
from ytdl_sub.script.functions.json_functions import JsonFunctions
from ytdl_sub.script.functions.map_functions import MapFunctions
from ytdl_sub.script.functions.numeric_functions import NumericFunctions
from ytdl_sub.script.functions.regex_functions import RegexFunctions
from ytdl_sub.script.functions.string_functions import StringFunctions
from ytdl_sub.script.types.resolvable import Resolvable
from ytdl_sub.script.utils.exceptions import FunctionDoesNotExistRuntimeException
class Functions(
StringFunctions,
NumericFunctions,
ConditionalFunctions,
ArrayFunctions,
MapFunctions,
BooleanFunctions,
ErrorFunctions,
RegexFunctions,
DateFunctions,
JsonFunctions,
):
_custom_functions: Dict[str, Callable[..., Resolvable]] = {}
@classmethod
def is_built_in(cls, name: str) -> bool:
"""
Returns
-------
True if the name exists as a built-in function or custom function. False otherwise.
"""
return hasattr(cls, name) or hasattr(cls, f"{name}_") or name in cls._custom_functions
@classmethod
def get(cls, name: str) -> Callable[..., Resolvable]:
"""
Returns
-------
The actual Python callable for the function of the given name.
Raises
------
FunctionDoesNotExistRuntimeException
If the function does not exist.
"""
if hasattr(cls, name):
return getattr(cls, name)
if hasattr(cls, f"{name}_"):
return getattr(cls, f"{name}_")
if name in cls._custom_functions:
return cls._custom_functions[name]
raise FunctionDoesNotExistRuntimeException(f"The function {name} does not exist")
@classmethod
def register_function(cls, function: Callable[..., Resolvable]) -> None:
"""
Adds a function to the suite of offered functions.
Parameters
----------
function
A static function whose name will be used as the offered function name.
Raises
------
ValueError
If the name already exists as a function.
"""
if cls.is_built_in(function.__name__):
raise ValueError(
f"Cannot register a function with name {function.__name__} "
f"because it already exists"
)
cls._custom_functions[function.__name__] = function

147
src/ytdl_sub/script/functions/array_functions.py Normal file
View file

@ -0,0 +1,147 @@
import itertools
from typing import List
from typing import Optional
from ytdl_sub.script.types.array import Array
from ytdl_sub.script.types.resolvable import AnyArgument
from ytdl_sub.script.types.resolvable import Boolean
from ytdl_sub.script.types.resolvable import Integer
from ytdl_sub.script.types.resolvable import Lambda
from ytdl_sub.script.types.resolvable import LambdaReduce
from ytdl_sub.script.types.resolvable import LambdaTwo
from ytdl_sub.script.types.resolvable import Resolvable
from ytdl_sub.script.utils.exceptions import UNREACHABLE
from ytdl_sub.script.utils.exceptions import ArrayValueDoesNotExist
from ytdl_sub.script.utils.exceptions import FunctionRuntimeException
class ArrayFunctions:
@staticmethod
def array(maybe_array: AnyArgument) -> Array:
"""
Tries to cast an unknown variable type to an Array.
Raises
------
FunctionRuntimeException
If the input type is not actually an Array.
"""
if not isinstance(maybe_array, Array):
raise FunctionRuntimeException(
f"Tried and failed to cast {maybe_array.type_name()} as an Array"
)
return maybe_array
@staticmethod
def array_size(array: Array) -> Integer:
"""
Returns the size of an Array.
"""
return Integer(len(array.value))
@staticmethod
def array_extend(*arrays: Array) -> Array:
"""
Combine multiple Arrays into a single Array.
"""
output: List[Resolvable] = []
for array in arrays:
output.extend(array.value)
return Array(output)
@staticmethod
def array_at(array: Array, idx: Integer) -> AnyArgument:
"""
Return the element in the Array at index ``idx``.
"""
return array.value[idx.value]
@staticmethod
def array_contains(array: Array, value: AnyArgument) -> Boolean:
"""
Return True if the value exists in the Array. False otherwise.
"""
return Boolean(value in array.value)
@staticmethod
def array_index(array: Array, value: AnyArgument) -> Integer:
"""
Return the index of the value within the Array if it exists. If it does not, it will
throw an error.
"""
if not ArrayFunctions.array_contains(array=array, value=value):
raise ArrayValueDoesNotExist(
"Tried to get the index of a value in an Array that does not exist"
)
if isinstance(value, Resolvable):
return Integer(array.value.index(value))
raise UNREACHABLE
@staticmethod
def array_slice(array: Array, start: Integer, end: Optional[Integer] = None) -> Array:
"""
Returns the slice of the Array.
"""
if end is not None:
return Array(array.value[start.value : end.value])
return Array(array.value[start.value :])
@staticmethod
def array_flatten(array: Array) -> Array:
"""
Flatten any nested Arrays into a single-dimensional Array.
"""
output: List[Resolvable] = []
for elem in array.value:
if isinstance(elem, Array):
output.extend(ArrayFunctions.array_flatten(elem).value)
else:
output.append(elem)
return Array(output)
@staticmethod
def array_reverse(array: Array) -> Array:
"""
Reverse an Array.
"""
return Array(list(reversed(array.value)))
@staticmethod
def array_product(*arrays: Array) -> Array:
"""
Returns the Cartesian product of elements from different arrays
"""
out: List[Resolvable] = []
for combo in itertools.product(*[arr.value for arr in arrays]):
out.append(Array(combo))
return Array(out)
# pylint: disable=unused-argument
@staticmethod
def array_apply(array: Array, lambda_function: Lambda) -> Array:
"""
Apply a lambda function on every element in the Array.
"""
return Array([Array([val]) for val in array.value])
@staticmethod
def array_enumerate(array: Array, lambda_function: LambdaTwo) -> Array:
"""
Apply a lambda function on every element in the Array, where each arg
passed to the lambda function is ``idx, element`` as two separate args.
"""
return Array([Array([Integer(idx), val]) for idx, val in enumerate(array.value)])
@staticmethod
def array_reduce(array: Array, lambda_reduce_function: LambdaReduce) -> AnyArgument:
"""
Apply a reduce function on pairs of elements in the Array, until one element remains.
Executes using the left-most and reduces in the right direction.
"""
return array

89
src/ytdl_sub/script/functions/boolean_functions.py Normal file
View file

@ -0,0 +1,89 @@
from ytdl_sub.script.types.resolvable import AnyArgument
from ytdl_sub.script.types.resolvable import Boolean
# pylint: disable=invalid-name
class BooleanFunctions:
"""
Comparison functions that output Booleans.
"""
@staticmethod
def bool(value: AnyArgument) -> Boolean:
"""
Cast any type to a Boolean.
"""
return Boolean(bool(value.value))
@staticmethod
def eq(left: AnyArgument, right: AnyArgument) -> Boolean:
"""
``==`` operator. Returns True if left == right. False otherwise.
"""
return Boolean(left.value == right.value)
@staticmethod
def ne(left: AnyArgument, right: AnyArgument) -> Boolean:
"""
``!=`` operator. Returns True if left != right. False otherwise.
"""
return Boolean(left.value != right.value)
@staticmethod
def lt(left: AnyArgument, right: AnyArgument) -> Boolean:
"""
``<`` operator. Returns True if left < right. False otherwise.
"""
return Boolean(left.value < right.value)
@staticmethod
def lte(left: AnyArgument, right: AnyArgument) -> Boolean:
"""
``<=`` operator. Returns True if left <= right. False otherwise.
"""
return Boolean(left.value <= right.value)
@staticmethod
def gt(left: AnyArgument, right: AnyArgument) -> Boolean:
"""
``>`` operator. Returns True if left > right. False otherwise.
"""
return Boolean(left.value > right.value)
@staticmethod
def gte(left: AnyArgument, right: AnyArgument) -> Boolean:
"""
``>=`` operator. Returns True if left >= right. False otherwise.
"""
return Boolean(left.value >= right.value)
@staticmethod
def and_(*values: AnyArgument) -> Boolean:
"""
``and`` operator. Returns True if all values evaluate to True. False otherwise.
"""
return Boolean(all(bool(val.value) for val in values))
@staticmethod
def or_(*values: AnyArgument) -> Boolean:
"""
``or`` operator. Returns True if any value evaluates to True. False otherwise.
"""
return Boolean(any(bool(val.value) for val in values))
@staticmethod
def xor(*values: AnyArgument) -> Boolean:
"""
``^`` operator. Returns True if exactly one value is set to True. False otherwise.
"""
bit_array = [bool(val.value) for val in values]
return Boolean(sum(bit_array) == 1)
@staticmethod
def not_(value: Boolean) -> Boolean:
"""
``not`` operator. Returns the opposite of value.
"""
return Boolean(not value.value)

31
src/ytdl_sub/script/functions/conditional_functions.py Normal file
View file

@ -0,0 +1,31 @@
from typing import Union
from ytdl_sub.script.types.resolvable import Boolean
from ytdl_sub.script.types.resolvable import ReturnableArgumentA
from ytdl_sub.script.types.resolvable import ReturnableArgumentB
class ConditionalFunctions:
@staticmethod
def if_(
condition: Boolean, true: ReturnableArgumentA, false: ReturnableArgumentB
) -> Union[ReturnableArgumentA, ReturnableArgumentB]:
"""
Conditional ``if`` statement that returns the ``true`` or ``false`` parameter
depending on the ``condition`` value.
"""
if condition.value:
return true
return false
@staticmethod
def if_passthrough(
maybe_true_arg: ReturnableArgumentA, else_arg: ReturnableArgumentB
) -> Union[ReturnableArgumentA, ReturnableArgumentB]:
"""
Conditional ``if`` statement that returns the ``maybe_true_arg`` if it evaluates to True,
otherwise returns ``else_arg``.
"""
if bool(maybe_true_arg.value):
return maybe_true_arg
return else_arg

13
src/ytdl_sub/script/functions/date_functions.py Normal file
View file

@ -0,0 +1,13 @@
from datetime import datetime
from ytdl_sub.script.types.resolvable import Integer
from ytdl_sub.script.types.resolvable import String
class DateFunctions:
@staticmethod
def datetime_strftime(posix_timestamp: Integer, date_format: String) -> String:
"""
Converts a posix timestamp to a date using strftime formatting.
"""
return String(datetime.utcfromtimestamp(posix_timestamp.value).strftime(date_format.value))

23
src/ytdl_sub/script/functions/error_functions.py Normal file
View file

@ -0,0 +1,23 @@
from ytdl_sub.script.types.resolvable import AnyArgument
from ytdl_sub.script.types.resolvable import ReturnableArgument
from ytdl_sub.script.types.resolvable import String
from ytdl_sub.script.utils.exceptions import UserThrownRuntimeError
class ErrorFunctions:
@staticmethod
def throw(error_message: String) -> AnyArgument:
"""
Explicitly throw an error with the provided error message.
"""
raise UserThrownRuntimeError(error_message)
@staticmethod
def assert_(value: ReturnableArgument, assert_message: String) -> ReturnableArgument:
"""
Explicitly throw an error with the provided assert message if ``value`` evaluates to False.
If it evaluates to True, it will return ``value``.
"""
if not bool(value.value):
raise UserThrownRuntimeError(assert_message)
return value

40
src/ytdl_sub/script/functions/json_functions.py Normal file
View file

@ -0,0 +1,40 @@
import json
from typing import Any
from ytdl_sub.script.types.array import Array
from ytdl_sub.script.types.map import Map
from ytdl_sub.script.types.resolvable import AnyArgument
from ytdl_sub.script.types.resolvable import Boolean
from ytdl_sub.script.types.resolvable import Float
from ytdl_sub.script.types.resolvable import Integer
from ytdl_sub.script.types.resolvable import Resolvable
from ytdl_sub.script.types.resolvable import String
from ytdl_sub.script.utils.exceptions import UNREACHABLE
def _from_json(out: Any) -> Resolvable:
# pylint: disable=too-many-return-statements
if out is None:
return String("")
if isinstance(out, int):
return Integer(out)
if isinstance(out, float):
return Float(out)
if isinstance(out, str):
return String(out)
if isinstance(out, bool):
return Boolean(out)
if isinstance(out, list):
return Array(value=[_from_json(arg) for arg in out])
if isinstance(out, dict):
return Map(value={_from_json(key): _from_json(value) for key, value in out.items()})
raise UNREACHABLE
class JsonFunctions:
@staticmethod
def from_json(argument: String) -> AnyArgument:
"""
Converts a JSON string into an actual type.
"""
return _from_json(json.loads(argument.value))

102
src/ytdl_sub/script/functions/map_functions.py Normal file
View file

@ -0,0 +1,102 @@
from typing import Optional
from ytdl_sub.script.types.array import Array
from ytdl_sub.script.types.map import Map
from ytdl_sub.script.types.resolvable import AnyArgument
from ytdl_sub.script.types.resolvable import Boolean
from ytdl_sub.script.types.resolvable import Hashable
from ytdl_sub.script.types.resolvable import Integer
from ytdl_sub.script.types.resolvable import LambdaThree
from ytdl_sub.script.types.resolvable import LambdaTwo
from ytdl_sub.script.types.resolvable import String
from ytdl_sub.script.utils.exceptions import FunctionRuntimeException
from ytdl_sub.script.utils.exceptions import KeyDoesNotExistRuntimeException
from ytdl_sub.script.utils.exceptions import KeyNotHashableRuntimeException
class MapFunctions:
@staticmethod
def map(maybe_mapping: AnyArgument) -> Map:
"""
Tries to cast an unknown variable type to a Map.
Raises
------
FunctionRuntimeException
If the input type is not actually a Map.
"""
if not isinstance(maybe_mapping, Map):
raise FunctionRuntimeException(
f"Tried and failed to cast {maybe_mapping.type_name()} as a Map"
)
return maybe_mapping
@staticmethod
def map_size(mapping: Map) -> Integer:
"""
Returns the size of a Map.
"""
return Integer(len(mapping.value))
@staticmethod
def map_contains(mapping: Map, key: AnyArgument) -> Boolean:
"""
Returns True if the key is in the Map. False otherwise.
"""
if not isinstance(key, Hashable):
raise KeyNotHashableRuntimeException(
f"Tried to use {key.type_name()} as a Map key, but it is not hashable."
)
return Boolean(key in mapping.value)
@staticmethod
def map_get(
mapping: Map, key: AnyArgument, default: Optional[AnyArgument] = None
) -> AnyArgument:
"""
Return ``key``'s value within the Map. If ``key`` does not exist, and ``default`` is
provided, it will return ``default``. Otherwise, will error.
"""
if not MapFunctions.map_contains(mapping=mapping, key=key).value:
if default is not None:
return default
raise KeyDoesNotExistRuntimeException(
f"Tried to call %map_get with key {key.value}, but it does not exist"
)
return mapping.value[key]
@staticmethod
def map_get_non_empty(mapping: Map, key: AnyArgument, default: AnyArgument) -> AnyArgument:
"""
Return ``key``'s value within the Map. If ``key`` does not exist or is an empty string,
return ``default``. Otherwise, will error.
"""
output = MapFunctions.map_get(mapping, key, default)
if isinstance(output, String) and output.value == "":
return default
return output
# pylint: disable=unused-argument
@staticmethod
def map_apply(mapping: Map, lambda_function: LambdaTwo) -> Array:
"""
Apply a lambda function on the Map, where each arg
passed to the lambda function is ``key, value`` as two separate args.
"""
return Array([Array([key, value]) for key, value in mapping.value.items()])
@staticmethod
def map_enumerate(mapping: Map, lambda_function: LambdaThree) -> Array:
"""
Apply a lambda function on the Map, where each arg
passed to the lambda function is ``idx, key, value`` as three separate args.
"""
return Array(
[
Array([Integer(idx), key_value[0], key_value[1]])
for idx, key_value in enumerate(mapping.value.items())
]
)

88
src/ytdl_sub/script/functions/numeric_functions.py Normal file
View file

@ -0,0 +1,88 @@
import math
from ytdl_sub.script.types.resolvable import AnyArgument
from ytdl_sub.script.types.resolvable import Float
from ytdl_sub.script.types.resolvable import Integer
from ytdl_sub.script.types.resolvable import Numeric
def _to_numeric(value: int | float) -> Numeric:
if int(value) == value:
return Integer(value=value)
return Float(value=value)
class NumericFunctions:
@staticmethod
def float(value: AnyArgument) -> Float:
"""
Cast to Float.
"""
return Float(value=float(value.value))
@staticmethod
def int(value: AnyArgument) -> Integer:
"""
Cast to Integer.
"""
return Integer(value=int(value.value))
@staticmethod
def add(*values: Numeric) -> Numeric:
"""
``+`` operator. Returns the sum of all values.
"""
return _to_numeric(sum(val.value for val in values))
@staticmethod
def sub(*values: Numeric) -> Numeric:
"""
``-`` operator. Subtracts all values from left to right.
"""
output = values[0].value
for val in values[1:]:
output -= val.value
return _to_numeric(output)
@staticmethod
def mul(*values: Numeric) -> Numeric:
"""
``*`` operator. Returns the product of all values.
"""
return _to_numeric(math.prod([val.value for val in values]))
@staticmethod
def pow(base: Numeric, exponent: Numeric) -> Numeric:
"""
``**`` operator. Returns the exponential of the base and exponent value.
"""
return _to_numeric(math.pow(base.value, exponent.value))
@staticmethod
def div(left: Numeric, right: Numeric) -> Numeric:
"""
``/`` operator. Returns ``left / right``.
"""
return _to_numeric(left.value / right.value)
@staticmethod
def mod(left: Numeric, right: Numeric) -> Numeric:
"""
``%`` operator. Returns ``left % right``.
"""
return _to_numeric(value=left.value % right.value)
@staticmethod
def max(*values: Numeric) -> Numeric:
"""
Returns max of all values.
"""
return _to_numeric(max(val.value for val in values))
@staticmethod
def min(*values: Numeric) -> Numeric:
"""
Returns min of all values.
"""
return _to_numeric(min(val.value for val in values))

42
src/ytdl_sub/script/functions/regex_functions.py Normal file
View file

@ -0,0 +1,42 @@
import re
from typing import AnyStr
from typing import Match
from ytdl_sub.script.types.array import Array
from ytdl_sub.script.types.resolvable import String
def _re_output_to_array(re_out: Match[AnyStr] | None) -> Array:
if re_out is None:
return Array([])
return Array(list([String(re_out.string)]) + list(String(group) for group in re_out.groups()))
class RegexFunctions:
@staticmethod
def regex_match(regex: String, string: String) -> Array:
"""
Checks for a match only at the beginning of the string. If a match exists, returns
the string as the first element of the Array. If there are capture groups, returns each
group as a subsequent element in the Array.
"""
return _re_output_to_array(re.match(regex.value, string.value))
@staticmethod
def regex_search(regex: String, string: String) -> Array:
"""
Checks for a match anywhere in the string. If a match exists, returns
the string as the first element of the Array. If there are capture groups, returns each
group as a subsequent element in the Array.
"""
return _re_output_to_array(re.search(regex.value, string.value))
@staticmethod
def regex_fullmatch(regex: String, string: String) -> Array:
"""
Checks for entire string to be a match. If a match exists, returns
the string as the first element of the Array. If there are capture groups, returns each
group as a subsequent element in the Array.
"""
return _re_output_to_array(re.fullmatch(regex.value, string.value))

94
src/ytdl_sub/script/functions/string_functions.py Normal file
View file

@ -0,0 +1,94 @@
from typing import Optional
from ytdl_sub.script.types.resolvable import AnyArgument
from ytdl_sub.script.types.resolvable import Integer
from ytdl_sub.script.types.resolvable import Numeric
from ytdl_sub.script.types.resolvable import String
class StringFunctions:
@staticmethod
def string(value: AnyArgument) -> String:
"""
Cast to String.
"""
return String(value=str(value.value))
@staticmethod
def slice(string: String, start: Integer, end: Optional[Integer] = None) -> String:
"""
Returns the slice of the Array.
"""
if end is not None:
return String(string.value[start.value : end.value])
return String(string.value[start.value :])
@staticmethod
def lower(string: String) -> String:
"""
Lower-case the entire String.
"""
return String(string.value.lower())
@staticmethod
def upper(string: String) -> String:
"""
Upper-case the entire String.
"""
return String(string.value.upper())
@staticmethod
def capitalize(string: String) -> String:
"""
Capitalize the first character in the string.
"""
return String(string.value.capitalize())
@staticmethod
def titlecase(string: String) -> String:
"""
Capitalize each word in the string.
"""
return String(string.value.title())
@staticmethod
def replace(
string: String, old: String, new: String, count: Optional[Integer] = None
) -> String:
"""
Replace the ``old`` part of the String with the ``new``. Optionally only replace it
``count`` number of times.
"""
if count:
return String(string.value.replace(old.value, new.value, count.value))
return String(string.value.replace(old.value, new.value))
@staticmethod
def concat(*values: String) -> String:
"""
Concatenate multiple Strings into a single String.
"""
return String("".join(val.value for val in values))
@staticmethod
def pad(string: String, length: Integer, char: String) -> String:
"""
Pads the string to the given length
"""
output = string.value
while len(output) < length.value:
output = f"{char}{output}"
return String(output)
@staticmethod
def pad_zero(numeric: Numeric, length: Integer) -> String:
"""
Pads a numeric with zeros to the given length
"""
return StringFunctions.pad(
string=String(str(numeric.value)),
length=length,
char=String("0"),
)

601
src/ytdl_sub/script/parser.py Normal file
View file

@ -0,0 +1,601 @@
import json
from enum import Enum
from typing import Dict
from typing import List
from typing import Optional
from typing import Set
from ytdl_sub.script.functions import Functions
from ytdl_sub.script.types.array import UnresolvedArray
from ytdl_sub.script.types.function import Argument
from ytdl_sub.script.types.function import BuiltInFunction
from ytdl_sub.script.types.function import CustomFunction
from ytdl_sub.script.types.function import Function
from ytdl_sub.script.types.map import UnresolvedMap
from ytdl_sub.script.types.resolvable import Boolean
from ytdl_sub.script.types.resolvable import Float
from ytdl_sub.script.types.resolvable import Integer
from ytdl_sub.script.types.resolvable import Lambda
from ytdl_sub.script.types.resolvable import NonHashable
from ytdl_sub.script.types.resolvable import String
from ytdl_sub.script.types.syntax_tree import SyntaxTree
from ytdl_sub.script.types.variable import FunctionArgument
from ytdl_sub.script.types.variable import Variable
from ytdl_sub.script.utils.exception_formatters import ParserExceptionFormatter
from ytdl_sub.script.utils.exceptions import UNREACHABLE
from ytdl_sub.script.utils.exceptions import CycleDetected
from ytdl_sub.script.utils.exceptions import FunctionDoesNotExist
from ytdl_sub.script.utils.exceptions import IncompatibleFunctionArguments
from ytdl_sub.script.utils.exceptions import InvalidCustomFunctionArgumentName
from ytdl_sub.script.utils.exceptions import InvalidSyntaxException
from ytdl_sub.script.utils.exceptions import InvalidVariableName
from ytdl_sub.script.utils.exceptions import UserException
from ytdl_sub.script.utils.exceptions import VariableDoesNotExist
from ytdl_sub.script.utils.name_validation import validate_variable_name
# pylint: disable=invalid-name
# pylint: disable=too-many-branches
# pylint: disable=too-many-return-statements
# pylint: disable=consider-using-ternary
class ParsedArgType(Enum):
SCRIPT = "script"
FUNCTION = "function"
ARRAY = "array"
MAP_KEY = "map key"
MAP_VALUE = "map value"
BRACKET_NOT_CLOSED = InvalidSyntaxException("Bracket not properly closed")
NUMERICS_ONLY_ARGS = InvalidSyntaxException(
"Numerics can only be used as arguments to functions, maps, or arrays"
)
NUMERICS_INVALID_CHAR = InvalidSyntaxException("Invalid value when parsing a numeric")
STRINGS_ONLY_ARGS = InvalidSyntaxException(
"Strings can only be used as arguments to functions, maps, or arrays"
)
STRINGS_NOT_CLOSED = InvalidSyntaxException(
"String was not closed properly. "
"Must open and close with the same type of quote (single/double)"
)
BOOLEAN_ONLY_ARGS = InvalidSyntaxException(
"Booleans can only be used as arguments to functions, maps, or arrays"
)
CUSTOM_FUNCTION_ARGUMENTS_ONLY_ARGS = InvalidSyntaxException(
"Custom function arguments can only be used as arguments to functions, maps, or arrays"
)
FUNCTION_INVALID_CHAR = InvalidSyntaxException("Invalid value when parsing a function")
def _UNEXPECTED_CHAR_ARGUMENT(arg_type: ParsedArgType):
return InvalidSyntaxException(f"Unexpected character when parsing {arg_type.value} arguments")
def _UNEXPECTED_COMMA_ARGUMENT(arg_type: ParsedArgType):
return InvalidSyntaxException(f"Unexpected comma when parsing {arg_type.value} arguments")
MAP_KEY_WITH_NO_VALUE = InvalidSyntaxException("Map has a key with no value")
MAP_KEY_MULTIPLE_VALUES = InvalidSyntaxException(
"Map key has multiple values when there should only be one"
)
MAP_MISSING_KEY = InvalidSyntaxException("Map has a missing key")
MAP_KEY_NOT_HASHABLE = InvalidSyntaxException(
"Map key must be a hashable type (Integer, Float, Boolean, String)"
)
def _is_variable_start(char: str) -> bool:
return char.isalpha() and char.islower()
def _is_function_name_char(char: str) -> bool:
return (char.isalpha() and char.islower()) or char.isnumeric() or char == "_"
def _is_numeric_start(char: str) -> bool:
return char.isnumeric() or char in (".", "-")
def _is_string_start_single_char(char: Optional[str]) -> bool:
return char in ["'", '"']
def _is_string_start_multi_char(string: Optional[str]) -> bool:
return string in ["'''", '"""']
def _is_null(string: Optional[str]) -> bool:
return string and string.lower() == "null"
def _is_breakable(char: str) -> bool:
return char in ["}", ",", ")", "]", ":"] or char.isspace()
def _is_boolean_true(string: Optional[str]) -> bool:
return string and string.lower() == "true"
def _is_boolean_false(string: Optional[str]) -> bool:
return string and string.lower() == "false"
def _is_custom_function_argument_start(char: str) -> bool:
return char == "$"
class _Parser:
def __init__(
self,
text: str,
name: Optional[str],
custom_function_names: Optional[Set[str]],
variable_names: Optional[Set[str]],
):
self._text = text
self._name = name
self._custom_function_names = custom_function_names
self._variable_names = variable_names
self._pos = 0
self._error_highlight_pos = 0
self._ast: List[Argument] = []
self._bracket_counter_pos_stack: List[int] = []
self._bracket_counter = 0
self._literal_str = ""
try:
self._syntax_tree = self._parse()
except UserException as exc:
raise ParserExceptionFormatter(
self._text, self._error_highlight_pos, self._pos, exc
).highlight() from exc
@property
def ast(self) -> SyntaxTree:
"""
Returns
-------
Abstract syntax tree of the parsed text
"""
return self._syntax_tree
def _set_highlight_position(self, pos: Optional[int] = None) -> None:
self._error_highlight_pos = pos if pos is not None else self._pos
def _read(self, increment_pos: bool = True, length: int = 1) -> Optional[str]:
if isinstance(self._text, int):
pass
if self._pos >= len(self._text):
return None
try:
ch = self._text[self._pos : (self._pos + length)]
except IndexError:
return None
if increment_pos:
self._pos += length
return ch
def _parse_variable(self) -> Variable:
var_name = ""
variable_start_pos = self._pos
while ch := self._read(increment_pos=False):
if ch.isspace() and not var_name:
self._pos += 1
continue
if _is_breakable(ch):
break
if not var_name:
variable_start_pos = self._pos
var_name += ch
self._pos += 1
try:
validate_variable_name(var_name)
except InvalidVariableName:
self._set_highlight_position(variable_start_pos)
raise
if self._variable_names is not None and var_name not in self._variable_names:
self._set_highlight_position(variable_start_pos)
raise VariableDoesNotExist(f"Variable {var_name} does not exist.")
return Variable(var_name)
def _parse_custom_function_argument(self) -> FunctionArgument:
"""
Begin parsing function args after the first ``$``, i.e. ``$0``
"""
var_name = ""
variable_start_pos = self._pos
while ch := self._read(increment_pos=False):
if ch.isspace() and not var_name:
raise InvalidCustomFunctionArgumentName(
"Custom function arguments, denoted by $, cannot have a space proceeding it."
)
if _is_breakable(ch):
break
var_name += ch
self._pos += 1
if not var_name.isnumeric():
self._set_highlight_position(variable_start_pos)
raise InvalidCustomFunctionArgumentName(
"Custom function arguments must be numeric and increment starting from zero."
)
return FunctionArgument.from_idx(idx=int(var_name), custom_function_name=self._name)
def _parse_numeric(self) -> Integer | Float:
numeric_string = ""
if self._read(increment_pos=False) == "-":
numeric_string += "-"
self._pos += 1
if has_decimal := (self._read(increment_pos=False) == "."):
numeric_string += "."
self._pos += 1
while ch := self._read(increment_pos=False):
if ch == "-":
raise NUMERICS_INVALID_CHAR
if ch == ".":
if has_decimal:
raise NUMERICS_INVALID_CHAR
has_decimal = True
self._pos += 1
numeric_string += ch
elif ch.isnumeric():
self._pos += 1
numeric_string += ch
elif _is_breakable(ch):
break
else:
self._set_highlight_position()
raise NUMERICS_INVALID_CHAR
if numeric_string in (".", "-"):
raise NUMERICS_INVALID_CHAR
try:
numeric_float = float(numeric_string)
except ValueError as exc:
raise UNREACHABLE from exc
if (numeric_int := int(numeric_float)) == numeric_float:
return Integer(value=numeric_int)
return Float(value=numeric_float)
def _parse_string(self, str_open_token: str) -> String:
"""
Begin parsing a string, including the quotation value
"""
self._set_highlight_position()
string_value = ""
if not _is_string_start_single_char(str_open_token) and not _is_string_start_multi_char(
str_open_token
):
raise UNREACHABLE
while ch := self._read(increment_pos=False):
if self._read(increment_pos=False, length=len(str_open_token)) == str_open_token:
self._pos += len(str_open_token)
return String(value=string_value)
self._pos += 1
string_value += ch
raise STRINGS_NOT_CLOSED
def _parse_function_arg(self, argument_parser: ParsedArgType) -> Argument:
if self._read(increment_pos=False) == "%":
self._pos += 1
return self._parse_function()
if _is_numeric_start(self._read(increment_pos=False)):
return self._parse_numeric()
if _is_boolean_true(self._read(increment_pos=False, length=4)):
self._pos += 4
return Boolean(value=True)
if _is_boolean_false(self._read(increment_pos=False, length=5)):
self._pos += 5
return Boolean(value=False)
if _is_null(self._read(increment_pos=False, length=4)):
self._pos += 4
return String(value="")
if _is_string_start_multi_char(str_open_token := self._read(increment_pos=False, length=3)):
self._pos += 3
return self._parse_string(str_open_token)
if _is_string_start_single_char(str_open_token := self._read(increment_pos=False)):
self._pos += 1
return self._parse_string(str_open_token)
if self._read(increment_pos=False) == "[":
self._pos += 1
return self._parse_array()
if self._read(increment_pos=False) == "{":
self._pos += 1
return self._parse_map()
if _is_custom_function_argument_start(self._read(increment_pos=False)):
self._pos += 1
return self._parse_custom_function_argument()
if _is_variable_start(self._read(increment_pos=False)):
return self._parse_variable()
self._set_highlight_position()
raise _UNEXPECTED_CHAR_ARGUMENT(arg_type=argument_parser)
def _parse_args(
self, argument_parser: ParsedArgType, breaking_chars: str = ")"
) -> List[Argument]:
"""
Begin parsing function args after the first ``(``, i.e. ``function_name(``
"""
comma_count = 0
arguments: List[Argument] = []
while ch := self._read(increment_pos=False):
if ch in breaking_chars:
# i.e. ["arg", ] which is invalid
if arguments and len(arguments) == comma_count:
raise _UNEXPECTED_COMMA_ARGUMENT(argument_parser)
break
if ch.isspace():
self._pos += 1
elif ch == ",":
self._set_highlight_position()
comma_count += 1
if len(arguments) != comma_count:
raise _UNEXPECTED_COMMA_ARGUMENT(argument_parser)
self._pos += 1
else:
arguments.append(self._parse_function_arg(argument_parser=argument_parser))
return arguments
def _parse_function(self) -> Function | Lambda:
"""
Begin parsing a function after reading the first ``%``
"""
function_name: str = ""
function_args: Optional[List[Argument]] = None
function_start_pos = self._pos
while ch := self._read():
if ch == ")":
# Had '(' to indicate there are args
if function_args is not None:
if self._name == function_name:
self._set_highlight_position(function_start_pos)
raise CycleDetected(
f"The custom function %{function_name} cannot call itself."
)
if Functions.is_built_in(function_name):
try:
return BuiltInFunction(
name=function_name, args=function_args
).validate_args()
except IncompatibleFunctionArguments:
self._set_highlight_position(function_start_pos)
raise
# Is custom function
if (
self._custom_function_names is not None
and function_name not in self._custom_function_names
):
self._set_highlight_position(function_start_pos)
raise FunctionDoesNotExist(
f"Function %{function_name} does not exist as a built-in or "
"custom function."
)
return CustomFunction(
name=function_name,
args=function_args,
)
# Go back one so the parent function can close using the ')'
self._pos -= 1
return Lambda(value=function_name)
if _is_function_name_char(ch):
function_name += ch
elif ch == "(":
function_args = self._parse_args(argument_parser=ParsedArgType.FUNCTION)
elif ch.isspace() or ch == ",":
# function with no args, it's a lambda
return Lambda(value=function_name)
else:
break
self._set_highlight_position(pos=self._pos - 1)
raise FUNCTION_INVALID_CHAR
def _parse_array(self) -> UnresolvedArray:
"""
Begin parsing an array after reading the first ``[``
"""
function_args: List[Argument] = []
while ch := self._read(increment_pos=False):
if ch == "]":
self._pos += 1
return UnresolvedArray(value=function_args)
function_args = self._parse_args(
argument_parser=ParsedArgType.ARRAY, breaking_chars="]"
)
raise UNREACHABLE
def _parse_map(self) -> UnresolvedMap:
"""
Begin parsing a map after reading the first ``{``
"""
output: Dict[Argument, Argument] = {}
key: Optional[Argument] = None
in_comma = False
self._set_highlight_position()
while ch := self._read(increment_pos=False):
if ch == "}":
if key is not None:
raise MAP_KEY_WITH_NO_VALUE # key args are parsed immediately
self._pos += 1
return UnresolvedMap(value=output)
if ch == ",":
if in_comma:
raise _UNEXPECTED_COMMA_ARGUMENT(ParsedArgType.MAP_KEY)
if key is not None:
raise MAP_KEY_WITH_NO_VALUE # key args are parsed immediately
if not output:
raise _UNEXPECTED_COMMA_ARGUMENT(ParsedArgType.MAP_KEY)
in_comma = True
self._pos += 1
elif key is None:
self._set_highlight_position()
in_comma = False
key_args = self._parse_args(
argument_parser=ParsedArgType.MAP_KEY, breaking_chars=":}"
)
if len(key_args) == 0 and self._read(increment_pos=False) == "}":
continue # will return the map next iteration
if len(key_args) == 0:
raise MAP_MISSING_KEY
if len(key_args) > 1:
raise MAP_KEY_MULTIPLE_VALUES
key = key_args[0]
elif key is not None and ch == ":":
self._set_highlight_position()
self._pos += 1
value_args = self._parse_args(
argument_parser=ParsedArgType.MAP_VALUE, breaking_chars=",}"
)
if len(value_args) == 0:
raise MAP_KEY_WITH_NO_VALUE
if isinstance(key, NonHashable):
raise MAP_KEY_NOT_HASHABLE
if len(value_args) > 1:
raise MAP_KEY_MULTIPLE_VALUES
output[key] = value_args[0]
key = None
else:
raise UNREACHABLE
def _parse_main_loop(self, ch: str) -> bool:
if ch == "}":
if self._bracket_counter == 0:
raise BRACKET_NOT_CLOSED
del self._bracket_counter_pos_stack[-1]
self._bracket_counter -= 1
return True
if ch == "{":
self._bracket_counter_pos_stack.append(self._pos - 1) # pos incremented when read
self._bracket_counter += 1
if self._literal_str:
self._ast.append(String(value=self._literal_str))
self._literal_str = ""
# Allow whitespace after bracket opening
while ch1 := self._read(increment_pos=False):
if not ch1.isspace():
break
self._pos += 1
if ch1 is None:
return False # will hit closing bracket error
if ch1 == "%":
self._pos += 1
self._ast.append(self._parse_function())
elif ch1 == "[":
self._pos += 1
self._ast.append(self._parse_array())
elif ch1 == "{":
self._pos += 1
self._ast.append(self._parse_map())
elif _is_variable_start(ch1):
self._ast.append(self._parse_variable())
elif _is_numeric_start(ch1):
raise NUMERICS_ONLY_ARGS
elif (
_is_string_start_single_char(ch1)
or _is_string_start_multi_char(self._read(increment_pos=False, length=3))
or _is_null(self._read(increment_pos=False, length=4))
):
raise STRINGS_ONLY_ARGS
elif _is_boolean_true(self._read(increment_pos=False, length=4)) or _is_boolean_false(
self._read(increment_pos=False, length=5)
):
raise BOOLEAN_ONLY_ARGS
elif _is_custom_function_argument_start(self._read(increment_pos=False)):
raise CUSTOM_FUNCTION_ARGUMENTS_ONLY_ARGS
else:
raise _UNEXPECTED_CHAR_ARGUMENT(arg_type=ParsedArgType.SCRIPT)
elif self._bracket_counter == 0:
# Only accumulate literal str if not in brackets
self._literal_str += ch
else:
# Should only be possible to get here if it's a space
assert ch.isspace()
return True
def _parse(self) -> SyntaxTree:
while ch := self._read():
continue_parse = self._parse_main_loop(ch)
if not continue_parse:
break
if self._bracket_counter != 0:
self._error_highlight_pos = self._bracket_counter_pos_stack[-1]
raise BRACKET_NOT_CLOSED
if self._literal_str:
self._ast.append(String(value=self._literal_str))
return SyntaxTree(ast=self._ast)
def parse(
text: str,
name: Optional[str] = None,
custom_function_names: Optional[Set[str]] = None,
variable_names: Optional[Set[str]] = None,
) -> SyntaxTree:
"""
Entrypoint for parsing ytdl-sub code into a Syntax Tree
"""
return _Parser(
text=json.dumps(text) if not isinstance(text, str) else text,
name=name,
custom_function_names=custom_function_names,
variable_names=variable_names,
).ast
# pylint: enable=invalid-name

484
src/ytdl_sub/script/script.py Normal file
View file

@ -0,0 +1,484 @@
# pylint: disable=missing-raises-doc
from typing import Dict
from typing import List
from typing import Optional
from typing import Set
from ytdl_sub.script.functions import Functions
from ytdl_sub.script.parser import parse
from ytdl_sub.script.script_output import ScriptOutput
from ytdl_sub.script.types.resolvable import Lambda
from ytdl_sub.script.types.resolvable import Resolvable
from ytdl_sub.script.types.syntax_tree import SyntaxTree
from ytdl_sub.script.types.variable import Variable
from ytdl_sub.script.utils.exceptions import UNREACHABLE
from ytdl_sub.script.utils.exceptions import CycleDetected
from ytdl_sub.script.utils.exceptions import IncompatibleFunctionArguments
from ytdl_sub.script.utils.exceptions import InvalidCustomFunctionArguments
from ytdl_sub.script.utils.exceptions import RuntimeException
from ytdl_sub.script.utils.name_validation import validate_variable_name
from ytdl_sub.script.utils.type_checking import FunctionSpec
def _is_function(override_name: str):
return override_name.startswith("%")
def _function_name(function_key: str) -> str:
"""
Drop the % in %custom_function
"""
return function_key[1:]
def _to_function_definition_name(function_key: str) -> str:
"""
Add % in %custom_function
"""
return f"%{function_key}"
class Script:
"""
Takes a dictionary of both
``{ variable_names: syntax }``
and
``{ %custom_function: syntax }``
"""
def _ensure_no_cycle(
self, name: str, dep: str, deps: List[str], definitions: Dict[str, SyntaxTree]
):
if dep not in definitions:
return # does not exist, will throw downstream in parser
if name in deps + [dep]:
type_name, pre = (
("custom functions", "%") if definitions is self._functions else ("variables", "")
)
cycle_deps = [name] + deps + [dep]
cycle_deps_str = " -> ".join([f"{pre}{name}" for name in cycle_deps])
raise CycleDetected(f"Cycle detected within these {type_name}: {cycle_deps_str}")
def _traverse_variable_dependencies(
self,
variable_name: str,
variable_dependency: SyntaxTree,
deps: List[str],
) -> None:
for dep in variable_dependency.variables:
self._ensure_no_cycle(
name=variable_name, dep=dep.name, deps=deps, definitions=self._variables
)
self._traverse_variable_dependencies(
variable_name=variable_name,
variable_dependency=self._variables[dep.name],
deps=deps + [dep.name],
)
def _ensure_no_variable_cycles(self, variables: Dict[str, SyntaxTree]):
for variable_name, variable_definition in variables.items():
self._traverse_variable_dependencies(
variable_name=variable_name,
variable_dependency=variable_definition,
deps=[],
)
def _traverse_custom_function_dependencies(
self,
custom_function_name: str,
custom_function_dependency: SyntaxTree,
deps: List[str],
) -> None:
for dep in custom_function_dependency.custom_functions:
self._ensure_no_cycle(
name=custom_function_name, dep=dep.name, deps=deps, definitions=self._functions
)
self._traverse_custom_function_dependencies(
custom_function_name=custom_function_name,
custom_function_dependency=self._functions[dep.name],
deps=deps + [dep.name],
)
def _ensure_no_custom_function_cycles(self):
for custom_function_name, custom_function in self._functions.items():
self._traverse_custom_function_dependencies(
custom_function_name=custom_function_name,
custom_function_dependency=custom_function,
deps=[],
)
def _ensure_custom_function_arguments_valid(self):
for custom_function_name, custom_function in self._functions.items():
indices = sorted([arg.index for arg in custom_function.function_arguments])
if indices != list(range(len(indices))):
if len(indices) == 1:
raise InvalidCustomFunctionArguments(
f"Custom function %{custom_function_name} has invalid function arguments: "
f"The argument must start with $0, not ${indices[0]}."
)
raise InvalidCustomFunctionArguments(
f"Custom function %{custom_function_name} has invalid function arguments: "
f"{', '.join(sorted(f'${idx}' for idx in indices))} "
f"do not increment from $0 to ${len(indices) - 1}."
)
def _ensure_custom_function_usage_num_input_arguments_valid(
self, prefix: str, name: str, definition: SyntaxTree
):
for nested_custom_function in definition.custom_functions:
if nested_custom_function.num_input_args != (
expected_num_args := len(
self._functions[nested_custom_function.name].function_arguments
)
):
raise InvalidCustomFunctionArguments(
f"{prefix}{name} has invalid usage of the custom "
f"function %{nested_custom_function.name}: Expects {expected_num_args} "
f"argument{'s' if expected_num_args > 1 else ''} but received "
f"{nested_custom_function.num_input_args}"
)
def _ensure_lambda_usage_num_input_arguments_valid(
self, prefix: str, name: str, definition: SyntaxTree
):
for function in definition.built_in_functions:
spec = FunctionSpec.from_callable(Functions.get(function.name))
if not (lambda_type := spec.is_lambda_like):
return
lambda_function_names = set(
lamb.value for lamb in SyntaxTree(function.args).lambdas if isinstance(lamb, Lambda)
)
# Only case len(lambda_function_names) > 1 is when used in if-statements
for lambda_function_name in lambda_function_names:
if Functions.is_built_in(lambda_function_name):
lambda_spec = FunctionSpec.from_callable(Functions.get(lambda_function_name))
if not lambda_spec.is_num_args_compatible(lambda_type.num_input_args()):
expected_args_str = str(lambda_spec.num_required_args)
if lambda_spec.num_required_args != len(lambda_spec.args):
expected_args_str = f"{expected_args_str} - {len(lambda_spec.args)}"
raise IncompatibleFunctionArguments(
f"{prefix}{name} has invalid usage of the "
f"function %{lambda_function_name} as a lambda: "
f"Expects {expected_args_str} "
f"argument{'s' if expected_args_str != '1' else ''} but will "
f"receive {lambda_type.num_input_args()}."
)
else: # is custom function
if lambda_function_name not in self._functions:
raise UNREACHABLE # Custom function should have been validated
expected_num_arguments = len(
self._functions[lambda_function_name].function_arguments
)
if lambda_type.num_input_args() != expected_num_arguments:
raise IncompatibleFunctionArguments(
f"{prefix}{name} has invalid usage of the custom "
f"function %{lambda_function_name} as a lambda: "
f"Expects {expected_num_arguments} "
f"argument{'s' if expected_num_arguments > 1 else ''} but will "
f"receive {lambda_type.num_input_args()}."
)
def _validate(self, added_variables: Optional[Set[str]] = None) -> None:
variables = self._variables
if added_variables is not None:
variables = {
name: ast for name, ast in self._variables.items() if name in added_variables
}
if added_variables is None:
self._ensure_no_custom_function_cycles()
self._ensure_custom_function_arguments_valid()
self._ensure_no_variable_cycles(variables)
to_validate = [("Variable ", variables)]
if added_variables is None:
to_validate.append(("Custom function %", self._functions))
for prefix, definitions in to_validate:
for name, definition in definitions.items():
self._ensure_custom_function_usage_num_input_arguments_valid(
prefix=prefix, name=name, definition=definition
)
self._ensure_lambda_usage_num_input_arguments_valid(
prefix=prefix, name=name, definition=definition
)
def __init__(self, script: Dict[str, str]):
function_names: Set[str] = {
_function_name(name) for name in script.keys() if _is_function(name)
}
variable_names: Set[str] = {
validate_variable_name(name) for name in script.keys() if not _is_function(name)
}
self._functions: Dict[str, SyntaxTree] = {
# custom_function_name must be passed to properly type custom function
# arguments uniquely if they're nested (i.e. $0 to $custom_func___0)
_function_name(function_key): parse(
text=function_value,
name=_function_name(function_key),
custom_function_names=function_names,
variable_names=variable_names,
)
for function_key, function_value in script.items()
if _is_function(function_key)
}
self._variables: Dict[str, SyntaxTree] = {
variable_key: parse(
text=variable_value,
name=variable_key,
custom_function_names=function_names,
variable_names=variable_names,
)
for variable_key, variable_value in script.items()
if not _is_function(variable_key)
}
self._validate()
def _update_internally(self, resolved_variables: Dict[str, Resolvable]) -> None:
for variable_name, resolved in resolved_variables.items():
self._variables[variable_name] = SyntaxTree(ast=[resolved])
def _resolve(
self,
pre_resolved: Optional[Dict[str, Resolvable]] = None,
unresolvable: Optional[Set[str]] = None,
update: bool = False,
output_filter: Optional[Set[str]] = None,
) -> ScriptOutput:
"""
Parameters
----------
pre_resolved
Optional. Variables that have been resolved elsewhere and could be used in this script
unresolvable
Optional. Variables that cannot be resolved, forcing any variable that depends on it
to not be resolved.
update
Optional. Whether to update the internal representation of variables with their
resolved value (if they get resolved).
Returns
-------
Dict of resolved values
"""
resolved: Dict[Variable, Resolvable] = {
Variable(name): value for name, value in (pre_resolved or {}).items()
}
unresolvable: Set[Variable] = {Variable(name) for name in (unresolvable or {})}
unresolved_filter = set(resolved.keys()).union(unresolvable)
unresolved: Dict[Variable, SyntaxTree] = {
Variable(name): ast
for name, ast in self._variables.items()
if Variable(name) not in unresolved_filter
}
while unresolved:
unresolved_count: int = len(unresolved)
for variable in list(unresolved.keys()):
definition = unresolved[variable]
# If the definition is already a resolvable, mark it as such
if resolvable := definition.maybe_resolvable:
resolved[variable] = resolvable
del unresolved[variable]
# If the variable's variable dependencies contain an unresolvable variable,
# declare it as unresolvable and continue
elif definition.contains(unresolvable):
unresolvable.add(variable)
del unresolved[variable]
# Otherwise, if it has dependencies that are all resolved, then
# resolve the definition
elif not definition.is_subset_of(variables=resolved.keys()):
resolved[variable] = unresolved[variable].resolve(
resolved_variables=resolved,
custom_functions=self._functions,
)
del unresolved[variable]
if len(unresolved) == unresolved_count:
# Implies a cycle within the variables. Should never reach
# since cycles are detected in __init__
raise UNREACHABLE
resolved_variables = {
variable.name: resolvable for variable, resolvable in resolved.items()
}
if update:
self._update_internally(resolved_variables=resolved_variables)
if output_filter:
for name in output_filter:
if name not in resolved_variables:
raise ValueError(f"Specified {name} to resolve, but it did not")
return ScriptOutput(
{
name: resolvable
for name, resolvable in resolved_variables.items()
if name in output_filter
}
)
return ScriptOutput(resolved_variables)
def resolve(
self,
resolved: Optional[Dict[str, Resolvable]] = None,
unresolvable: Optional[Set[str]] = None,
update: bool = False,
) -> ScriptOutput:
"""
Resolves the script
Parameters
----------
resolved
Optional. Pre-resolved variables that should be used instead of what is in the script.
unresolvable
Optional. Unresolvable variables that will be ignored in resolution, including all
variables with a dependency to them.
update
Whether to update the script's internal values with the resolved variables instead of
their original definition. This helps avoid re-evaluated the same variables repeatedly.
Returns
-------
ScriptOutput
Containing all resolved variables.
"""
return self._resolve(
pre_resolved=resolved, unresolvable=unresolvable, update=update, output_filter=None
)
def add(self, variables: Dict[str, str], unresolvable: Optional[Set[str]] = None) -> "Script":
"""
Adds parses and adds new variables to the script.
Parameters
----------
variables
Mapping containing variable name to definition.
unresolvable
Optional. Set of unresolved variables that the new variables may contain, but the
script does not (yet).
Returns
-------
Script
self
"""
added_variables_to_validate: Set[str] = set()
for variable_name, variable_definition in variables.items():
self._variables[variable_name] = parse(
text=variable_definition,
name=variable_name,
custom_function_names=set(self._functions.keys()),
variable_names=set(self._variables.keys())
.union(variables.keys())
.union(unresolvable or set()),
)
if self._variables[variable_name].maybe_resolvable is None:
added_variables_to_validate.add(variable_name)
if added_variables_to_validate:
self._validate(added_variables=added_variables_to_validate)
return self
def resolve_once(
self,
variable_definitions: Dict[str, str],
resolved: Optional[Dict[str, Resolvable]] = None,
unresolvable: Optional[Set[str]] = None,
) -> Dict[str, Resolvable]:
"""
Given a new set of variable definitions, resolve them using the Script, but do not
add them to the Script itself.
Parameters
----------
variable_definitions
Variables to resolve, but not store in the Script
resolved
Optional. Pre-resolved variables that should be used instead of what is in the script.
unresolvable
Optional. Unresolvable variables that will be ignored in resolution, including all
variables with a dependency to them.
Returns
-------
Dict[str, Resolvable]
Dict containing the variable names to their resolved values.
"""
try:
self.add(variable_definitions)
return self._resolve(
pre_resolved=resolved,
unresolvable=unresolvable,
output_filter=set(list(variable_definitions.keys())),
).output
finally:
for name in variable_definitions.keys():
if name in self._variables:
del self._variables[name]
def get(self, variable_name: str) -> Resolvable:
"""
Parameters
----------
variable_name
Name of the resolved variable to get.
Returns
-------
Resolvable
The resolved variable of the given name.
Raises
------
RuntimeException
If the variable has not been resolved yet in the Script.
"""
if variable_name not in self._variables:
raise RuntimeException(
f"Tried to get resolved variable {variable_name}, but it does not exist"
)
if (resolvable := self._variables[variable_name].maybe_resolvable) is not None:
return resolvable
raise RuntimeException(f"Tried to get unresolved variable {variable_name}")
@property
def variable_names(self) -> Set[str]:
"""
Returns
-------
Set[str]
Names of all the variables within the Script.
"""
return set(list(self._variables.keys()))
@property
def function_names(self) -> Set[str]:
"""
Returns
-------
Set[str]
Names of all functions within the Script.
"""
return set(_to_function_definition_name(name) for name in self._functions.keys())

52
src/ytdl_sub/script/script_output.py Normal file
View file

@ -0,0 +1,52 @@
from dataclasses import dataclass
from typing import Any
from typing import Dict
from ytdl_sub.script.types.resolvable import Resolvable
from ytdl_sub.script.utils.exceptions import ScriptVariableNotResolved
@dataclass(frozen=True)
class ScriptOutput:
output: Dict[str, Resolvable]
def as_native(self) -> Dict[str, Any]:
"""
Returns
-------
The script output as native python types
"""
return {name: out.native for name, out in self.output.items()}
def get(self, name: str) -> Resolvable:
"""
Returns
-------
The script output's variable as a resolvable type
Raises
------
ScriptVariableNotResolved
The variable name requested did not resolve
"""
if name not in self.output:
raise ScriptVariableNotResolved(
f"Tried to access resolved variable {name}, but it has not resolved"
)
return self.output[name]
def get_native(self, name: str) -> Any:
"""
Returns
-------
The script output's variable as native python type
"""
return self.get(name).native
def get_str(self, name: str) -> str:
"""
Returns
-------
The script output's variable as a string
"""
return str(self.get(name))

0
src/ytdl_sub/script/types/__init__.py Normal file
View file

58
src/ytdl_sub/script/types/array.py Normal file
View file

@ -0,0 +1,58 @@
from abc import ABC
from dataclasses import dataclass
from typing import Any
from typing import Dict
from typing import List
from typing import Type
from ytdl_sub.script.types.resolvable import Argument
from ytdl_sub.script.types.resolvable import FutureResolvable
from ytdl_sub.script.types.resolvable import NonHashable
from ytdl_sub.script.types.resolvable import Resolvable
from ytdl_sub.script.types.resolvable import ResolvableToJson
from ytdl_sub.script.types.variable import Variable
from ytdl_sub.script.types.variable_dependency import VariableDependency
@dataclass(frozen=True)
class _Array(NonHashable, ABC):
value: List[Resolvable]
@classmethod
def type_name(cls) -> str:
return "Array"
@dataclass(frozen=True)
class UnresolvedArray(_Array, VariableDependency, FutureResolvable):
value: List[Argument]
@property
def _iterable_arguments(self) -> List[Argument]:
return self.value
def resolve(
self,
resolved_variables: Dict[Variable, Resolvable],
custom_functions: Dict[str, "VariableDependency"],
) -> Resolvable:
return Array(
[
self._resolve_argument_type(
arg=arg,
resolved_variables=resolved_variables,
custom_functions=custom_functions,
)
for arg in self.value
]
)
def future_resolvable_type(self) -> Type[Resolvable]:
return Array
@dataclass(frozen=True)
class Array(_Array, ResolvableToJson):
@property
def native(self) -> Any:
return [val.native for val in self.value]

288
src/ytdl_sub/script/types/function.py Normal file
View file

@ -0,0 +1,288 @@
import copy
import functools
from abc import ABC
from dataclasses import dataclass
from typing import Callable
from typing import Dict
from typing import List
from typing import Type
from typing import Union
from ytdl_sub.script.functions import Functions
from ytdl_sub.script.types.array import Array
from ytdl_sub.script.types.array import UnresolvedArray
from ytdl_sub.script.types.resolvable import Argument
from ytdl_sub.script.types.resolvable import BuiltInFunctionType
from ytdl_sub.script.types.resolvable import FunctionType
from ytdl_sub.script.types.resolvable import FutureResolvable
from ytdl_sub.script.types.resolvable import Lambda
from ytdl_sub.script.types.resolvable import NamedCustomFunction
from ytdl_sub.script.types.resolvable import Resolvable
from ytdl_sub.script.types.resolvable import ReturnableArgument
from ytdl_sub.script.types.resolvable import ReturnableArgumentA
from ytdl_sub.script.types.resolvable import ReturnableArgumentB
from ytdl_sub.script.types.variable import FunctionArgument
from ytdl_sub.script.types.variable import Variable
from ytdl_sub.script.types.variable_dependency import VariableDependency
from ytdl_sub.script.utils.exception_formatters import FunctionArgumentsExceptionFormatter
from ytdl_sub.script.utils.exceptions import UNREACHABLE
from ytdl_sub.script.utils.exceptions import FunctionRuntimeException
from ytdl_sub.script.utils.exceptions import RuntimeException
from ytdl_sub.script.utils.exceptions import UserThrownRuntimeError
from ytdl_sub.script.utils.type_checking import FunctionSpec
from ytdl_sub.script.utils.type_checking import is_union
@dataclass(frozen=True)
class Function(FunctionType, VariableDependency, ABC):
@property
def _iterable_arguments(self) -> List[Argument]:
return self.args
class CustomFunction(Function, NamedCustomFunction):
def resolve(
self,
resolved_variables: Dict[Variable, Resolvable],
custom_functions: Dict[str, "VariableDependency"],
) -> Resolvable:
resolved_args: List[Resolvable] = [
self._resolve_argument_type(
arg=arg, resolved_variables=resolved_variables, custom_functions=custom_functions
)
for arg in self.args
]
if self.name in custom_functions:
if len(self.args) != len(custom_functions[self.name].function_arguments):
# Should be validated in the Script
raise UNREACHABLE
resolved_variables_with_args = copy.deepcopy(resolved_variables)
for i, arg in enumerate(resolved_args):
function_arg = FunctionArgument.from_idx(idx=i, custom_function_name=self.name)
if function_arg in resolved_variables_with_args:
# function args should always be unique since they are only defined once
# in the custom function as %custom_function_name___idx
# and returned as a set from each custom function.
raise UNREACHABLE
resolved_variables_with_args[function_arg] = arg
return custom_functions[self.name].resolve(
resolved_variables=resolved_variables_with_args,
custom_functions=custom_functions,
)
# Implies the custom function does not exist. This should have
# been checked in the parser with
raise UNREACHABLE
class BuiltInFunction(Function, BuiltInFunctionType):
def validate_args(self) -> "BuiltInFunction":
"""
Ensures the args are compatible with the BuiltInFunction.
"""
if not self.function_spec.is_compatible(input_args=self.args):
raise FunctionArgumentsExceptionFormatter(
input_spec=self.function_spec,
function_instance=self,
).highlight()
return self
# pylint: disable=missing-raises-doc
@property
def callable(self) -> Callable[..., Resolvable]:
"""
Returns
-------
The actual callable of the BuiltInFunction
"""
try:
return Functions.get(self.name)
except Exception as exc:
# Should be validated in the parser
raise UNREACHABLE from exc
# pylint: enable=missing-raises-doc
@functools.cached_property
def function_spec(self) -> FunctionSpec:
"""
Returns
-------
The FunctionSpec of the BuiltInFunction
"""
return FunctionSpec.from_callable(self.callable)
@classmethod
def _arg_output_type(cls, arg: Argument) -> Type[Argument]:
if isinstance(arg, BuiltInFunction):
return arg.output_type()
if isinstance(arg, FutureResolvable):
return arg.future_resolvable_type()
return type(arg)
@classmethod
def _instantiate_lambda(cls, lambda_function_name: str, args: List[Argument]) -> Function:
return (
BuiltInFunction(name=lambda_function_name, args=args)
if Functions.is_built_in(lambda_function_name)
else CustomFunction(name=lambda_function_name, args=args)
)
def _output_type(self, union_args: List[Type[Argument]]) -> Type[Resolvable]:
union_types_list = set()
for union_type in union_args:
possible_output_type = union_type
if union_type in (ReturnableArgument, ReturnableArgumentA, ReturnableArgumentB):
generic_arg_index = self.function_spec.args.index(union_type)
possible_output_type = self._arg_output_type(self.args[generic_arg_index])
if is_union(possible_output_type):
union_types_list.update(possible_output_type.__args__)
else:
union_types_list.add(possible_output_type)
return Union[tuple(union_types_list)]
def output_type(self) -> Type[Resolvable]:
"""
Returns
-------
The BuiltInFunction's true output type.
"""
if is_union(self.function_spec.return_type):
return self._output_type(self.function_spec.return_type.__args__)
return self._output_type([self.function_spec.return_type])
def _resolve_lambda_function(
self,
resolved_arguments: List[Resolvable | Lambda],
resolved_variables: Dict[Variable, Resolvable],
custom_functions: Dict[str, "VariableDependency"],
) -> Resolvable:
"""
Resolve the lambda function by
1. Calling the actual built-in function, which actually forms the input args to the
lambda. NOTE: the lambda argument MUST BE the last argument in the input spec
and output a ResolvedArray, where each element is the input args to the lambda.
2. Preemptively creating the lambda's unresolved output array using output args from (1)
3. Resolve it like any other syntax
"""
function_input_lambda_args = [arg for arg in resolved_arguments if isinstance(arg, Lambda)]
if not self.function_spec.is_lambda_function or len(function_input_lambda_args) != 1:
raise UNREACHABLE
lambda_function_name = function_input_lambda_args[0].value
try:
lambda_args = self.callable(*resolved_arguments)
except Exception as exc:
raise FunctionRuntimeException(
f"Runtime error occurred when executing the function %{self.name}: {str(exc)}"
) from exc
assert isinstance(lambda_args, Array)
return self._resolve_argument_type(
arg=UnresolvedArray(
[
self._instantiate_lambda(
lambda_function_name=lambda_function_name, args=lambda_arg.value
)
for lambda_arg in lambda_args.value
]
),
resolved_variables=resolved_variables,
custom_functions=custom_functions,
)
def _resolve_lambda_reduce_function(
self,
resolved_arguments: List[Resolvable | Lambda],
resolved_variables: Dict[Variable, Resolvable],
custom_functions: Dict[str, "VariableDependency"],
) -> Resolvable:
"""
Resolve the lambda reduce function by
1. Preemptively create the 'reduce-like' call-stack as unresolvable
2. Resolve it like any other syntax
"""
function_input_lambda_args = [arg for arg in resolved_arguments if isinstance(arg, Lambda)]
if not self.function_spec.is_lambda_reduce_function or len(function_input_lambda_args) != 1:
raise UNREACHABLE
lambda_function_name = function_input_lambda_args[0].value
try:
lambda_array = self.callable(*resolved_arguments)
except Exception as exc:
raise FunctionRuntimeException(
f"Runtime error occurred when executing the function %{self.name}: {str(exc)}"
) from exc
assert isinstance(lambda_array, Array)
if len(lambda_array.value) == 1:
return lambda_array.value[0]
reduced = self._instantiate_lambda(
lambda_function_name=lambda_function_name,
args=[lambda_array.value[0], lambda_array.value[1]],
)
for idx in range(2, len(lambda_array.value)):
reduced = self._instantiate_lambda(
lambda_function_name=lambda_function_name, args=[reduced, lambda_array.value[idx]]
)
return self._resolve_argument_type(
arg=reduced,
resolved_variables=resolved_variables,
custom_functions=custom_functions,
)
def resolve(
self,
resolved_variables: Dict[Variable, Resolvable],
custom_functions: Dict[str, "VariableDependency"],
) -> Resolvable:
# Resolve all non-lambda arguments
resolved_arguments: List[Resolvable | Lambda] = [
self._resolve_argument_type(
arg=arg,
resolved_variables=resolved_variables,
custom_functions=custom_functions,
)
for arg in self.args
]
# If a lambda is in a function's arg, resolve it differently
if self.function_spec.is_lambda_function:
return self._resolve_lambda_function(
resolved_arguments=resolved_arguments,
resolved_variables=resolved_variables,
custom_functions=custom_functions,
)
# If a lambda is in a function's arg, resolve it differently
if self.function_spec.is_lambda_reduce_function:
return self._resolve_lambda_reduce_function(
resolved_arguments=resolved_arguments,
resolved_variables=resolved_variables,
custom_functions=custom_functions,
)
try:
return self.callable(*resolved_arguments)
except (UserThrownRuntimeError, RuntimeException):
raise
except Exception as exc:
raise FunctionRuntimeException(
f"Runtime error occurred when executing the function %{self.name}: {str(exc)}"
) from exc

66
src/ytdl_sub/script/types/map.py Normal file
View file

@ -0,0 +1,66 @@
import itertools
from abc import ABC
from dataclasses import dataclass
from typing import Any
from typing import Dict
from typing import List
from typing import Type
from ytdl_sub.script.types.resolvable import Argument
from ytdl_sub.script.types.resolvable import FutureResolvable
from ytdl_sub.script.types.resolvable import Hashable
from ytdl_sub.script.types.resolvable import NonHashable
from ytdl_sub.script.types.resolvable import Resolvable
from ytdl_sub.script.types.resolvable import ResolvableToJson
from ytdl_sub.script.types.variable import Variable
from ytdl_sub.script.types.variable_dependency import VariableDependency
from ytdl_sub.script.utils.exceptions import KeyNotHashableRuntimeException
@dataclass(frozen=True)
class _Map(NonHashable, ABC):
value: Dict[Hashable, Resolvable]
@classmethod
def type_name(cls) -> str:
return "Map"
@dataclass(frozen=True)
class UnresolvedMap(_Map, VariableDependency, FutureResolvable):
value: Dict[Argument, Argument]
@property
def _iterable_arguments(self) -> List[Argument]:
return list(itertools.chain(*self.value.items()))
def resolve(
self,
resolved_variables: Dict[Variable, Resolvable],
custom_functions: Dict[str, VariableDependency],
) -> Resolvable:
output: Dict[Hashable, Resolvable] = {}
for key, value in self.value.items():
resolved_key = self._resolve_argument_type(
arg=key, resolved_variables=resolved_variables, custom_functions=custom_functions
)
if not isinstance(resolved_key, Hashable):
raise KeyNotHashableRuntimeException(
f"Tried to use {resolved_key.type_name()} as a Map key, but it is not hashable."
)
output[resolved_key] = self._resolve_argument_type(
arg=value, resolved_variables=resolved_variables, custom_functions=custom_functions
)
return Map(output)
def future_resolvable_type(self) -> Type[Resolvable]:
return Map
@dataclass(frozen=True)
class Map(_Map, ResolvableToJson):
@property
def native(self) -> Any:
return {key.native: value.native for key, value in self.value.items()}

253
src/ytdl_sub/script/types/resolvable.py Normal file
View file

@ -0,0 +1,253 @@
import json
from abc import ABC
from abc import abstractmethod
from dataclasses import dataclass
from typing import Any
from typing import Generic
from typing import List
from typing import Type
from typing import TypeVar
T = TypeVar("T")
NumericT = TypeVar("NumericT", bound=int | float)
@dataclass(frozen=True)
class NamedType(ABC):
@classmethod
def type_name(cls) -> str:
"""
Returns
-------
The type name to present to users. Defaults to the class name.
"""
return cls.__name__
@dataclass(frozen=True)
class Argument(NamedType, ABC):
"""
Any possible argument type that has not been resolved yet
"""
@dataclass(frozen=True)
class ValueArgument(Argument, ABC):
"""
Argument that has a value
"""
value: Any
@dataclass(frozen=True)
class NamedArgument(Argument, ABC):
"""
Argument that has an explicit name (i.e. custom function or variable)
"""
name: str
@dataclass(frozen=True)
class ReturnableArgument(ValueArgument, NamedType, ABC):
"""
AnyType to express generics in functions that are part of the return type
"""
@dataclass(frozen=True)
class ReturnableArgumentA(ValueArgument, NamedType, ABC):
"""
AnyType to express generics in functions when more than one are present (i.e. `if`)
"""
@dataclass(frozen=True)
class ReturnableArgumentB(ValueArgument, NamedType, ABC):
"""
AnyType to express generics in functions when more than one are present (i.e. `if`)
"""
@dataclass(frozen=True)
class AnyArgument(ReturnableArgument, ReturnableArgumentA, ReturnableArgumentB, ABC):
"""
Human-readable name for Resolvable
"""
@dataclass(frozen=True)
class Resolvable(AnyArgument, ABC):
"""
A type that is resolved into a native Python type (and have no dependencies to other types).
"""
def __str__(self) -> str:
return str(self.value)
@property
def native(self) -> Any:
"""
Returns
-------
The resolvable in its native form
"""
return self.value
@dataclass(frozen=True)
class FutureResolvable(AnyArgument, ABC):
"""
Used when parsing, it is an unresolved type that will eventually resolve to a known type
(i.e. Maps, Arrays)
"""
@abstractmethod
def future_resolvable_type(self) -> Type[Resolvable]:
"""
The resolvable type that this type is known to turn into
"""
@dataclass(frozen=True)
class Hashable(Resolvable, ABC):
"""
Resolvable type that can be used as hashes (i.e. in Maps)
"""
@dataclass(frozen=True)
class NonHashable(NamedType, ABC):
"""
Type that is known to never be hashable.
"""
@dataclass(frozen=True)
class ResolvableToJson(Resolvable, ABC):
"""
Types whose string values should be resolved to JSON (i.e. Maps, Arrays)
"""
def __str__(self):
return json.dumps(self.native)
@dataclass(frozen=True)
class ResolvableT(Hashable, ABC, Generic[T]):
"""
Resolvable types that resolve to the generic T
"""
value: T
@dataclass(frozen=True)
class Numeric(ResolvableT[NumericT], ABC, Generic[NumericT]):
"""
Resolvable numeric types (int/float)
"""
@dataclass(frozen=True)
class Integer(Numeric[int], Argument):
"""
Resolved Integer type
"""
@dataclass(frozen=True)
class Float(Numeric[float], Argument):
"""
Resolved float type
"""
@dataclass(frozen=True)
class Boolean(ResolvableT[bool], Argument):
"""
Resolved bool type
"""
@dataclass(frozen=True)
class String(ResolvableT[str], Argument):
"""
Resolved String type
"""
@dataclass(frozen=True)
class NamedCustomFunction(NamedArgument, ABC):
"""
A custom function with a defined name (but unknown args)
"""
@dataclass(frozen=True)
class ParsedCustomFunction(NamedCustomFunction):
num_input_args: int
@dataclass(frozen=True)
class FunctionType(NamedArgument, ABC):
args: List[Argument]
@dataclass(frozen=True)
class BuiltInFunctionType(FunctionType, ABC):
@abstractmethod
def output_type(self) -> Type[Resolvable]:
"""
Returns
-------
The known Resolvable type that a BuiltInFunction will output
"""
@dataclass(frozen=True)
class Lambda(Resolvable):
value: str
@property
def native(self) -> Any:
return f"%{self.value}"
@classmethod
def num_input_args(cls) -> int:
"""
Returns
-------
The number of input args the Lambda function takes
"""
return 1
@dataclass(frozen=True)
class LambdaTwo(Lambda):
"""
Type-hinting for functions that apply lambdas with two inputs per element
"""
@classmethod
def num_input_args(cls) -> int:
return 2
@dataclass(frozen=True)
class LambdaThree(Lambda):
"""
Type-hinting for functions that apply lambdas with three inputs per element
"""
@classmethod
def num_input_args(cls) -> int:
return 3
@dataclass(frozen=True)
class LambdaReduce(LambdaTwo):
"""
Type-hinting for functions that apply a reduce-operation using a lambda (two arguments)
"""

52
src/ytdl_sub/script/types/syntax_tree.py Normal file
View file

@ -0,0 +1,52 @@
from dataclasses import dataclass
from typing import Dict
from typing import List
from typing import Optional
from ytdl_sub.script.types.resolvable import Argument
from ytdl_sub.script.types.resolvable import Resolvable
from ytdl_sub.script.types.resolvable import String
from ytdl_sub.script.types.variable import Variable
from ytdl_sub.script.types.variable_dependency import VariableDependency
@dataclass(frozen=True)
class SyntaxTree(VariableDependency):
ast: List[Argument]
@property
def _iterable_arguments(self) -> List[Argument]:
return self.ast
def resolve(
self,
resolved_variables: Dict[Variable, Resolvable],
custom_functions: Dict[str, VariableDependency],
) -> Resolvable:
resolved: List[Resolvable] = []
for token in self.ast:
resolved.append(
self._resolve_argument_type(
arg=token,
resolved_variables=resolved_variables,
custom_functions=custom_functions,
)
)
# If only one resolvable resides in the AST, return as that
if len(resolved) == 1:
return resolved[0]
# Otherwise, to concat multiple resolved outputs, we must concat as strings
return String("".join([str(res) for res in resolved]))
@property
def maybe_resolvable(self) -> Optional[Resolvable]:
"""
Returns
-------
A resolvable if the AST contains a single type that is resolvable. None otherwise.
"""
if len(self.ast) == 1 and isinstance(self.ast[0], Resolvable):
return self.ast[0]
return None

28
src/ytdl_sub/script/types/variable.py Normal file
View file

@ -0,0 +1,28 @@
from dataclasses import dataclass
from typing import Optional
from ytdl_sub.script.types.resolvable import NamedArgument
@dataclass(frozen=True)
class Variable(NamedArgument):
pass
@dataclass(frozen=True)
class FunctionArgument(Variable):
"""Arguments for custom functions, i.e. $0, $1, etc"""
index: int
@classmethod
def from_idx(cls, idx: int, custom_function_name: Optional[str]) -> "FunctionArgument":
"""
Returns
-------
FunctionArgument whose variable name is the index, and optionally contains the custom
function name its defined in as a prefix.
"""
if custom_function_name:
return FunctionArgument(name=f"${custom_function_name}___{idx}", index=idx)
return FunctionArgument(name=f"${idx}", index=idx)

174
src/ytdl_sub/script/types/variable_dependency.py Normal file
View file

@ -0,0 +1,174 @@
from abc import ABC
from abc import abstractmethod
from dataclasses import dataclass
from typing import Dict
from typing import Iterable
from typing import List
from typing import Set
from typing import Type
from typing import TypeVar
from typing import final
from ytdl_sub.script.types.resolvable import Argument
from ytdl_sub.script.types.resolvable import BuiltInFunctionType
from ytdl_sub.script.types.resolvable import FunctionType
from ytdl_sub.script.types.resolvable import Lambda
from ytdl_sub.script.types.resolvable import NamedCustomFunction
from ytdl_sub.script.types.resolvable import ParsedCustomFunction
from ytdl_sub.script.types.resolvable import Resolvable
from ytdl_sub.script.types.variable import FunctionArgument
from ytdl_sub.script.types.variable import Variable
from ytdl_sub.script.utils.exceptions import UNREACHABLE
TypeT = TypeVar("TypeT")
@dataclass(frozen=True)
class VariableDependency(ABC):
@property
@abstractmethod
def _iterable_arguments(self) -> List[Argument]:
"""
Returns
-------
Any arguments in the VariableDependency that may or may not need to be resolved.
"""
def _recurse_get(self, ttype: Type[TypeT], subclass: bool = False) -> List[TypeT]:
output: List[TypeT] = []
for arg in self._iterable_arguments:
if subclass and issubclass(type(arg), ttype):
output.append(arg)
elif isinstance(arg, ttype):
output.append(arg)
if isinstance(arg, VariableDependency):
# pylint: disable=protected-access
output.extend(arg._recurse_get(ttype))
# pylint: enable=protected-access
return output
@final
@property
def variables(self) -> Set[Variable]:
"""
Returns
-------
All Variables that this depends on.
"""
return set(self._recurse_get(Variable))
@final
@property
def built_in_functions(self) -> List[BuiltInFunctionType]:
"""
Returns
-------
All BuiltInFunctions that this depends on.
"""
return self._recurse_get(BuiltInFunctionType)
@final
@property
def function_arguments(self) -> Set[FunctionArgument]:
"""
Returns
-------
All FunctionArguments that this depends on.
"""
return set(self._recurse_get(FunctionArgument))
@final
@property
def lambdas(self) -> Set[Lambda]:
"""
Returns
-------
All Lambdas that this depends on.
"""
return set(self._recurse_get(Lambda, subclass=True))
# pylint: disable=missing-raises-doc
@final
@property
def custom_functions(self) -> Set[ParsedCustomFunction]:
"""
Returns
-------
All CustomFunctions that this depends on.
"""
output: Set[ParsedCustomFunction] = set()
for arg in self._iterable_arguments:
if isinstance(arg, NamedCustomFunction):
if not isinstance(arg, FunctionType):
# A NamedCustomFunction should also always be a FunctionType
raise UNREACHABLE
# Custom funcs aren't hashable, so recreate just the base-class portion
output.add(ParsedCustomFunction(name=arg.name, num_input_args=len(arg.args)))
if isinstance(arg, VariableDependency):
output.update(arg.custom_functions)
return output
# pylint: enable=missing-raises-doc
@abstractmethod
def resolve(
self,
resolved_variables: Dict[Variable, Resolvable],
custom_functions: Dict[str, "VariableDependency"],
) -> Resolvable:
"""
Parameters
----------
resolved_variables
Lookup of variables that have been resolved
custom_functions
Lookup of any custom functions that have been parsed
Returns
-------
Resolved value
"""
@classmethod
def _resolve_argument_type(
cls,
arg: Argument,
resolved_variables: Dict[Variable, Resolvable],
custom_functions: Dict[str, "VariableDependency"],
) -> Resolvable:
if isinstance(arg, Resolvable):
return arg
if isinstance(arg, Variable):
if arg not in resolved_variables:
# All variables should exist and be resolved at this point
raise UNREACHABLE
return resolved_variables[arg]
if isinstance(arg, VariableDependency):
return arg.resolve(
resolved_variables=resolved_variables, custom_functions=custom_functions
)
raise UNREACHABLE
@final
def is_subset_of(self, variables: Iterable[Variable]) -> bool:
"""
Returns
-------
True if it contains all input variables as a dependency. False otherwise.
"""
return not self.variables.issubset(variables)
@final
def contains(self, variables: Iterable[Variable]) -> bool:
"""
Returns
-------
True if it contains any of the input variables. False otherwise.
"""
return len(self.variables.intersection(variables)) > 0

0
src/ytdl_sub/script/utils/__init__.py Normal file
View file

153
src/ytdl_sub/script/utils/exception_formatters.py Normal file
View file

@ -0,0 +1,153 @@
import sys
from typing import List
from typing import Type
from typing import TypeVar
from typing import Union
from ytdl_sub.script.types.resolvable import BuiltInFunctionType
from ytdl_sub.script.types.resolvable import NamedType
from ytdl_sub.script.utils.exceptions import IncompatibleFunctionArguments
from ytdl_sub.script.utils.exceptions import UserException
from ytdl_sub.script.utils.type_checking import FunctionSpec
from ytdl_sub.script.utils.type_checking import get_optional_type
from ytdl_sub.script.utils.type_checking import is_optional
from ytdl_sub.script.utils.type_checking import is_union
TUserException = TypeVar("TUserException", bound=UserException)
class ParserExceptionFormatter:
def __init__(self, text: str, start: int, end: int, exception: TUserException):
self._text = text
self._start = start
self._end = end
self._exception = exception
def _exception_text(self, border: int):
"""
Format for single-line exceptions
"""
text_left = max(0, self._start - border)
text_right = min(len(self._text), self._start + border)
relative_start = self._start - text_left
exception_text: str = ""
if text_left > 3:
exception_text = "… "
relative_start += len(exception_text)
exception_text += self._text[text_left:text_right]
if text_right < len(self._text) - 3:
exception_text += " …"
exception_text += "\n"
exception_text += f"{' ' * relative_start}^"
return "\n" + exception_text
@property
def _is_multi_line(self) -> bool:
return "\n" in self._text
def _exception_text_lines(self, border_lines: int = 0) -> str:
"""
Format for multi-line exceptions
"""
split_text = self._text.split("\n")
start_line: int = sys.maxsize
end_line: int = -1
pos: int = 0
for idx, line in enumerate(split_text):
if self._start <= pos < self._end:
start_line = min(start_line, idx)
end_line = max(end_line, idx + 1)
pos += len(line)
true_start_line = start_line
start_line = max(0, start_line - border_lines)
end_line = min(len(split_text), end_line + border_lines)
# Get min leading spaces between all lines to return
min_leading_spaces = sys.maxsize
for line in split_text[start_line:end_line]:
min_leading_spaces = min(min_leading_spaces, len(line) - len(line.lstrip()))
to_return: List[str] = []
for idx in range(start_line, end_line):
if idx == true_start_line:
to_return.append(f">>> {split_text[idx][min_leading_spaces:]}")
else:
to_return.append(f" {split_text[idx][min_leading_spaces:]}")
return "\n" + "\n".join(to_return)
def highlight(self) -> TUserException:
"""
Returns
-------
Exception with human-readable error highlighting for invalid syntax
"""
if self._is_multi_line:
invalid_syntax = self._exception_text_lines(border_lines=3)
else:
invalid_syntax = self._exception_text(border=20)
return self._exception.__class__(f"{invalid_syntax}\n{str(self._exception)}")
class FunctionArgumentsExceptionFormatter:
def __init__(
self,
input_spec: FunctionSpec,
function_instance: BuiltInFunctionType,
):
self._args = input_spec.args
self._varargs = input_spec.varargs
self._name = function_instance.name
self._input_args = function_instance.args
@classmethod
def _to_human_readable_name(cls, python_type: Type[NamedType] | Type[Union[NamedType]]) -> str:
if is_optional(python_type):
return f"Optional[{cls._to_human_readable_name(get_optional_type(python_type))}]"
if is_union(python_type):
return ", ".join(
sorted(cls._to_human_readable_name(arg) for arg in python_type.__args__)
)
return python_type.type_name()
def _expected_args_str(self) -> str:
if self._args is not None:
return f"({', '.join([self._to_human_readable_name(type_) for type_ in self._args])})"
if self._varargs is not None:
return f"({self._to_human_readable_name(self._varargs)}, ...)"
return "()"
def _received_args_str(self) -> str:
received_type_names: List[str] = []
for arg in self._input_args:
if isinstance(arg, BuiltInFunctionType):
if is_union(arg.output_type()):
readable_type_names = ", ".join(
sorted(type_.type_name() for type_ in arg.output_type().__args__)
)
received_type_names.append(f"%{arg.name}(...)->Union[{readable_type_names}]")
else:
received_type_names.append(f"%{arg.name}(...)->{arg.output_type().type_name()}")
else:
received_type_names.append(arg.type_name())
return f"({', '.join(name for name in received_type_names)})"
def highlight(self) -> IncompatibleFunctionArguments:
"""
Returns
-------
Exception with human-readable error highlighting for incompatible function arguments
"""
return IncompatibleFunctionArguments(
f"Incompatible arguments passed to function {self._name}.\n"
f"Expected {self._expected_args_str()}\nReceived {self._received_args_str()}"
)

96
src/ytdl_sub/script/utils/exceptions.py Normal file
View file

@ -0,0 +1,96 @@
from abc import ABC
from ytdl_sub.utils.exceptions import ValidationException
###################################################################################################
# USER EXCEPTIONS
class UserException(ValidationException, ABC):
"""It's the user's fault!"""
class InvalidSyntaxException(UserException):
"""Syntax is incorrect"""
class InvalidVariableName(UserException):
"""Variable name is invalid"""
class InvalidFunctionName(UserException):
"""Custom function name is invalid"""
class InvalidCustomFunctionArguments(UserException):
"""Custom function arguments are invalid (i.e. they do not increment)"""
class InvalidCustomFunctionArgumentName(UserException):
"""Custom function argument name (i.e. $0) is invalid"""
class IncompatibleFunctionArguments(UserException):
"""Function has invalid arguments"""
class FunctionDoesNotExist(UserException):
"""Tried to use a function that does not exist"""
class VariableDoesNotExist(UserException):
"""Tried to use a variable that does not exist"""
class ScriptBuilderMissingDefinitions(UserException):
"""Tried to build an incomplete ScriptBuilder"""
class CycleDetected(UserException):
"""A cycle exists within a user's script"""
class UserThrownRuntimeError(ValidationException):
"""An error explicitly thrown by the user via a function"""
class _UnreachableSyntaxException(InvalidSyntaxException):
"""For use in places where code _should_ never reach, but might from bugs"""
UNREACHABLE = _UnreachableSyntaxException(
"If you see this error, you have discovered a bug in the script parser!\n"
"Please upload your config/subscription file(s) to and make a GitHub issue at "
"https://github.com/jmbannon/ytdl-sub/issues"
)
###################################################################################################
# RUNTIME EXCEPTIONS
class RuntimeException(ValueError, ABC):
"""Exception thrown at runtime during resolution"""
class ScriptVariableNotResolved(RuntimeException):
"""Tried to get a variable's resolved value from a script, but has not resolved yet"""
class FunctionRuntimeException(RuntimeException):
"""Exception thrown when a ytdl-sub function has an error occur at runtime"""
class KeyNotHashableRuntimeException(RuntimeException):
"""Map tried to use a non-hashable key at runtime"""
class ArrayValueDoesNotExist(RuntimeException):
"""Tried to get an index of a value in an Array that does not exist"""
class FunctionDoesNotExistRuntimeException(RuntimeException):
"""Tried to get a function that does not exist"""
class KeyDoesNotExistRuntimeException(RuntimeException):
"""Tried to access a key on a map that does not exist, with no default"""

60
src/ytdl_sub/script/utils/name_validation.py Normal file
View file

@ -0,0 +1,60 @@
import re
from ytdl_sub.script.functions import Functions
from ytdl_sub.script.utils.exceptions import InvalidFunctionName
from ytdl_sub.script.utils.exceptions import InvalidVariableName
_NAME_REGEX_VALIDATOR = re.compile(r"^[a-z][a-z0-9_]*$")
def is_valid_name(name: str) -> bool:
"""
Returns
-------
True if the name adheres to the ``snake_case`` format. False otherwise.
"""
return re.match(_NAME_REGEX_VALIDATOR, name) is not None
def validate_variable_name(variable_name: str) -> str:
"""
Raises
------
InvalidVariableName
if the variable name is invalid
"""
if not is_valid_name(variable_name):
raise InvalidVariableName(
f"Variable name '{variable_name}' is invalid:"
" Names must be lower_snake_cased and begin with a letter."
)
if Functions.is_built_in(variable_name):
raise InvalidVariableName(
f"Variable name '{variable_name}' is invalid:"
" The name is used by a built-in function and cannot be overwritten."
)
return variable_name
def validate_custom_function_name(custom_function_name: str) -> None:
"""
Raises
------
InvalidFunctionName
If the function name is invalid
InvalidVariableName
if the variable name is invalid
"""
if not is_valid_name(custom_function_name):
raise InvalidFunctionName(
f"Custom function name '%{custom_function_name}' is invalid:"
" Names must be %lower_snake_cased and begin with a letter."
)
if Functions.is_built_in(custom_function_name):
raise InvalidFunctionName(
f"Custom function name '%{custom_function_name}' is invalid:"
" The name is used by a built-in function and cannot be overwritten."
)

243
src/ytdl_sub/script/utils/type_checking.py Normal file
View file

@ -0,0 +1,243 @@
# pylint: disable=missing-raises-doc
import inspect
from dataclasses import dataclass
from inspect import FullArgSpec
from typing import Callable
from typing import List
from typing import Optional
from typing import Type
from typing import TypeVar
from typing import Union
from typing import get_origin
from ytdl_sub.script.types.resolvable import Argument
from ytdl_sub.script.types.resolvable import BuiltInFunctionType
from ytdl_sub.script.types.resolvable import FutureResolvable
from ytdl_sub.script.types.resolvable import Lambda
from ytdl_sub.script.types.resolvable import LambdaReduce
from ytdl_sub.script.types.resolvable import LambdaThree
from ytdl_sub.script.types.resolvable import LambdaTwo
from ytdl_sub.script.types.resolvable import NamedCustomFunction
from ytdl_sub.script.types.resolvable import NamedType
from ytdl_sub.script.types.resolvable import Resolvable
from ytdl_sub.script.types.variable import Variable
from ytdl_sub.script.utils.exceptions import UNREACHABLE
TLambda = TypeVar("TLambda", bound=Lambda)
def is_union(arg_type: Type) -> bool:
"""
Returns
-------
True if typing is Union. False otherwise.
"""
return get_origin(arg_type) is Union
def is_optional(arg_type: Type) -> bool:
"""
Returns
-------
True if typing is Optional. False otherwise.
"""
return is_union(arg_type) and type(None) in arg_type.__args__
def get_optional_type(optional_type: Type) -> Type[NamedType]:
"""
Returns
-------
Type within the Optional[Type]
"""
return [arg for arg in optional_type.__args__ if arg != type(None)][0]
def _is_type_compatible(
arg_type: Type[NamedType],
expected_arg_type: Type[Resolvable | Optional[Resolvable]],
) -> bool:
"""
Returns
-------
True if arg is compatible with expected_arg_type. False otherwise.
"""
if is_union(expected_arg_type):
# See if the arg is a valid against the union
valid_type = False
# if the input arg is a union, do a direct comparison
if is_union(arg_type):
valid_type = arg_type == expected_arg_type
# otherwise, iterate the union to see if it's compatible
else:
for union_type in expected_arg_type.__args__:
if issubclass(arg_type, union_type):
valid_type = True
break
if not valid_type:
return False
# If the input is a union and the expected type is not, see if
# each possible union input is compatible with the expected type
elif is_union(arg_type):
for union_type in arg_type.__args__:
if not _is_type_compatible(union_type, expected_arg_type):
return False
elif issubclass(arg_type, (NamedCustomFunction, Variable)):
return True # custom-function/variable can be anything, so pass for now
elif issubclass(arg_type, Lambda) and issubclass(expected_arg_type, arg_type):
# lambda, check if expected_arg_type is a subclass
# Do not return on just that to also allow lambdas to be returned as
# ReturnableArguments (i.e in an %if statement)
return True
elif not issubclass(arg_type, expected_arg_type):
return False
return True
def is_type_compatible(
arg: NamedType,
expected_arg_type: Type[Resolvable | Optional[Resolvable]],
) -> bool:
"""
Returns
-------
True if arg is compatible with expected_arg_type. False otherwise.
"""
arg_type: Type[NamedType] = arg.__class__
if isinstance(arg, BuiltInFunctionType):
arg_type = arg.output_type() # built-in function
elif isinstance(arg, FutureResolvable):
arg_type = arg.future_resolvable_type()
return _is_type_compatible(arg_type, expected_arg_type)
@dataclass(frozen=True)
class FunctionSpec:
return_type: Type[Resolvable]
args: Optional[List[Type[Resolvable | Optional[Resolvable]]]] = None
varargs: Optional[Type[Resolvable]] = None
def __post_init__(self):
assert (self.args is None) ^ (self.varargs is None)
def _is_args_compatible(self, input_args: List[Argument]) -> bool:
assert self.args is not None
if len(input_args) > len(self.args):
return False
for idx, arg in enumerate(self.args):
input_arg = input_args[idx] if idx < len(input_args) else None
if not is_type_compatible(arg=input_arg, expected_arg_type=arg):
return False
return True
def _is_varargs_compatible(self, input_args: List[Argument]) -> bool:
"""
Returns
-------
True if the input args are compatible with the spec's varargs. False otherwise.
"""
assert self.varargs is not None
for input_arg in input_args:
if not is_type_compatible(arg=input_arg, expected_arg_type=self.varargs):
return False
return True
def is_compatible(self, input_args: List[Argument]) -> bool:
"""
Returns
-------
True if input_args is compatible. False otherwise.
"""
if self.args is not None:
return self._is_args_compatible(input_args=input_args)
if self.varargs is not None:
return self._is_varargs_compatible(input_args=input_args)
raise UNREACHABLE # TODO: functions with no args
def is_num_args_compatible(self, num_input_args: int) -> bool:
"""
Returns
-------
True if the number of input args is compatible with the function spec. False otherwise.
"""
if self.args is not None:
return self.num_required_args <= num_input_args <= len(self.args)
return True # varargs can take any number
@property
def num_required_args(self) -> int:
"""
Returns
-------
The minimum number of args required to call the function.
"""
if self.args is not None:
return sum(1 for arg in self.args if not is_optional(arg))
return 0 # varargs can take any number
@property
def is_lambda_reduce_function(self) -> Optional[Type[LambdaReduce]]:
"""
Returns
-------
True if the function is a Lambda-reduce function. False otherwise.
"""
return LambdaReduce if LambdaReduce in (self.args or []) else None
@property
def is_lambda_function(self) -> Optional[Type[Lambda | LambdaTwo | LambdaThree]]:
"""
Returns
-------
True if the function is a Lambda function (excluding reduce). False otherwise.
"""
if LambdaThree in (self.args or []):
return LambdaThree
if LambdaTwo in (self.args or []):
return LambdaTwo
if Lambda in (self.args or []):
return Lambda
return None
@property
def is_lambda_like(self) -> Optional[Type[TLambda]]:
"""
Returns
-------
True if the function is a Lambda type (including reduce).
"""
if l_type := self.is_lambda_reduce_function:
return l_type
if l_type := self.is_lambda_function:
return l_type
return None
@classmethod
def from_callable(cls, callable_ref: Callable[..., Resolvable]) -> "FunctionSpec":
"""
Returns
-------
FunctionSpec from a built-in function.
"""
arg_spec: FullArgSpec = inspect.getfullargspec(callable_ref)
if arg_spec.varargs:
return FunctionSpec(
return_type=arg_spec.annotations["return"],
varargs=arg_spec.annotations[arg_spec.varargs],
)
return FunctionSpec(
return_type=arg_spec.annotations["return"],
args=[arg_spec.annotations[arg_name] for arg_name in arg_spec.args],
)

4
src/ytdl_sub/subscriptions/base_subscription.py
View file

@ -3,10 +3,10 @@ from pathlib import Path
from typing import Optional
from ytdl_sub.config.config_validator import ConfigOptions
from ytdl_sub.config.overrides import Overrides
from ytdl_sub.config.plugin.preset_plugins import PresetPlugins
from ytdl_sub.config.preset import Preset
from ytdl_sub.config.preset import PresetPlugins
from ytdl_sub.config.preset_options import OutputOptions
from ytdl_sub.config.preset_options import Overrides
from ytdl_sub.config.preset_options import YTDLOptions
from ytdl_sub.downloaders.url.validators import MultiUrlValidator
from ytdl_sub.utils.file_handler import FileHandlerTransactionLog

32
src/ytdl_sub/subscriptions/subscription_download.py
View file

@ -7,8 +7,10 @@ from pathlib import Path
from typing import List
from typing import Optional
from ytdl_sub.config.plugin import Plugin
from ytdl_sub.config.plugin import SplitPlugin
from ytdl_sub.config.plugin.plugin import Plugin
from ytdl_sub.config.plugin.plugin import SplitPlugin
from ytdl_sub.config.plugin.plugin_mapping import PluginMapping
from ytdl_sub.config.plugin.plugin_operation import PluginOperation
from ytdl_sub.downloaders.info_json.info_json_downloader import InfoJsonDownloader
from ytdl_sub.downloaders.info_json.info_json_downloader import InfoJsonDownloaderOptions
from ytdl_sub.downloaders.source_plugin import SourcePlugin
@ -206,7 +208,9 @@ class SubscriptionDownload(BaseSubscription, ABC):
@classmethod
def _preprocess_entry(cls, plugins: List[Plugin], entry: Entry) -> Optional[Entry]:
maybe_entry: Optional[Entry] = entry
for plugin in sorted(plugins, key=lambda _plugin: _plugin.priority.modify_entry_metadata):
for plugin in PluginMapping.order_plugins_by(
plugins, PluginOperation.MODIFY_ENTRY_METADATA
):
if (maybe_entry := plugin.modify_entry_metadata(maybe_entry)) is None:
return None
@ -216,7 +220,7 @@ class SubscriptionDownload(BaseSubscription, ABC):
self, plugins: List[Plugin], dry_run: bool, entry: Entry, entry_metadata: FileMetadata
):
# Post-process the entry with all plugins
for plugin in sorted(plugins, key=lambda _plugin: _plugin.priority.post_process):
for plugin in PluginMapping.order_plugins_by(plugins, PluginOperation.POST_PROCESS):
optional_plugin_entry_metadata = plugin.post_process_entry(entry)
if optional_plugin_entry_metadata:
entry_metadata.extend(optional_plugin_entry_metadata)
@ -236,7 +240,7 @@ class SubscriptionDownload(BaseSubscription, ABC):
entry_: Optional[Entry] = entry
# First, modify the entry with all plugins
for plugin in sorted(plugins, key=lambda _plugin: _plugin.priority.modify_entry):
for plugin in PluginMapping.order_plugins_by(plugins, PluginOperation.MODIFY_ENTRY):
# Break if it is None, it is indicated to not process any further
if (entry_ := plugin.modify_entry(entry_)) is None:
break
@ -253,18 +257,10 @@ class SubscriptionDownload(BaseSubscription, ABC):
) -> None:
entry_: Optional[Entry] = entry
plugins_pre_split = sorted(
[plugin for plugin in plugins if not plugin.priority.modify_entry_after_split],
key=lambda _plugin: _plugin.priority.modify_entry,
)
plugins_post_split = sorted(
[plugin for plugin in plugins if plugin.priority.modify_entry_after_split],
key=lambda _plugin: _plugin.priority.modify_entry,
)
# First, modify the entry with pre_split plugins
for plugin in plugins_pre_split:
for plugin in PluginMapping.order_plugins_by(
plugins, PluginOperation.MODIFY_ENTRY, before_split=True
):
# Break if it is None, it is indicated to not process any further
if (entry_ := plugin.modify_entry(entry_)) is None:
break
@ -274,7 +270,9 @@ class SubscriptionDownload(BaseSubscription, ABC):
for split_entry, split_entry_metadata in split_plugin.split(entry=entry_):
split_entry_: Optional[Entry] = split_entry
for plugin in plugins_post_split:
for plugin in PluginMapping.order_plugins_by(
plugins, PluginOperation.MODIFY_ENTRY, before_split=False
):
# Return if it is None, it is indicated to not process any further.
# Break out of the plugin loop
if (split_entry_ := plugin.modify_entry(split_entry_)) is None:

2
src/ytdl_sub/subscriptions/subscription_validators.py
View file

@ -8,7 +8,7 @@ from typing import Optional
from typing import final
from ytdl_sub.config.config_file import ConfigFile
from ytdl_sub.config.preset_options import Overrides
from ytdl_sub.config.overrides import Overrides
from ytdl_sub.entries.variables.override_variables import SUBSCRIPTION_NAME
from ytdl_sub.entries.variables.override_variables import SUBSCRIPTION_VALUE
from ytdl_sub.entries.variables.override_variables import OverrideVariables

2
src/ytdl_sub/subscriptions/subscription_ytdl_options.py
View file

@ -7,7 +7,7 @@ from typing import TypeVar
from yt_dlp import match_filter_func
from ytdl_sub.config.plugin import Plugin
from ytdl_sub.config.plugin.plugin import Plugin
from ytdl_sub.config.preset import Preset
from ytdl_sub.downloaders.ytdl_options_builder import YTDLOptionsBuilder
from ytdl_sub.plugins.audio_extract import AudioExtractPlugin

55
src/ytdl_sub/utils/chapters.py
View file

@ -1,14 +1,16 @@
import re
from typing import Dict
from typing import List
from typing import Optional
from typing import Tuple
from ytdl_sub.entries.entry import Entry
from ytdl_sub.entries.variables.kwargs import CHAPTERS
from ytdl_sub.entries.variables.kwargs import YTDL_SUB_CUSTOM_CHAPTERS
from ytdl_sub.entries.entry import ytdl_sub_chapters_from_comments
from ytdl_sub.entries.script.variable_definitions import VARIABLES
from ytdl_sub.entries.script.variable_definitions import VariableDefinitions
from ytdl_sub.utils.file_handler import FileMetadata
v: VariableDefinitions = VARIABLES
class Timestamp:
@ -157,6 +159,17 @@ class Chapters:
"""
return self.timestamps[0].timestamp_sec == 0
def to_yt_dlp_chapter_metadata(self) -> List[Dict[str, str | float]]:
"""
Returns
-------
Metadata dict
"""
return [
{"start_time": ts.timestamp_sec, "title": title}
for ts, title in zip(self.timestamps, self.titles)
]
def to_file_metadata_dict(self) -> Dict:
"""
Returns
@ -165,7 +178,7 @@ class Chapters:
"""
return {ts.readable_str: title for ts, title in zip(self.timestamps, self.titles)}
def to_file_metadata(self, title: Optional[str] = None) -> FileMetadata:
def to_file_metadata(self, title: str) -> FileMetadata:
"""
Parameters
----------
@ -216,8 +229,23 @@ class Chapters:
# If more than 3 timestamps were parsed, return it
if len(timestamps) >= 3:
return Chapters(timestamps=timestamps, titles=titles)
# Otherwise return empty chapters
return Chapters(timestamps=[], titles=[])
return cls.from_empty()
@classmethod
def from_yt_dlp_chapters(cls, chapters: List[Dict[str, str | float]]):
"""
Create a Chapters object from the raw ``chapters`` metadata returned in an info.json
"""
timestamps: List[Timestamp] = []
titles: List[str] = []
for chapter in chapters:
timestamps.append(Timestamp.from_seconds(int(float(chapter["start_time"]))))
titles.append(chapter["title"])
return Chapters(timestamps=timestamps, titles=titles)
@classmethod
def from_entry_chapters(cls, entry: Entry) -> "Chapters":
@ -231,19 +259,12 @@ class Chapters:
-------
Chapters object
"""
timestamps: List[Timestamp] = []
titles: List[str] = []
if chapters := (
entry.try_get(ytdl_sub_chapters_from_comments, list) or entry.get(v.chapters, list)
):
return cls.from_yt_dlp_chapters(chapters)
if entry.kwargs_contains(CHAPTERS):
for chapter in entry.kwargs_get(CHAPTERS, []):
timestamps.append(Timestamp.from_seconds(int(float(chapter["start_time"]))))
titles.append(chapter["title"])
elif entry.kwargs_contains(YTDL_SUB_CUSTOM_CHAPTERS):
for start_time, title in entry.kwargs_get(YTDL_SUB_CUSTOM_CHAPTERS, {}).items():
timestamps.append(Timestamp.from_str(start_time))
titles.append(title)
return Chapters(timestamps=timestamps, titles=titles)
return cls.from_empty()
@classmethod
def from_empty(cls) -> "Chapters":

2
src/ytdl_sub/utils/datetime.py
View file

@ -3,7 +3,7 @@ from typing import Optional
from yt_dlp import DateRange
from yt_dlp.utils import datetime_from_str
from ytdl_sub.config.preset_options import Overrides
from ytdl_sub.config.overrides import Overrides
from ytdl_sub.validators.string_datetime import StringDatetimeValidator

61
src/ytdl_sub/utils/file_path.py Normal file
View file

@ -0,0 +1,61 @@
import os
from pathlib import Path
from typing import Tuple
from ytdl_sub.config.defaults import MAX_FILE_NAME_BYTES
from ytdl_sub.utils.file_handler import get_file_extension
class FilePathTruncater:
_EXTENSION_BYTES = len("-thumb.jpg".encode("utf-8")) + 8
_DEFAULT_MAX_BASE_FILE_NAME_BYTES: int = MAX_FILE_NAME_BYTES - _EXTENSION_BYTES
_MAX_BASE_FILE_NAME_BYTES: int = _DEFAULT_MAX_BASE_FILE_NAME_BYTES
@classmethod
def set_max_file_name_bytes(cls, max_file_name_bytes: int) -> None:
"""Actually sets the max _base_ file name in bytes (excludes extension)"""
max_base_file_name_bytes = max_file_name_bytes - cls._EXTENSION_BYTES
# bound between (extension_bytes + 20, MAX_FILE_NAME_BYTES)
max_base_file_name_bytes = max(max_base_file_name_bytes, 16)
max_base_file_name_bytes = min(
max_base_file_name_bytes, MAX_FILE_NAME_BYTES - cls._EXTENSION_BYTES
)
cls._MAX_BASE_FILE_NAME_BYTES = max_base_file_name_bytes
@classmethod
def _is_file_name_too_long(cls, file_name: str) -> bool:
return len(file_name.encode("utf-8")) > cls._MAX_BASE_FILE_NAME_BYTES
@classmethod
def _get_extension_split(cls, file_name: str) -> Tuple[str, str, str]:
if file_name.endswith("-thumb.jpg"):
ext = "-thumb.jpg"
delimiter = ""
else:
ext = get_file_extension(file_name)
delimiter = "."
return file_name[: -len(ext)], ext, delimiter
@classmethod
def _truncate_file_name(cls, file_name: str) -> str:
file_sub_name, file_ext, delimiter = cls._get_extension_split(file_name)
desired_size = cls._MAX_BASE_FILE_NAME_BYTES - len(file_ext.encode("utf-8")) - 1
while len(file_sub_name.encode("utf-8")) > desired_size:
file_sub_name = file_sub_name[:-1]
return f"{file_sub_name}{delimiter}{file_ext}"
@classmethod
def maybe_truncate_file_path(cls, file_path: str) -> str:
"""Turn into a Path, then a string, to get correct directory separators"""
file_directory, file_name = os.path.split(Path(file_path))
if cls._is_file_name_too_long(file_name):
return str(Path(file_directory) / cls._truncate_file_name(file_name))
return str(file_path)

40
src/ytdl_sub/utils/script.py Normal file
View file

@ -0,0 +1,40 @@
import json
import re
from typing import Any
from typing import Dict
class ScriptUtils:
@classmethod
def add_sanitized_variables(cls, variables: Dict[str, str]) -> Dict[str, str]:
"""
Helper to add sanitized variables to a Script
"""
sanitized_variables = {
f"{name}_sanitized": f"{{%sanitize({name})}}" for name in variables.keys()
}
return dict(variables, **sanitized_variables)
@classmethod
def to_script(cls, value: Any) -> str:
"""
Converts a python value to a script value
"""
if value is None:
out = ""
elif isinstance(value, str):
out = value
elif isinstance(value, int):
out = f"{{%int({value})}}"
elif isinstance(value, float):
out = f"{{%float({value})}}"
elif isinstance(value, bool):
out = f"{{%bool({value})}}"
else:
dumped_json = json.dumps(value, ensure_ascii=False, sort_keys=True)
# Remove triple-single-quotes from JSON to avoid parsing issues
dumped_json = re.sub("'{3,}", "'", dumped_json)
out = f"{{%from_json('''{dumped_json}''')}}"
return out

51
src/ytdl_sub/utils/scriptable.py Normal file
View file

@ -0,0 +1,51 @@
import copy
from abc import ABC
from typing import Any
from typing import Dict
from typing import Set
from ytdl_sub.entries.script.function_scripts import CUSTOM_FUNCTION_SCRIPTS
from ytdl_sub.entries.script.variable_definitions import Variable
from ytdl_sub.entries.script.variable_scripts import UNRESOLVED_VARIABLES
from ytdl_sub.entries.script.variable_scripts import VARIABLE_SCRIPTS
from ytdl_sub.script.script import Script
from ytdl_sub.utils.script import ScriptUtils
class Scriptable(ABC):
"""
Shared class between Entry and Overrides to manage their underlying Script.
"""
def __init__(self):
self.script = Script(
ScriptUtils.add_sanitized_variables(
dict(copy.deepcopy(VARIABLE_SCRIPTS), **copy.deepcopy(CUSTOM_FUNCTION_SCRIPTS))
)
)
self.unresolvable: Set[str] = copy.deepcopy(UNRESOLVED_VARIABLES)
def update_script(self) -> None:
"""
Updates any potential variables to a resolvable. This is done
to avoid re-resolving the same variables over-and-over.
"""
self.script.resolve(unresolvable=self.unresolvable, update=True)
def add(self, values: Dict[str | Variable, Any]) -> None:
"""
Add new values to the script
"""
values_as_str: Dict[str, str] = {
(key.variable_name if isinstance(key, Variable) else key): val
for key, val in values.items()
}
self.unresolvable -= set(list(values_as_str.keys()))
self.script.add(
ScriptUtils.add_sanitized_variables(
{name: ScriptUtils.to_script(value) for name, value in values_as_str.items()}
),
unresolvable=self.unresolvable,
)
self.update_script()

122
src/ytdl_sub/validators/file_path_validators.py
View file

@ -1,12 +1,8 @@
import os
from pathlib import Path
from typing import Any
from typing import Dict
from typing import Tuple
from ytdl_sub.config.defaults import MAX_FILE_NAME_BYTES
from ytdl_sub.utils.file_handler import get_file_extension
from ytdl_sub.utils.subtitles import SUBTITLE_EXTENSIONS
from ytdl_sub.script.script import Script
from ytdl_sub.validators.string_formatter_validators import OverridesStringFormatterValidator
from ytdl_sub.validators.string_formatter_validators import StringFormatterValidator
from ytdl_sub.validators.validators import StringValidator
@ -35,57 +31,8 @@ class FFprobeFileValidator(FFmpegFileValidator):
_ffmpeg_dependency = "ffprobe"
class FilePathValidatorMixin:
_EXTENSION_BYTES = len("-thumb.jpg".encode("utf-8")) + 8
_DEFAULT_MAX_BASE_FILE_NAME_BYTES: int = MAX_FILE_NAME_BYTES - _EXTENSION_BYTES
_MAX_BASE_FILE_NAME_BYTES: int = _DEFAULT_MAX_BASE_FILE_NAME_BYTES
@classmethod
def set_max_file_name_bytes(cls, max_file_name_bytes: int) -> None:
"""Actually sets the max _base_ file name in bytes (excludes extension)"""
max_base_file_name_bytes = max_file_name_bytes - cls._EXTENSION_BYTES
# bound between (extension_bytes + 20, MAX_FILE_NAME_BYTES)
max_base_file_name_bytes = max(max_base_file_name_bytes, 16)
max_base_file_name_bytes = min(
max_base_file_name_bytes, MAX_FILE_NAME_BYTES - cls._EXTENSION_BYTES
)
cls._MAX_BASE_FILE_NAME_BYTES = max_base_file_name_bytes
@classmethod
def _is_file_name_too_long(cls, file_name: str) -> bool:
return len(file_name.encode("utf-8")) > cls._MAX_BASE_FILE_NAME_BYTES
@classmethod
def _get_extension_split(cls, file_name: str) -> Tuple[str, str]:
ext = get_file_extension(file_name)
return file_name[: -len(ext)], ext
@classmethod
def _truncate_file_name(cls, file_name: str) -> str:
file_sub_name, file_ext = cls._get_extension_split(file_name)
desired_size = cls._MAX_BASE_FILE_NAME_BYTES - len(file_ext.encode("utf-8")) - 1
while len(file_sub_name.encode("utf-8")) > desired_size:
file_sub_name = file_sub_name[:-1]
return f"{file_sub_name}.{file_ext}"
@classmethod
def _maybe_truncate_file_path(cls, file_path: Path) -> str:
"""Turn into a Path, then a string, to get correct directory separators"""
file_directory, file_name = os.path.split(Path(file_path))
if cls._is_file_name_too_long(file_name):
return str(Path(file_directory) / cls._truncate_file_name(file_name))
return str(file_path)
# pylint: disable=line-too-long
class StringFormatterFileNameValidator(StringFormatterValidator, FilePathValidatorMixin):
class StringFormatterFileNameValidator(StringFormatterValidator):
"""
Same as a
:class:`StringFormatterValidator <ytdl_sub.validators.string_formatter_validators.StringFormatterValidator>`
@ -97,51 +44,30 @@ class StringFormatterFileNameValidator(StringFormatterValidator, FilePathValidat
_expected_value_type_name = "filepath"
@classmethod
def _is_file_name_too_long(cls, file_name: str) -> bool:
return len(file_name.encode("utf-8")) > cls._MAX_BASE_FILE_NAME_BYTES
@classmethod
def _get_extension_split(cls, file_name: str) -> Tuple[str, str]:
"""
Returns
-------
file_name, ext (including .)
"""
if file_name.endswith(".info.json"):
ext = ".info.json"
elif file_name.endswith("-thumb.jpg"):
ext = "-thumb.jpg"
elif any(file_name.endswith(f".{subtitle_ext}") for subtitle_ext in SUBTITLE_EXTENSIONS):
file_name_split = file_name.split(".")
ext = file_name_split[-1]
# Try to capture .lang.ext
if len(file_name_split) > 2 and len(file_name_split[-2]) < 6:
ext = f".{file_name_split[-2]}.{file_name_split[-1]}"
else:
ext = f".{file_name.rsplit('.', maxsplit=1)[-1]}"
return file_name[: -len(ext)], ext
@classmethod
def _truncate_file_name(cls, file_name: str) -> str:
file_sub_name, file_ext = cls._get_extension_split(file_name)
while len(file_sub_name.encode("utf-8")) > cls._MAX_BASE_FILE_NAME_BYTES:
file_sub_name = file_sub_name[:-1]
return f"{file_sub_name}{file_ext}"
def apply_formatter(self, variable_dict: Dict[str, str]) -> str:
"""Turn into a Path, then a string, to get correct directory separators"""
file_path = Path(super().apply_formatter(variable_dict))
return self._maybe_truncate_file_path(file_path)
def post_process(self, resolved: str) -> str:
return (
Script(
{
"tmp_var_1": resolved,
"tmp_var_2": "{%to_native_filepath(%truncate_filepath_if_too_long(tmp_var_1))}",
}
)
.resolve()
.get_str("tmp_var_2")
)
class OverridesStringFormatterFilePathValidator(OverridesStringFormatterValidator):
_expected_value_type_name = "static filepath"
def apply_formatter(self, variable_dict: Dict[str, str]) -> str:
"""Turn into a Path, then a string, to get correct directory separators"""
return os.path.realpath(super().apply_formatter(variable_dict))
def post_process(self, resolved: str) -> str:
return (
Script(
{
"tmp_var_1": resolved,
"tmp_var_2": "{%to_native_filepath(%truncate_filepath_if_too_long(tmp_var_1))}",
}
)
.resolve()
.get_str("tmp_var_2")
)

9
src/ytdl_sub/validators/string_datetime.py
View file

@ -1,5 +1,3 @@
from typing import Dict
from yt_dlp.utils import datetime_from_str
from ytdl_sub.validators.string_formatter_validators import OverridesStringFormatterValidator
@ -21,10 +19,9 @@ class StringDatetimeValidator(OverridesStringFormatterValidator):
_expected_value_type_name = "datetime string"
def apply_formatter(self, variable_dict: Dict[str, str]) -> str:
output = super().apply_formatter(variable_dict)
def post_process(self, resolved: str) -> str:
try:
_ = datetime_from_str(output)
_ = datetime_from_str(resolved)
except Exception as exc:
raise self._validation_exception(f"Invalid datetime string: {str(exc)}")
return output
return resolved

204
src/ytdl_sub/validators/string_formatter_validators.py
View file

@ -1,20 +1,23 @@
import re
from collections import OrderedDict
from keyword import iskeyword
from typing import Dict
from typing import List
from typing import Set
from typing import Union
from typing import final
from yt_dlp.utils import sanitize_filename
from ytdl_sub.entries.script.variable_definitions import VARIABLES
from ytdl_sub.script.parser import parse
from ytdl_sub.script.script import Script
from ytdl_sub.script.utils.exceptions import UserException
from ytdl_sub.script.utils.exceptions import VariableDoesNotExist
from ytdl_sub.utils.exceptions import InvalidVariableNameException
from ytdl_sub.utils.exceptions import StringFormattingException
from ytdl_sub.utils.exceptions import StringFormattingVariableNotFoundException
from ytdl_sub.validators.validators import DictValidator
from ytdl_sub.validators.validators import ListValidator
from ytdl_sub.validators.validators import LiteralDictValidator
from ytdl_sub.validators.validators import StringValidator
from ytdl_sub.validators.validators import Validator
_fields_validator = re.compile(r"{([a-z][a-z0-9_]+?)}")
_fields_validator = re.compile(r"{([a-z][a-z0-9_]*?)}")
_fields_validator_exception_message: str = (
"{variable_names} must start with a lowercase letter, should only contain lowercase letters, "
@ -74,57 +77,16 @@ class StringFormatterValidator(StringValidator):
"""
_expected_value_type_name = "format string"
_variable_not_found_error_msg_formatter = (
"Format variable '{variable_name}' does not exist. Available variables: {available_fields}"
)
_max_format_recursion = 8
def __validate_and_get_format_variables(self) -> List[str]:
"""
Returns
-------
list[str]
List of format variables in the format string
Raises
------
ValidationException
If the format string contains invalid variable formatting
"""
open_bracket_count = self.format_string.count("{")
close_bracket_count = self.format_string.count("}")
if open_bracket_count != close_bracket_count:
raise self._validation_exception(
"Brackets are reserved for {variable_names} and should contain "
"a single open and close bracket.",
exception_class=StringFormattingException,
)
format_variables: List[str] = list(re.findall(_fields_validator, self.format_string))
if len(format_variables) != open_bracket_count:
raise self._validation_exception(
error_message=_fields_validator_exception_message,
exception_class=StringFormattingException,
)
for variable in format_variables:
if iskeyword(variable):
raise self._validation_exception(
f"'{variable}' is a Python keyword and cannot be used as a variable.",
exception_class=StringFormattingException,
)
return format_variables
def __init__(self, name, value: str):
super().__init__(name=name, value=value)
self.format_variables = self.__validate_and_get_format_variables()
try:
_ = parse(str(value))
except UserException as exc:
raise self._validation_exception(exc) from exc
@final
@property
@final
def format_string(self) -> str:
"""
Returns
@ -133,67 +95,17 @@ class StringFormatterValidator(StringValidator):
"""
return self._value
def _apply_formatter(
self, formatter: "StringFormatterValidator", variable_dict: Dict[str, str]
) -> "StringFormatterValidator":
# Ensure the variable names exist within the entry and overrides
for variable_name in formatter.format_variables:
# If the variable exists, but is sanitized...
if (
variable_name.endswith("_sanitized")
and variable_name.removesuffix("_sanitized") in variable_dict
):
# Resolve just the non-sanitized version, then sanitize it
variable_dict[variable_name] = sanitize_filename(
StringFormatterValidator(
name=self._name, value=f"{{{variable_name.removesuffix('_sanitized')}}}"
).apply_formatter(variable_dict)
)
# If the variable doesn't exist, error
elif variable_name not in variable_dict:
available_fields = ", ".join(sorted(variable_dict.keys()))
raise self._validation_exception(
self._variable_not_found_error_msg_formatter.format(
variable_name=variable_name, available_fields=available_fields
),
exception_class=StringFormattingVariableNotFoundException,
)
# pylint: disable=no-self-use
return StringFormatterValidator(
name=self._name,
value=formatter.format_string.format(**OrderedDict(variable_dict)),
)
def apply_formatter(self, variable_dict: Dict[str, str]) -> str:
def post_process(self, resolved: str) -> str:
"""
Calls `format` on the format string using the variable_dict as input kwargs
Parameters
----------
variable_dict
kwargs to pass to the format string
Returns
-------
Format string formatted
Apply any post processing to the resolved value
"""
formatter = self
recursion_depth = 0
max_depth = self._max_format_recursion
return resolved
while formatter.format_variables and recursion_depth < max_depth:
formatter = self._apply_formatter(formatter=formatter, variable_dict=variable_dict)
recursion_depth += 1
if formatter.format_variables:
raise self._validation_exception(
f"Attempted to format but failed after reaching max recursion depth of "
f"{max_depth}. Try to keep variables dependent on only one other variable at max. "
f"Unresolved variables: {', '.join(sorted(formatter.format_variables))}",
exception_class=StringFormattingException,
)
return formatter.format_string
# pylint: enable=no-self-use
# pylint: disable=line-too-long
@ -208,12 +120,6 @@ class OverridesStringFormatterValidator(StringFormatterValidator):
:class:`nfo_output_directory <ytdl_sub.plugins.output_directory_nfo_tags.OutputDirectoryNfoTagsOptions>`
"""
_variable_not_found_error_msg_formatter = (
"Override variable '{variable_name}' does not exist. For this field, ensure your override "
"variable does not contain any source variables - it is a requirement that this be a "
"static string. Available override variables: {available_fields}"
)
# pylint: enable=line-too-long
@ -221,15 +127,15 @@ class OverridesStringFormatterValidator(StringFormatterValidator):
class OverridesIntegerFormatterValidator(StringFormatterValidator):
_expected_value_type_name = "integer"
def apply_formatter(self, variable_dict: Dict[str, str]) -> str:
output = super().apply_formatter(variable_dict)
def post_process(self, resolved: str) -> str:
try:
int(output)
int(resolved)
except Exception as exc:
raise self._validation_exception(
f"Expected an integer, but received '{output}'"
f"Expected an integer, but received '{resolved}'"
) from exc
return output
return resolved
class ListFormatterValidator(ListValidator[StringFormatterValidator]):
@ -268,3 +174,63 @@ class OverridesDictFormatterValidator(DictFormatterValidator):
"""
_key_validator = OverridesStringFormatterValidator
def _validate_formatter(
mock_script: Script,
unresolved_variables: Set[str],
formatter_validator: Union[StringFormatterValidator, OverridesStringFormatterValidator],
) -> None:
try:
unresolvable = unresolved_variables
if isinstance(formatter_validator, OverridesStringFormatterValidator):
unresolvable = unresolved_variables.union({VARIABLES.entry_metadata.variable_name})
mock_script.resolve_once(
{"tmp_var": formatter_validator.format_string},
unresolvable=unresolvable,
)
except VariableDoesNotExist as exc:
raise StringFormattingVariableNotFoundException(exc) from exc
def validate_formatters(
script: Script,
unresolved_variables: Set[str],
validator: Validator,
) -> None:
"""
Ensure all OverridesStringFormatterValidator's only contain variables from the overrides
and resolve.
"""
if isinstance(validator, DictValidator):
# pylint: disable=protected-access
# Usage of protected variables in other validators is fine. The reason to keep
# them protected is for readability when using them in subscriptions.
for validator_value in validator._validator_dict.values():
validate_formatters(
script=script,
unresolved_variables=unresolved_variables,
validator=validator_value,
)
# pylint: enable=protected-access
elif isinstance(validator, ListValidator):
for list_value in validator.list:
validate_formatters(
script=script,
unresolved_variables=unresolved_variables,
validator=list_value,
)
elif isinstance(validator, (StringFormatterValidator, OverridesStringFormatterValidator)):
_validate_formatter(
mock_script=script,
unresolved_variables=unresolved_variables,
formatter_validator=validator,
)
elif isinstance(validator, (DictFormatterValidator, OverridesDictFormatterValidator)):
for validator_value in validator.dict.values():
_validate_formatter(
mock_script=script,
unresolved_variables=unresolved_variables,
formatter_validator=validator_value,
)

30
src/ytdl_sub/ytdl_additions/enhanced_download_archive.py
View file

@ -14,7 +14,9 @@ from yt_dlp import DateRange
from yt_dlp.utils import make_archive_id
from ytdl_sub.entries.entry import Entry
from ytdl_sub.entries.variables.kwargs import SPLIT_BY_CHAPTERS_PARENT_ENTRY
from ytdl_sub.entries.entry import ytdl_sub_split_by_chapters_parent_uid
from ytdl_sub.entries.script.variable_definitions import VARIABLES
from ytdl_sub.entries.script.variable_definitions import VariableDefinitions
from ytdl_sub.utils.file_handler import FileHandler
from ytdl_sub.utils.file_handler import FileHandlerTransactionLog
from ytdl_sub.utils.file_handler import FileMetadata
@ -22,6 +24,8 @@ from ytdl_sub.utils.logger import Logger
logger = Logger.get("archive")
v: VariableDefinitions = VARIABLES
@dataclass
class DownloadMapping:
@ -71,8 +75,8 @@ class DownloadMapping:
DownloadMapping for the entry
"""
return DownloadMapping(
upload_date=entry.upload_date_standardized,
extractor=entry.extractor,
upload_date=entry.get(v.upload_date_standardized, str),
extractor=entry.download_archive_extractor,
file_names=set(),
)
@ -219,10 +223,14 @@ class DownloadMappings:
-------
self
"""
if entry.uid not in self.entry_ids:
self._entry_mappings[entry.uid] = DownloadMapping.from_entry(entry=entry)
uid = entry.uid
if parent_uid := entry.try_get(ytdl_sub_split_by_chapters_parent_uid, str):
uid = parent_uid
self._entry_mappings[entry.uid].file_names.add(entry_file_path)
if uid not in self.entry_ids:
self._entry_mappings[uid] = DownloadMapping.from_entry(entry=entry)
self._entry_mappings[uid].file_names.add(entry_file_path)
return self
def remove_entry(self, entry_id: str) -> "DownloadMappings":
@ -646,15 +654,7 @@ class EnhancedDownloadArchive:
if output_file_name is None:
output_file_name = file_name
# If the entry is created from splitting via chapters, store it to the mapping
# using its parent entry
if entry and entry.kwargs_contains(SPLIT_BY_CHAPTERS_PARENT_ENTRY):
parent_entry = Entry(
entry_dict=entry.kwargs(SPLIT_BY_CHAPTERS_PARENT_ENTRY),
working_directory=entry.working_directory(),
)
self.mapping.add_entry(parent_entry, entry_file_path=output_file_name)
elif entry:
if entry:
self.mapping.add_entry(entry=entry, entry_file_path=output_file_name)
is_modified = self._file_handler.move_file_to_output_directory(

9
tests/conftest.py
View file

@ -19,6 +19,7 @@ from resources import file_fixture_path
from yt_dlp.utils import sanitize_filename
from ytdl_sub.config.config_file import ConfigFile
from ytdl_sub.entries.script.custom_functions import CustomFunctions
from ytdl_sub.subscriptions.subscription_download import SubscriptionDownload
from ytdl_sub.utils.file_handler import FileHandler
from ytdl_sub.utils.logger import Logger
@ -26,6 +27,14 @@ from ytdl_sub.utils.logger import LoggerLevels
from ytdl_sub.utils.yaml import load_yaml
@pytest.fixture(autouse=True)
def register_custom_functions():
"""
Clean logs after every test
"""
CustomFunctions.register()
@pytest.fixture(autouse=True)
def cleanup_debug_file():
"""

28
tests/e2e/plugins/test_regex.py
View file

@ -302,32 +302,40 @@ class TestRegex:
preset_dict=regex_subscription_dict,
)
def test_regex_fails_capture_group_is_source_variable(
def test_regex_fails_capture_group_is_entry_variable(
self, regex_subscription_dict, default_config
):
regex_subscription_dict["regex"]["from"]["title"]["capture_group_names"][0] = "uid"
regex_subscription_dict["regex"]["from"]["playlist_uid"] = {
"match": [".*http:\\/\\/(.+).com.*"],
"capture_group_names": ["uid"],
}
with pytest.raises(
ValidationException,
match=re.escape(
"'uid' cannot be used as a capture group name because it is a source variable"
"Cannot use the variable name uid because it exists as a built-in "
"ytdl-sub variable name."
),
):
_ = Subscription.from_dict(
config=default_config,
preset_name="test_regex_fails_capture_group_is_source_variable",
preset_name="test_regex_fails_capture_group_is_entry_variable",
preset_dict=regex_subscription_dict,
)
def test_regex_fails_capture_group_is_override_variable(
self, regex_subscription_dict, default_config
):
regex_subscription_dict["regex"]["from"]["title"]["capture_group_names"][
0
] = "in_regex_default"
regex_subscription_dict["regex"]["from"]["playlist_uid"] = {
"match": [".*http:\\/\\/(.+).com.*"],
"capture_group_names": ["contains_regex_default"],
}
with pytest.raises(
ValidationException,
match=re.escape(
"'in_regex_default' cannot be used as a capture group name because it is an override variable"
"Override variable with name contains_regex_default cannot be used since it is "
"added by a plugin."
),
):
_ = Subscription.from_dict(
@ -344,9 +352,7 @@ class TestRegex:
)
with pytest.raises(
ValidationException,
match=re.escape(
"cannot regex capture 'dne' because it is not a source or override variable"
),
match=re.escape("cannot regex capture 'dne' because it is not a defined variable"),
):
_ = Subscription.from_dict(
config=default_config,

3
tests/e2e/plugins/test_split_by_chapters.py
View file

@ -34,7 +34,7 @@ def yt_album_as_chapters_with_regex_preset_dict(yt_album_as_chapters_preset_dict
"from": {
# Ensure regex can handle override variables that come from the
# post-metadata stage
"override_chapter_title": {
"chapter_title": {
"match": r"\d+\. (.+)",
"capture_group_names": "captured_track_title",
"capture_group_defaults": "{chapter_title}",
@ -56,7 +56,6 @@ def yt_album_as_chapters_with_regex_preset_dict(yt_album_as_chapters_preset_dict
}
},
"overrides": {
"override_chapter_title": "{chapter_title}",
"track_title": "{captured_track_title}",
"track_album": "{captured_track_album}",
"track_artist": "{captured_track_artist}",

2
tests/unit/config/test_config_file.py
View file

@ -5,7 +5,7 @@ from typing import Optional
import pytest
from ytdl_sub.config.config_file import ConfigFile
from ytdl_sub.config.plugin_mapping import PluginMapping
from ytdl_sub.config.plugin.plugin_mapping import PluginMapping
from ytdl_sub.config.preset import PRESET_KEYS
from ytdl_sub.utils.exceptions import ValidationException

151
tests/unit/config/test_preset.py
View file

@ -129,7 +129,7 @@ class TestPreset:
):
with pytest.raises(
StringFormattingVariableNotFoundException,
match="Format variable 'dne_var' does not exist",
match="Variable dne_var does not exist.",
):
_ = Preset(
config=config_file,
@ -145,7 +145,7 @@ class TestPreset:
):
with pytest.raises(
StringFormattingVariableNotFoundException,
match="Override variable 'dne_var' does not exist",
match="Variable dne_var does not exist",
):
_ = Preset(
config=config_file,
@ -161,7 +161,7 @@ class TestPreset:
):
with pytest.raises(
StringFormattingVariableNotFoundException,
match="Format variable 'dne_var' does not exist",
match="Variable dne_var does not exist",
):
_ = Preset(
config=config_file,
@ -182,7 +182,7 @@ class TestPreset:
):
with pytest.raises(
StringFormattingVariableNotFoundException,
match="Format variable 'dne_var' does not exist",
match="Variable dne_var does not exist",
):
_ = Preset(
config=config_file,
@ -208,22 +208,151 @@ class TestPreset:
},
)
def test_preset_with_multi_url__contains_all_empty_urls_errors(
self, config_file, output_options
@pytest.mark.parametrize(
"entry_variable_name",
[
"title",
"playlist_uid",
"source_title",
"playlist_max_upload_year",
],
)
def test_preset_error_override_variable_collides_with_entry_variable(
self, config_file, output_options, youtube_video, entry_variable_name: str
):
with pytest.raises(
ValidationException,
match=re.escape(
"Validation error in test.download: Must contain at least one "
"url that is non-empty"
f"Override variable with name {entry_variable_name} cannot be used since"
" it is a built-in ytdl-sub entry variable name."
),
):
_ = Preset(
config=config_file,
name="test",
value={
"download": [{"url": "{url}"}, {"url": "{url2}"}],
"output_options": output_options,
"overrides": {"url": "", "url2": ""},
"download": youtube_video,
"output_options": {"output_directory": "dir", "file_name": "{dne_var}"},
"overrides": {entry_variable_name: "fail"},
},
)
def test_preset_error_override_variable_collides_added_variable(
self, config_file, output_options, youtube_video
):
with pytest.raises(
ValidationException,
match=re.escape(
f"Override variable with name subtitles_ext cannot be used since"
" it is added by a plugin."
),
):
_ = Preset(
config=config_file,
name="test",
value={
"download": youtube_video,
"output_options": {"output_directory": "dir", "file_name": "ack"},
"subtitles": {
"embed_subtitles": True,
},
"overrides": {"subtitles_ext": "collide"},
},
)
@pytest.mark.parametrize(
"function_name",
[
"%extract_field_from_siblings",
"%extract_field_from_metadata_array",
"%sanitize",
"%array",
],
)
def test_preset_error_override_variable_collides_with_custom_function(
self, config_file, output_options, youtube_video, function_name: str
):
with pytest.raises(
ValidationException,
match=re.escape(
f"Override function definition with name {function_name} cannot be used since"
" it is a built-in ytdl-sub function name."
),
):
_ = Preset(
config=config_file,
name="test",
value={
"download": youtube_video,
"output_options": {"output_directory": "dir", "file_name": "{dne_var}"},
"overrides": {function_name: "fail"},
},
)
def test_preset_error_override_added_variable_collides_with_built_in(
self, config_file, output_options
):
with pytest.raises(
ValidationException,
match=re.escape(
"Cannot use the variable name title because it exists as a "
"built-in ytdl-sub variable name."
),
):
_ = Preset(
config=config_file,
name="test",
value={
"download": {
"url": "youtube.com/watch?v=123abc",
"variables": {"title": "nope"},
},
"output_options": {"output_directory": "dir", "file_name": "acjk"},
},
)
def test_preset_error_override_added_variable_collides_with_override(
self, config_file, output_options
):
with pytest.raises(
ValidationException,
match=re.escape(
"Override variable with name the_bad_one cannot be used since "
"it is added by a plugin."
),
):
_ = Preset(
config=config_file,
name="test",
value={
"download": {
"url": "youtube.com/watch?v=123abc",
"variables": {"the_bad_one": "should error"},
},
"output_options": {"output_directory": "dir", "file_name": "acjk"},
"overrides": {"the_bad_one": "ack"},
},
)
def test_preset_error_added_url_variable_cannot_resolve(self, config_file, output_options):
with pytest.raises(
ValidationException,
match=re.escape(
"variable the_bad_one cannot use the variables subtitles_ext because it "
"depends on other variables that are computed later in execution"
),
):
_ = Preset(
config=config_file,
name="test",
value={
"download": {
"url": "youtube.com/watch?v=123abc",
"variables": {"the_bad_one": "{subtitles_ext}"},
},
"subtitles": {
"embed_subtitles": True,
},
"output_options": {"output_directory": "dir", "file_name": "acjk"},
},
)

18
tests/unit/config/test_subscription.py
View file

@ -257,7 +257,6 @@ def test_subscription_file_value_applies_sub_file_takes_precedence(
assert value_sub.get("test_file_subscription_value") == "is_overwritten"
assert value_sub.get("test_config_subscription_value") == "original"
assert value_sub.get("subscription_name") == "test_value"
assert value_sub.get("subscription_name_sanitized") == "test_value"
assert value_sub.get("subscription_value") == "is_overwritten"
assert value_sub.get("current_override") == "__preset__" # ensure __preset__ takes precedence
@ -276,7 +275,6 @@ def test_subscription_file_value_applies_from_config(
assert value_sub.get("test_file_subscription_value") == "original"
assert value_sub.get("test_config_subscription_value") == "is_overwritten"
assert value_sub.get("subscription_name") == "test_value"
assert value_sub.get("subscription_name_sanitized") == "test_value"
assert value_sub.get("subscription_value") == "is_overwritten"
assert value_sub.get("current_override") == "__preset__" # ensure __preset__ takes precedence
@ -297,13 +295,11 @@ def test_subscription_file_value_applies_from_config_and_nested(
assert sub_1.get("test_config_subscription_value") == "is_1_overwritten"
assert sub_1.get("subscription_name") == "test_1"
assert sub_1.get("subscription_name_sanitized") == "test_1"
assert sub_1.get("subscription_value") == "is_1_overwritten"
assert sub_1.get("current_override") == "__preset__" # ensure __preset__ takes precedence
assert sub_2_1.get("test_config_subscription_value") == "is_2_1_overwritten"
assert sub_2_1.get("subscription_name") == "test_2_1"
assert sub_2_1.get("subscription_name_sanitized") == "test_2_1"
assert sub_2_1.get("subscription_value") == "is_2_1_overwritten"
assert sub_2_1.get("current_override") == "__preset__" # ensure __preset__ takes precedence
@ -322,7 +318,6 @@ def test_subscription_list(
sub_2_1 = [sub for sub in subs if sub.name == "test_2_1"][0].overrides.dict_with_format_strings
assert sub_2_1.get("subscription_name") == "test_2_1"
assert sub_2_1.get("subscription_name_sanitized") == "test_2_1"
assert sub_2_1.get("subscription_value") == "is_2_1_overwritten"
assert sub_2_1.get("subscription_value_1") == "is_2_1_overwritten"
assert sub_2_1.get("subscription_value_2") == "is_2_1_list_2"
@ -343,7 +338,6 @@ def test_subscription_overrides_tilda(
sub_2_1 = [sub for sub in subs if sub.name == "test_2_1"][0].overrides.dict_with_format_strings
assert sub_2_1.get("subscription_name") == "test_2_1"
assert sub_2_1.get("subscription_name_sanitized") == "test_2_1"
assert sub_2_1.get("current_override") == "test_2_1" # tilda sub takes precedence
@ -371,7 +365,6 @@ def test_subscription_file_value_applies_from_config_and_nested_and_indent_varia
assert sub_1.get("test_config_subscription_value") == "is_1_overwritten"
assert sub_1.get("subscription_name") == "test_1"
assert sub_1.get("subscription_name_sanitized") == "test_1"
assert sub_1.get("subscription_value") == "is_1_overwritten"
assert sub_1.get("subscription_indent_1") == "INDENT_1"
assert sub_1.get("subscription_indent_2") == "INDENT_2"
@ -379,7 +372,6 @@ def test_subscription_file_value_applies_from_config_and_nested_and_indent_varia
assert sub_2_1.get("test_config_subscription_value") == "is_2_1_overwritten"
assert sub_2_1.get("subscription_name") == "test_2_1"
assert sub_2_1.get("subscription_name_sanitized") == "test_2_1"
assert sub_2_1.get("subscription_value") == "is_2_1_overwritten"
assert sub_2_1.get("subscription_indent_1") == "INDENT_1"
assert sub_2_1.get("subscription_indent_2") == "original_2"
@ -417,7 +409,6 @@ def test_subscription_file_value_applies_from_config_and_nested_and_indent_varia
assert sub_1.get("test_config_subscription_value") == "is_1_overwritten"
assert sub_1.get("subscription_name") == "test_1"
assert sub_1.get("subscription_name_sanitized") == "test_1"
assert sub_1.get("subscription_value") == "is_1_overwritten"
assert sub_1.get("subscription_indent_1") == "INDENT_1"
assert sub_1.get("subscription_indent_2") == "INDENT_2"
@ -426,7 +417,6 @@ def test_subscription_file_value_applies_from_config_and_nested_and_indent_varia
assert sub_2_1.get("test_config_subscription_value") == "is_2_1_overwritten"
assert sub_2_1.get("subscription_name") == "test_2_1"
assert sub_2_1.get("subscription_name_sanitized") == "test_2_1"
assert sub_2_1.get("subscription_value") == "is_2_1_overwritten"
assert sub_2_1.get("subscription_indent_1") == "INDENT_1"
assert sub_2_1.get("subscription_indent_2") == "original_2"
@ -499,7 +489,6 @@ def test_tv_show_subscriptions(config_file: ConfigFile, tv_show_subscriptions_pa
jake_train_overrides = subs[3].overrides.dict_with_format_strings
assert jake_train_overrides["subscription_name"] == "Jake Trains"
assert jake_train_overrides["subscription_name_sanitized"] == "Jake Trains"
assert jake_train_overrides["subscription_value"] == "https://www.youtube.com/@JakeTrains"
assert jake_train_overrides["subscription_indent_1"] == "Kids"
assert jake_train_overrides["subscription_indent_2"] == "TV-Y"
@ -517,7 +506,6 @@ def test_advanced_tv_show_subscriptions(
jake_train_overrides = subs[3].overrides.dict_with_format_strings
assert jake_train_overrides["subscription_name"] == "Jake Trains"
assert jake_train_overrides["subscription_name_sanitized"] == "Jake Trains"
assert jake_train_overrides["subscription_value"] == "https://www.youtube.com/@JakeTrains"
assert jake_train_overrides["subscription_indent_1"] == "Kids"
assert jake_train_overrides["subscription_indent_2"] == "TV-Y"
@ -526,10 +514,6 @@ def test_advanced_tv_show_subscriptions(
overrides = subs[5].overrides
assert overrides.apply_formatter(overrides.dict["subscription_name"]) == "Gardening with Ciscoe"
assert (
overrides.apply_formatter(overrides.dict["subscription_name_sanitized"])
== "Gardening with Ciscoe"
)
assert (
overrides.apply_formatter(overrides.dict["url"])
== "https://www.youtube.com/@gardeningwithciscoe4430"
@ -550,7 +534,6 @@ def test_music_subscriptions(default_config: ConfigFile, music_subscriptions_pat
monk = subs[2].overrides.dict_with_format_strings
assert monk["subscription_name"] == "Stan Getz"
assert monk["subscription_name_sanitized"] == "Stan Getz"
assert monk["subscription_value"] == "https://www.youtube.com/@stangetzofficial/releases"
assert monk["subscription_indent_1"] == "Jazz"
@ -565,7 +548,6 @@ def test_music_video_subscriptions(default_config: ConfigFile, music_video_subsc
monk = subs[1].overrides.dict_with_format_strings
assert monk["subscription_name"] == "Michael Jackson"
assert monk["subscription_name_sanitized"] == "Michael Jackson"
assert (
monk["subscription_value"]
== "https://www.youtube.com/playlist?list=OLAK5uy_mnY03zP6abNWH929q2XhGzWD_2uKJ_n8E"

44
tests/unit/conftest.py
View file

@ -12,18 +12,10 @@ from resources import copy_file_fixture
from ytdl_sub.config.config_file import ConfigFile
from ytdl_sub.downloaders.url.downloader import MultiUrlDownloader
from ytdl_sub.downloaders.ytdlp import YTDLP
from ytdl_sub.entries.variables.kwargs import DESCRIPTION
from ytdl_sub.entries.variables.kwargs import EPOCH
from ytdl_sub.entries.variables.kwargs import EXT
from ytdl_sub.entries.variables.kwargs import EXTRACTOR
from ytdl_sub.entries.variables.kwargs import PLAYLIST_COUNT
from ytdl_sub.entries.variables.kwargs import PLAYLIST_ENTRY
from ytdl_sub.entries.variables.kwargs import PLAYLIST_INDEX
from ytdl_sub.entries.variables.kwargs import PLAYLIST_TITLE
from ytdl_sub.entries.variables.kwargs import TITLE
from ytdl_sub.entries.variables.kwargs import UID
from ytdl_sub.entries.variables.kwargs import UPLOAD_DATE
from ytdl_sub.entries.variables.kwargs import WEBPAGE_URL
from ytdl_sub.entries.script.variable_definitions import VARIABLES
from ytdl_sub.entries.script.variable_definitions import VariableDefinitions
v: VariableDefinitions = VARIABLES
@pytest.fixture
@ -62,23 +54,23 @@ def mock_entry_dict_factory(mock_downloaded_file_path) -> Callable:
is_extracted_audio: bool = False,
) -> Dict:
entry_dict = {
UID: uid,
EPOCH: 1596878400,
PLAYLIST_TITLE: playlist_title,
PLAYLIST_INDEX: playlist_index,
PLAYLIST_COUNT: playlist_count,
EXTRACTOR: "mock-entry-dict",
"extractor_key": "mock-entry-dict",
TITLE: f"Mock Entry {uid}",
EXT: "mp4",
UPLOAD_DATE: upload_date,
WEBPAGE_URL: f"https://{uid}.com",
PLAYLIST_ENTRY: {"thumbnails": []},
DESCRIPTION: "The Description",
v.uid.metadata_key: uid,
v.epoch.metadata_key: 1596878400,
v.playlist_title.metadata_key: playlist_title,
v.playlist_index.metadata_key: playlist_index,
v.playlist_count.metadata_key: playlist_count,
v.extractor.metadata_key: "mock-entry-extractor",
v.extractor_key.metadata_key: "mock-entry-dict",
v.title.metadata_key: f"Mock Entry {uid}",
v.ext.metadata_key: "mp4",
v.upload_date.metadata_key: upload_date,
v.webpage_url.metadata_key: f"https://{uid}.com",
v.playlist_metadata.metadata_key: {"thumbnails": []},
v.description.metadata_key: "The Description",
}
if is_youtube_channel:
entry_dict[PLAYLIST_ENTRY]["thumbnails"] = [
entry_dict[v.playlist_metadata.metadata_key]["thumbnails"] = [
{
"id": "avatar_uncropped",
"url": "https://avatar_uncropped.com",

20
tests/unit/entries/conftest.py
View file

@ -163,7 +163,7 @@ def mock_entry_kwargs(
"id": uid,
"epoch": 1596878400,
"extractor": extractor,
"extractor_key": extractor,
"extractor_key": "test_extractor_key",
"title": title,
"ext": ext,
"upload_date": upload_date,
@ -174,20 +174,4 @@ def mock_entry_kwargs(
@pytest.fixture
def mock_entry(mock_entry_kwargs):
return Entry(entry_dict=mock_entry_kwargs, working_directory=".")
@pytest.fixture
def validate_entry_dict_contains_valid_formatters():
def _validate_entry_dict_contains_valid_formatters(entry: Entry):
for key, value in entry.to_dict().items():
expected_string = f"test {value} formatting works"
formatter = StringFormatterValidator(
name="test", value=f"test {{{key}}} formatting works"
)
assert formatter.apply_formatter(entry.to_dict()) == expected_string
return True
return _validate_entry_dict_contains_valid_formatters
return Entry(entry_dict=mock_entry_kwargs, working_directory=".").initialize_script()

59
tests/unit/entries/test_entry.py
View file

@ -1,22 +1,19 @@
import pytest
from ytdl_sub.entries.entry import Entry
from ytdl_sub.entries.script.variable_definitions import VARIABLES
from ytdl_sub.entries.script.variable_definitions import VariableDefinitions
v: VariableDefinitions = VARIABLES
class TestEntry(object):
def test_entry_to_dict(self, mock_entry, mock_entry_to_dict):
assert mock_entry.to_dict() == mock_entry_to_dict
out = mock_entry.to_dict()
def test_entry_dict_contains_valid_formatters(
self, mock_entry, validate_entry_dict_contains_valid_formatters
):
assert validate_entry_dict_contains_valid_formatters(mock_entry)
def test_entry_missing_kwarg(self, mock_entry):
key = "dne"
expected_error_msg = f"Expected '{key}' in Entry but does not exist."
assert mock_entry.kwargs_contains(key) is False
with pytest.raises(KeyError, match=expected_error_msg):
mock_entry.kwargs(key)
# HACK: Ensure legacy variables are in new output and equal
for key, expected_value in mock_entry_to_dict.items():
assert out[key] == expected_value, f"{key} does not equal"
@pytest.mark.parametrize(
"upload_date, year_rev, month_rev, day_rev, month_rev_pad, day_rev_pad",
@ -26,16 +23,23 @@ class TestEntry(object):
],
)
def test_entry_reverse_variables(
self, mock_entry, upload_date, year_rev, month_rev, day_rev, month_rev_pad, day_rev_pad
self,
mock_entry_kwargs,
upload_date,
year_rev,
month_rev,
day_rev,
month_rev_pad,
day_rev_pad,
):
mock_entry._kwargs["upload_date"] = upload_date
assert mock_entry.upload_year_truncated_reversed == year_rev
assert mock_entry.upload_month_reversed == month_rev
assert mock_entry.upload_day_reversed == day_rev
assert mock_entry.upload_month_reversed_padded == month_rev_pad
assert mock_entry.upload_day_reversed_padded == day_rev_pad
mock_entry_kwargs["upload_date"] = upload_date
entry = Entry(entry_dict=mock_entry_kwargs, working_directory=".").initialize_script()
assert entry.get(v.upload_year_truncated_reversed, int) == year_rev
assert entry.get(v.upload_month_reversed, int) == month_rev
assert entry.get(v.upload_day_reversed, int) == day_rev
assert entry.get(v.upload_month_reversed_padded, str) == month_rev_pad
assert entry.get(v.upload_day_reversed_padded, str) == day_rev_pad
@pytest.mark.parametrize(
"upload_date, day_year, day_year_rev, day_year_pad, day_year_rev_pad",
@ -45,11 +49,12 @@ class TestEntry(object):
],
)
def test_entry_upload_day_of_year_variables(
self, mock_entry, upload_date, day_year, day_year_rev, day_year_pad, day_year_rev_pad
self, mock_entry_kwargs, upload_date, day_year, day_year_rev, day_year_pad, day_year_rev_pad
):
mock_entry._kwargs["upload_date"] = upload_date
mock_entry_kwargs["upload_date"] = upload_date
entry = Entry(entry_dict=mock_entry_kwargs, working_directory=".").initialize_script()
assert mock_entry.upload_day_of_year == day_year
assert mock_entry.upload_day_of_year_reversed == day_year_rev
assert mock_entry.upload_day_of_year_padded == day_year_pad
assert mock_entry.upload_day_of_year_reversed_padded == day_year_rev_pad
assert entry.get(v.upload_day_of_year, int) == day_year
assert entry.get(v.upload_day_of_year_reversed, int) == day_year_rev
assert entry.get(v.upload_day_of_year_padded, str) == day_year_pad
assert entry.get(v.upload_day_of_year_reversed_padded, str) == day_year_rev_pad

Some files were not shown because too many files have changed in this diff Show more