From e92b1cd12a8fdd54dd087cc58854f1a2f138e073 Mon Sep 17 00:00:00 2001 From: Jesse Bannon Date: Mon, 18 Dec 2023 16:08:15 -0800 Subject: [PATCH] [BACKEND][HUGE] Function Support in variable syntax (#838) A complete gutting of the internals of ytdl-sub to support functions in our variable syntax, in addition to being able to access a yt-dlp entry's .info.json fields using functions. Functionally, ytdl-sub should still look and behave the same from a user-perspective. With so many lines of code changed (+8927, -2708), no doubt there will be new issues. Please make a GH issue or reach out on Discord if your config/subscriptions break in any way/shape/form. Details on how to use function support will come soon in the form of proper documentation in our readthedocs. --- Makefile | 4 +- docs/config.rst | 4 +- pyproject.toml | 6 + setup.cfg | 1 - src/ytdl_sub/cli/entrypoint.py | 2 + src/ytdl_sub/config/config_file.py | 4 +- src/ytdl_sub/config/overrides.py | 176 ++++ src/ytdl_sub/config/plugin/__init__.py | 0 src/ytdl_sub/config/{ => plugin}/plugin.py | 36 +- src/ytdl_sub/config/plugin/plugin_mapping.py | 200 ++++ .../config/plugin/plugin_operation.py | 9 + src/ytdl_sub/config/plugin/preset_plugins.py | 46 + src/ytdl_sub/config/plugin_mapping.py | 79 -- src/ytdl_sub/config/preset.py | 163 +-- src/ytdl_sub/config/preset_options.py | 167 --- src/ytdl_sub/config/validators/__init__.py | 0 src/ytdl_sub/config/validators/options.py | 59 ++ .../config/validators/variable_validation.py | 178 ++++ .../info_json/info_json_downloader.py | 32 +- src/ytdl_sub/downloaders/source_plugin.py | 8 +- src/ytdl_sub/downloaders/url/downloader.py | 73 +- src/ytdl_sub/downloaders/url/validators.py | 59 +- .../downloaders/ytdl_options_builder.py | 2 +- src/ytdl_sub/entries/base_entry.py | 354 +------ src/ytdl_sub/entries/entry.py | 133 ++- src/ytdl_sub/entries/entry_parent.py | 162 +-- src/ytdl_sub/entries/script/__init__.py | 0 .../entries/script/custom_functions.py | 178 ++++ .../entries/script/function_scripts.py | 37 + .../entries/script/variable_definitions.py | 991 ++++++++++++++++++ .../entries/script/variable_scripts.py | 296 ++++++ .../entries/variables/entry_variables.py | 857 --------------- src/ytdl_sub/entries/variables/kwargs.py | 68 -- .../entries/variables/override_variables.py | 26 + src/ytdl_sub/plugins/audio_extract.py | 18 +- src/ytdl_sub/plugins/chapters.py | 61 +- src/ytdl_sub/plugins/date_range.py | 4 +- src/ytdl_sub/plugins/embed_thumbnail.py | 6 +- src/ytdl_sub/plugins/file_convert.py | 45 +- src/ytdl_sub/plugins/format.py | 4 +- src/ytdl_sub/plugins/internal/view.py | 8 +- src/ytdl_sub/plugins/match_filters.py | 4 +- src/ytdl_sub/plugins/music_tags.py | 12 +- src/ytdl_sub/plugins/nfo_tags.py | 4 +- .../plugins/output_directory_nfo_tags.py | 2 +- src/ytdl_sub/plugins/regex.py | 239 ++--- src/ytdl_sub/plugins/split_by_chapters.py | 121 +-- src/ytdl_sub/plugins/subtitles.py | 39 +- src/ytdl_sub/plugins/throttle_protection.py | 6 +- src/ytdl_sub/plugins/video_tags.py | 4 +- src/ytdl_sub/script/__init__.py | 0 src/ytdl_sub/script/functions/__init__.py | 82 ++ .../script/functions/array_functions.py | 147 +++ .../script/functions/boolean_functions.py | 89 ++ .../script/functions/conditional_functions.py | 31 + .../script/functions/date_functions.py | 13 + .../script/functions/error_functions.py | 23 + .../script/functions/json_functions.py | 40 + .../script/functions/map_functions.py | 102 ++ .../script/functions/numeric_functions.py | 88 ++ .../script/functions/regex_functions.py | 42 + .../script/functions/string_functions.py | 94 ++ src/ytdl_sub/script/parser.py | 601 +++++++++++ src/ytdl_sub/script/script.py | 484 +++++++++ src/ytdl_sub/script/script_output.py | 52 + src/ytdl_sub/script/types/__init__.py | 0 src/ytdl_sub/script/types/array.py | 58 + src/ytdl_sub/script/types/function.py | 288 +++++ src/ytdl_sub/script/types/map.py | 66 ++ src/ytdl_sub/script/types/resolvable.py | 253 +++++ src/ytdl_sub/script/types/syntax_tree.py | 52 + src/ytdl_sub/script/types/variable.py | 28 + .../script/types/variable_dependency.py | 174 +++ src/ytdl_sub/script/utils/__init__.py | 0 .../script/utils/exception_formatters.py | 153 +++ src/ytdl_sub/script/utils/exceptions.py | 96 ++ src/ytdl_sub/script/utils/name_validation.py | 60 ++ src/ytdl_sub/script/utils/type_checking.py | 243 +++++ .../subscriptions/base_subscription.py | 4 +- .../subscriptions/subscription_download.py | 32 +- .../subscriptions/subscription_validators.py | 2 +- .../subscription_ytdl_options.py | 2 +- src/ytdl_sub/utils/chapters.py | 55 +- src/ytdl_sub/utils/datetime.py | 2 +- src/ytdl_sub/utils/file_path.py | 61 ++ src/ytdl_sub/utils/script.py | 40 + src/ytdl_sub/utils/scriptable.py | 51 + .../validators/file_path_validators.py | 122 +-- src/ytdl_sub/validators/string_datetime.py | 9 +- .../validators/string_formatter_validators.py | 204 ++-- .../enhanced_download_archive.py | 30 +- tests/conftest.py | 9 + tests/e2e/plugins/test_regex.py | 28 +- tests/e2e/plugins/test_split_by_chapters.py | 3 +- tests/unit/config/test_config_file.py | 2 +- tests/unit/config/test_preset.py | 151 ++- tests/unit/config/test_subscription.py | 18 - tests/unit/conftest.py | 44 +- tests/unit/entries/conftest.py | 20 +- tests/unit/entries/test_entry.py | 59 +- .../prebuilt_presets/test_prebuilt_presets.py | 29 +- tests/unit/script/__init__.py | 0 tests/unit/script/conftest.py | 14 + tests/unit/script/functions/__init__.py | 0 .../script/functions/test_array_functions.py | 98 ++ .../functions/test_boolean_functions.py | 128 +++ .../functions/test_conditional_functions.py | 35 + .../script/functions/test_date_functions.py | 17 + .../script/functions/test_error_functions.py | 25 + .../script/functions/test_json_functions.py | 31 + .../script/functions/test_map_functions.py | 140 +++ .../functions/test_numeric_functions.py | 78 ++ .../script/functions/test_regex_functions.py | 45 + .../script/functions/test_string_functions.py | 109 ++ tests/unit/script/test_parser.py | 210 ++++ tests/unit/script/test_script.py | 63 ++ tests/unit/script/types/__init__.py | 0 tests/unit/script/types/test_array.py | 119 +++ tests/unit/script/types/test_bool.py | 66 ++ .../unit/script/types/test_custom_function.py | 235 +++++ tests/unit/script/types/test_float.py | 81 ++ tests/unit/script/types/test_function.py | 97 ++ tests/unit/script/types/test_integer.py | 81 ++ .../unit/script/types/test_lambda_function.py | 127 +++ tests/unit/script/types/test_map.py | 200 ++++ tests/unit/script/types/test_string.py | 67 ++ tests/unit/script/types/test_variable.py | 130 +++ tests/unit/utils/test_script_utils.py | 33 + .../validators/test_file_path_validators.py | 26 +- .../test_string_formatter_validator.py | 155 +-- tools/linter | 4 +- 131 files changed, 9015 insertions(+), 2722 deletions(-) create mode 100644 src/ytdl_sub/config/overrides.py create mode 100644 src/ytdl_sub/config/plugin/__init__.py rename src/ytdl_sub/config/{ => plugin}/plugin.py (77%) create mode 100644 src/ytdl_sub/config/plugin/plugin_mapping.py create mode 100644 src/ytdl_sub/config/plugin/plugin_operation.py create mode 100644 src/ytdl_sub/config/plugin/preset_plugins.py delete mode 100644 src/ytdl_sub/config/plugin_mapping.py create mode 100644 src/ytdl_sub/config/validators/__init__.py create mode 100644 src/ytdl_sub/config/validators/options.py create mode 100644 src/ytdl_sub/config/validators/variable_validation.py create mode 100644 src/ytdl_sub/entries/script/__init__.py create mode 100644 src/ytdl_sub/entries/script/custom_functions.py create mode 100644 src/ytdl_sub/entries/script/function_scripts.py create mode 100644 src/ytdl_sub/entries/script/variable_definitions.py create mode 100644 src/ytdl_sub/entries/script/variable_scripts.py delete mode 100644 src/ytdl_sub/entries/variables/entry_variables.py delete mode 100644 src/ytdl_sub/entries/variables/kwargs.py create mode 100644 src/ytdl_sub/script/__init__.py create mode 100644 src/ytdl_sub/script/functions/__init__.py create mode 100644 src/ytdl_sub/script/functions/array_functions.py create mode 100644 src/ytdl_sub/script/functions/boolean_functions.py create mode 100644 src/ytdl_sub/script/functions/conditional_functions.py create mode 100644 src/ytdl_sub/script/functions/date_functions.py create mode 100644 src/ytdl_sub/script/functions/error_functions.py create mode 100644 src/ytdl_sub/script/functions/json_functions.py create mode 100644 src/ytdl_sub/script/functions/map_functions.py create mode 100644 src/ytdl_sub/script/functions/numeric_functions.py create mode 100644 src/ytdl_sub/script/functions/regex_functions.py create mode 100644 src/ytdl_sub/script/functions/string_functions.py create mode 100644 src/ytdl_sub/script/parser.py create mode 100644 src/ytdl_sub/script/script.py create mode 100644 src/ytdl_sub/script/script_output.py create mode 100644 src/ytdl_sub/script/types/__init__.py create mode 100644 src/ytdl_sub/script/types/array.py create mode 100644 src/ytdl_sub/script/types/function.py create mode 100644 src/ytdl_sub/script/types/map.py create mode 100644 src/ytdl_sub/script/types/resolvable.py create mode 100644 src/ytdl_sub/script/types/syntax_tree.py create mode 100644 src/ytdl_sub/script/types/variable.py create mode 100644 src/ytdl_sub/script/types/variable_dependency.py create mode 100644 src/ytdl_sub/script/utils/__init__.py create mode 100644 src/ytdl_sub/script/utils/exception_formatters.py create mode 100644 src/ytdl_sub/script/utils/exceptions.py create mode 100644 src/ytdl_sub/script/utils/name_validation.py create mode 100644 src/ytdl_sub/script/utils/type_checking.py create mode 100644 src/ytdl_sub/utils/file_path.py create mode 100644 src/ytdl_sub/utils/script.py create mode 100644 src/ytdl_sub/utils/scriptable.py create mode 100644 tests/unit/script/__init__.py create mode 100644 tests/unit/script/conftest.py create mode 100644 tests/unit/script/functions/__init__.py create mode 100644 tests/unit/script/functions/test_array_functions.py create mode 100644 tests/unit/script/functions/test_boolean_functions.py create mode 100644 tests/unit/script/functions/test_conditional_functions.py create mode 100644 tests/unit/script/functions/test_date_functions.py create mode 100644 tests/unit/script/functions/test_error_functions.py create mode 100644 tests/unit/script/functions/test_json_functions.py create mode 100644 tests/unit/script/functions/test_map_functions.py create mode 100644 tests/unit/script/functions/test_numeric_functions.py create mode 100644 tests/unit/script/functions/test_regex_functions.py create mode 100644 tests/unit/script/functions/test_string_functions.py create mode 100644 tests/unit/script/test_parser.py create mode 100644 tests/unit/script/test_script.py create mode 100644 tests/unit/script/types/__init__.py create mode 100644 tests/unit/script/types/test_array.py create mode 100644 tests/unit/script/types/test_bool.py create mode 100644 tests/unit/script/types/test_custom_function.py create mode 100644 tests/unit/script/types/test_float.py create mode 100644 tests/unit/script/types/test_function.py create mode 100644 tests/unit/script/types/test_integer.py create mode 100644 tests/unit/script/types/test_lambda_function.py create mode 100644 tests/unit/script/types/test_map.py create mode 100644 tests/unit/script/types/test_string.py create mode 100644 tests/unit/script/types/test_variable.py create mode 100644 tests/unit/utils/test_script_utils.py diff --git a/Makefile b/Makefile index 30ca26b3..df03c227 100644 --- a/Makefile +++ b/Makefile @@ -17,12 +17,10 @@ lint: @-isort . @-black . @-pylint src/ - @-pydocstyle src/* check_lint: isort . --check-only --diff \ && black . --check \ - && pylint src/ \ - && pydocstyle src/* + && pylint src/ wheel: clean $(shell echo "__pypi_version__ = \"$(PYPI_VERSION)\"\n__local_version__ = \"$(LOCAL_VERSION)\"" > src/ytdl_sub/__init__.py) cat src/ytdl_sub/__init__.py diff --git a/docs/config.rst b/docs/config.rst index b1f05b7f..d70b0e88 100644 --- a/docs/config.rst +++ b/docs/config.rst @@ -95,7 +95,7 @@ ytdl_options overrides """"""""" -.. autoclass:: ytdl_sub.config.preset_options.Overrides() +.. autoclass:: ytdl_sub.config.overrides.Overrides() .. _parent preset: @@ -423,7 +423,7 @@ Traditional subscriptions that can override presets will still work when using ` Source Variables ---------------- -.. autoclass:: ytdl_sub.entries.variables.entry_variables.EntryVariables +.. autoclass:: ytdl_sub.entries.script.variable_definitions.VariableDefinitions() :members: :inherited-members: :undoc-members: diff --git a/pyproject.toml b/pyproject.toml index f04e9334..d78c5633 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,6 +5,7 @@ force_single_line = true [tool.black] line_length = 100 +target-version = ["py310"] [tool.pylint.MASTER] disable = [ @@ -43,3 +44,8 @@ ignore = [ include = [ "src/*" ] + +[tool.coverage.report] +exclude_also = [ + "raise UNREACHABLE.*", +] \ No newline at end of file diff --git a/setup.cfg b/setup.cfg index 87546248..db9e581d 100644 --- a/setup.cfg +++ b/setup.cfg @@ -49,7 +49,6 @@ lint = black==22.3.0 isort==5.10.1 pylint==2.13.5 - pydocstyle[toml]==6.1.1 docs = sphinx==4.5.0 sphinx-rtd-theme==1.0.0 diff --git a/src/ytdl_sub/cli/entrypoint.py b/src/ytdl_sub/cli/entrypoint.py index 9332df66..bc7fc04e 100644 --- a/src/ytdl_sub/cli/entrypoint.py +++ b/src/ytdl_sub/cli/entrypoint.py @@ -221,6 +221,7 @@ def main() -> List[Subscription]: "full backup before usage. You have been warned!", ) + logger.info("Validating subscriptions...") subscriptions = _download_subscriptions_from_yaml_files( config=config, subscription_paths=args.subscription_paths, @@ -230,6 +231,7 @@ def main() -> List[Subscription]: # One-off download elif args.subparser == "dl": + logger.info("Validating presets...") subscriptions.append( _download_subscription_from_cli( config=config, dry_run=args.dry_run, extra_args=extra_args diff --git a/src/ytdl_sub/config/config_file.py b/src/ytdl_sub/config/config_file.py index 67a906cc..d0c117ef 100644 --- a/src/ytdl_sub/config/config_file.py +++ b/src/ytdl_sub/config/config_file.py @@ -6,8 +6,8 @@ from ytdl_sub.config.config_validator import ConfigValidator from ytdl_sub.config.preset import Preset from ytdl_sub.utils.exceptions import FileNotFoundException from ytdl_sub.utils.ffmpeg import FFMPEG +from ytdl_sub.utils.file_path import FilePathTruncater from ytdl_sub.utils.yaml import load_yaml -from ytdl_sub.validators.file_path_validators import FilePathValidatorMixin class ConfigFile(ConfigValidator): @@ -36,7 +36,7 @@ class ConfigFile(ConfigValidator): ffprobe_path=self.config_options.ffprobe_path, ) - FilePathValidatorMixin.set_max_file_name_bytes( + FilePathTruncater.set_max_file_name_bytes( max_file_name_bytes=self.config_options.file_name_max_bytes ) diff --git a/src/ytdl_sub/config/overrides.py b/src/ytdl_sub/config/overrides.py new file mode 100644 index 00000000..a1ef2f81 --- /dev/null +++ b/src/ytdl_sub/config/overrides.py @@ -0,0 +1,176 @@ +from typing import Any +from typing import Dict +from typing import Optional +from typing import Set + +import mergedeep + +from ytdl_sub.entries.entry import Entry +from ytdl_sub.entries.script.variable_definitions import VARIABLES +from ytdl_sub.entries.variables.override_variables import SUBSCRIPTION_NAME +from ytdl_sub.entries.variables.override_variables import OverrideVariables +from ytdl_sub.script.parser import parse +from ytdl_sub.script.script import Script +from ytdl_sub.utils.exceptions import ValidationException +from ytdl_sub.utils.script import ScriptUtils +from ytdl_sub.utils.scriptable import Scriptable +from ytdl_sub.validators.string_formatter_validators import DictFormatterValidator +from ytdl_sub.validators.string_formatter_validators import StringFormatterValidator + + +class Overrides(DictFormatterValidator, Scriptable): + """ + Optional. This section allows you to define variables that can be used in any string formatter. + For example, if you want your file and thumbnail files to match without copy-pasting a large + format string, you can define something like: + + .. code-block:: yaml + + presets: + my_example_preset: + overrides: + output_directory: "/path/to/media" + custom_file_name: "{upload_date_standardized}.{title_sanitized}" + + # Then use the override variables in the output options + output_options: + output_directory: "{output_directory}" + file_name: "{custom_file_name}.{ext}" + thumbnail_name: "{custom_file_name}.{thumbnail_ext}" + + Override variables can contain explicit values and other variables, including both override + and source variables. + + In addition, any override variable defined will automatically create a ``sanitized`` variable + for use. In the example above, ``output_directory_sanitized`` will exist and perform + sanitization on the value when used. + """ + + @classmethod + def partial_validate(cls, name: str, value: Any) -> None: + dict_formatter = DictFormatterValidator(name=name, value=value) + _ = [parse(format_string) for format_string in dict_formatter.dict_with_format_strings] + + def __init__(self, name, value): + DictFormatterValidator.__init__(self, name, value) + Scriptable.__init__(self) + + for key in self._keys: + self.ensure_variable_name_valid(key) + + self.unresolvable.add(VARIABLES.entry_metadata.variable_name) + + def ensure_added_plugin_variable_valid(self, added_variable: str) -> bool: + """ + Returns False if the variable exists as a non-override. + + Raises + ------ + ValidationException + If the variable is already added as an override variable. + """ + try: + self.ensure_variable_name_valid(added_variable) + except ValidationException: + return False + + if added_variable in self.keys: + raise self._validation_exception( + f"Override variable with name {added_variable} cannot be used since it is" + " added by a plugin." + ) + + return True + + def ensure_variable_name_valid(self, name: str) -> None: + """ + Ensures the variable name does not collide with any entry variables or built-in functions. + """ + if OverrideVariables.is_entry_variable_name(name): + raise self._validation_exception( + f"Override variable with name {name} cannot be used since it is a" + " built-in ytdl-sub entry variable name." + ) + + if OverrideVariables.is_function_name(name): + raise self._validation_exception( + f"Override function definition with name {name} cannot be used since it is" + " a built-in ytdl-sub function name." + ) + + def initial_variables( + self, unresolved_variables: Optional[Dict[str, str]] = None + ) -> Dict[str, str]: + """ + Returns + ------- + Variables and format strings for all Override variables + additional variables (Optional) + """ + initial_variables: Dict[str, str] = {} + mergedeep.merge( + initial_variables, + self.dict_with_format_strings, + unresolved_variables if unresolved_variables else {}, + {SUBSCRIPTION_NAME: self.subscription_name}, + ) + return ScriptUtils.add_sanitized_variables(initial_variables) + + def initialize_script(self, unresolved_variables: Set[str]) -> "Overrides": + """ + Initialize the override script with override variables + any unresolved variables + """ + self.script.add( + self.initial_variables( + unresolved_variables={ + var_name: f"{{%throw('Plugin variable {var_name} has not been created yet')}}" + for var_name in unresolved_variables + } + ) + ) + self.unresolvable.update(unresolved_variables) + self.update_script() + return self + + @property + def subscription_name(self) -> str: + """ + Returns + ------- + Name of the subscription + """ + return self._root_name + + def apply_formatter( + self, + formatter: StringFormatterValidator, + entry: Optional[Entry] = None, + function_overrides: Dict[str, str] = None, + ) -> str: + """ + Parameters + ---------- + formatter + Formatter to apply + entry + Optional. Entry to add source variables to the formatter + function_overrides + Optional. Explicit values to override the overrides themselves and source variables + + Returns + ------- + The format_string after .format has been called + """ + script: Script = self.script + unresolvable: Set[str] = self.unresolvable + if entry: + script = entry.script + unresolvable = entry.unresolvable + + return formatter.post_process( + str( + script.resolve_once( + dict({"tmp_var": formatter.format_string}, **(function_overrides or {})), + unresolvable=unresolvable, + )["tmp_var"] + ) + ) diff --git a/src/ytdl_sub/config/plugin/__init__.py b/src/ytdl_sub/config/plugin/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/ytdl_sub/config/plugin.py b/src/ytdl_sub/config/plugin/plugin.py similarity index 77% rename from src/ytdl_sub/config/plugin.py rename to src/ytdl_sub/config/plugin/plugin.py index 9ddc083c..24d616f6 100644 --- a/src/ytdl_sub/config/plugin.py +++ b/src/ytdl_sub/config/plugin/plugin.py @@ -7,44 +7,13 @@ from typing import Optional from typing import Tuple from typing import Type -from ytdl_sub.config.preset_options import Overrides -from ytdl_sub.config.preset_options import TOptionsValidator +from ytdl_sub.config.overrides import Overrides +from ytdl_sub.config.validators.options import TOptionsValidator from ytdl_sub.entries.entry import Entry from ytdl_sub.utils.file_handler import FileMetadata from ytdl_sub.ytdl_additions.enhanced_download_archive import DownloadArchiver from ytdl_sub.ytdl_additions.enhanced_download_archive import EnhancedDownloadArchive - -class PluginPriority: - """ - Defines priority for plugins, 0 is highest priority - """ - - # If modify_entry priority is >= to this value, run after split - MODIFY_ENTRY_AFTER_SPLIT = 10 - - # if post_process is >= to this value, run after file_convert - POST_PROCESS_AFTER_FILE_CONVERT = 10 - - MODIFY_ENTRY_FIRST = 0 - - def __init__( - self, modify_entry_metadata: int = 5, modify_entry: int = 5, post_process: int = 5 - ): - self.modify_entry_metadata = modify_entry_metadata - self.modify_entry = modify_entry - self.post_process = post_process - - @property - def modify_entry_after_split(self) -> bool: - """ - Returns - ------- - True if the plugin should modify an entry after a potential split. False otherwise. - """ - return self.modify_entry >= PluginPriority.MODIFY_ENTRY_AFTER_SPLIT - - # pylint: disable=no-self-use,unused-argument @@ -53,7 +22,6 @@ class BasePlugin(DownloadArchiver, Generic[TOptionsValidator], ABC): Shared code amongst all SourcePlugins (downloaders) and Plugins (post-download modification) """ - priority: PluginPriority = PluginPriority() plugin_options_type: Type[TOptionsValidator] def __init__( diff --git a/src/ytdl_sub/config/plugin/plugin_mapping.py b/src/ytdl_sub/config/plugin/plugin_mapping.py new file mode 100644 index 00000000..2918c746 --- /dev/null +++ b/src/ytdl_sub/config/plugin/plugin_mapping.py @@ -0,0 +1,200 @@ +from typing import Dict +from typing import List +from typing import Optional +from typing import Tuple +from typing import Type + +from ytdl_sub.config.plugin.plugin import Plugin +from ytdl_sub.config.plugin.plugin import SplitPlugin +from ytdl_sub.config.plugin.plugin_operation import PluginOperation +from ytdl_sub.config.validators.options import OptionsValidator +from ytdl_sub.downloaders.url.downloader import UrlDownloaderCollectionVariablePlugin +from ytdl_sub.downloaders.url.downloader import UrlDownloaderThumbnailPlugin +from ytdl_sub.plugins.audio_extract import AudioExtractPlugin +from ytdl_sub.plugins.chapters import ChaptersPlugin +from ytdl_sub.plugins.date_range import DateRangePlugin +from ytdl_sub.plugins.embed_thumbnail import EmbedThumbnailPlugin +from ytdl_sub.plugins.file_convert import FileConvertPlugin +from ytdl_sub.plugins.format import FormatPlugin +from ytdl_sub.plugins.internal.view import ViewPlugin +from ytdl_sub.plugins.match_filters import MatchFiltersPlugin +from ytdl_sub.plugins.music_tags import MusicTagsPlugin +from ytdl_sub.plugins.nfo_tags import NfoTagsPlugin +from ytdl_sub.plugins.output_directory_nfo_tags import OutputDirectoryNfoTagsPlugin +from ytdl_sub.plugins.regex import RegexPlugin +from ytdl_sub.plugins.split_by_chapters import SplitByChaptersPlugin +from ytdl_sub.plugins.subtitles import SubtitlesPlugin +from ytdl_sub.plugins.throttle_protection import ThrottleProtectionPlugin +from ytdl_sub.plugins.video_tags import VideoTagsPlugin + + +class PluginMapping: + """ + Maps plugins defined in the preset to its respective plugin class + """ + + _MAPPING: Dict[str, Type[Plugin]] = { + "_view": ViewPlugin, + "audio_extract": AudioExtractPlugin, + "date_range": DateRangePlugin, + "embed_thumbnail": EmbedThumbnailPlugin, + "file_convert": FileConvertPlugin, + "format": FormatPlugin, + "match_filters": MatchFiltersPlugin, + "music_tags": MusicTagsPlugin, + "video_tags": VideoTagsPlugin, + "nfo_tags": NfoTagsPlugin, + "output_directory_nfo_tags": OutputDirectoryNfoTagsPlugin, + "regex": RegexPlugin, + "subtitles": SubtitlesPlugin, + "chapters": ChaptersPlugin, + "split_by_chapters": SplitByChaptersPlugin, + "throttle_protection": ThrottleProtectionPlugin, + } + + # All other plugins are added after the defined ordered ones + _ORDER_MODIFY_ENTRY_METADATA: List[Type[Plugin]] = [ + ThrottleProtectionPlugin, + UrlDownloaderCollectionVariablePlugin, + SubtitlesPlugin, + # add all others + ] + + _ORDER_MODIFY_ENTRY: List[Type[Plugin]] = [ + UrlDownloaderThumbnailPlugin, + AudioExtractPlugin, + FileConvertPlugin, + ChaptersPlugin, + SplitByChaptersPlugin, + RegexPlugin, + # add all others + ] + + _ORDER_POST_PROCESS: List[Type[Plugin]] = [ + AudioExtractPlugin, + FileConvertPlugin, + ChaptersPlugin, + SubtitlesPlugin, + MusicTagsPlugin, + VideoTagsPlugin, + NfoTagsPlugin, + EmbedThumbnailPlugin, + ] + + @classmethod + def _order_by( + cls, plugin_types: List[Type[Plugin]], operation: PluginOperation + ) -> List[Type[Plugin]]: + if operation == PluginOperation.MODIFY_ENTRY_METADATA: + ordering = cls._ORDER_MODIFY_ENTRY_METADATA + elif operation == PluginOperation.MODIFY_ENTRY: + ordering = cls._ORDER_MODIFY_ENTRY + elif operation == PluginOperation.POST_PROCESS: + ordering = cls._ORDER_POST_PROCESS + else: + raise ValueError("PluginOperation does not support ordering") + + ordered_plugin_operations: List[Type[Plugin]] = [] + for pl_type in reversed(ordering): + for plugin_type in plugin_types: + if plugin_type == pl_type: + ordered_plugin_operations.insert(0, plugin_type) + else: + ordered_plugin_operations.append(plugin_type) + + return ordered_plugin_operations + + @classmethod + def order_options_by( + cls, zipped: List[Tuple[Type[Plugin], OptionsValidator]], operation: PluginOperation + ) -> List[OptionsValidator]: + """ + Returns + ------- + Ordered plugin options with respect to the PluginOperation. + """ + ordered_types: List[Type[Plugin]] = cls._order_by( + plugin_types=[val[0] for val in zipped], operation=operation + ) + + ordered_options: List[OptionsValidator] = [] + for plugin_type, plugin_options in zipped: + sorted_idx = ordered_types.index(plugin_type) + ordered_options.insert(sorted_idx, plugin_options) + + return ordered_options + + @classmethod + def _is_modified_after_split(cls, plugin: Plugin) -> bool: + if type(plugin) not in cls._ORDER_MODIFY_ENTRY: + return True + return cls._ORDER_MODIFY_ENTRY.index(type(plugin)) > cls._ORDER_MODIFY_ENTRY.index( + SplitByChaptersPlugin + ) + + @classmethod + def order_plugins_by( + cls, plugins: List[Plugin], operation: PluginOperation, before_split: Optional[bool] = None + ) -> List[Plugin]: + """ + Returns + ------- + Ordered plugins with respect to the PluginOperation. Optionally only return plugins + before/after a split plugin. + """ + ordered_types: List[Type[Plugin]] = cls._order_by( + plugin_types=[type(plugin) for plugin in plugins], operation=operation + ) + + ordered_plugins: List[Plugin] = [] + for plugin in plugins: + sorted_idx = ordered_types.index(type(plugin)) + ordered_plugins.insert(sorted_idx, plugin) + + if before_split is None: + return ordered_plugins + + # Remove the split plugin if differentiating + ordered_plugins = [ + plugin for plugin in ordered_plugins if not isinstance(plugin, SplitPlugin) + ] + if before_split: + return [ + plugin for plugin in ordered_plugins if not cls._is_modified_after_split(plugin) + ] + + # before_split is False + return [plugin for plugin in ordered_plugins if cls._is_modified_after_split(plugin)] + + @classmethod + def plugins(cls) -> List[str]: + """ + Returns + ------- + Available download sources + """ + return sorted(list(cls._MAPPING.keys())) + + @classmethod + def get(cls, plugin: str) -> Type[Plugin]: + """ + Parameters + ---------- + plugin + Name of the plugin + + Returns + ------- + The plugin class + + Raises + ------ + ValueError + Raised if the plugin does not exist + """ + if plugin not in cls.plugins(): + raise ValueError( + f"Tried to use plugin '{plugin}' that does not exist. Available plugins: " + f"{', '.join(cls.plugins())}" + ) + return cls._MAPPING[plugin] diff --git a/src/ytdl_sub/config/plugin/plugin_operation.py b/src/ytdl_sub/config/plugin/plugin_operation.py new file mode 100644 index 00000000..d3323aca --- /dev/null +++ b/src/ytdl_sub/config/plugin/plugin_operation.py @@ -0,0 +1,9 @@ +from enum import Enum + + +class PluginOperation(Enum): + ANY = -2 + DOWNLOADER = -1 + MODIFY_ENTRY_METADATA = 0 + MODIFY_ENTRY = 1 + POST_PROCESS = 2 diff --git a/src/ytdl_sub/config/plugin/preset_plugins.py b/src/ytdl_sub/config/plugin/preset_plugins.py new file mode 100644 index 00000000..2a86af6a --- /dev/null +++ b/src/ytdl_sub/config/plugin/preset_plugins.py @@ -0,0 +1,46 @@ +from typing import List +from typing import Optional +from typing import Tuple +from typing import Type + +from ytdl_sub.config.plugin.plugin import Plugin +from ytdl_sub.config.validators.options import OptionsValidator +from ytdl_sub.config.validators.options import TOptionsValidator + + +class PresetPlugins: + def __init__(self): + self.plugin_types: List[Type[Plugin]] = [] + self.plugin_options: List[OptionsValidator] = [] + + def add(self, plugin_type: Type[Plugin], plugin_options: OptionsValidator) -> "PresetPlugins": + """ + Add a pair of plugin type and options to the list + """ + self.plugin_types.append(plugin_type) + self.plugin_options.append(plugin_options) + return self + + def zipped(self) -> List[Tuple[Type[Plugin], OptionsValidator]]: + """ + Returns + ------- + Plugin and PluginOptions zipped + """ + return list(zip(self.plugin_types, self.plugin_options)) + + def get(self, plugin_type: Type[TOptionsValidator]) -> Optional[TOptionsValidator]: + """ + Parameters + ---------- + plugin_type + Fetch the plugin options for this type + + Returns + ------- + Options of this plugin if they exit. Otherwise, return None. + """ + plugin_option_types = [type(plugin_options) for plugin_options in self.plugin_options] + if plugin_type in plugin_option_types: + return self.plugin_options[plugin_option_types.index(plugin_type)] + return None diff --git a/src/ytdl_sub/config/plugin_mapping.py b/src/ytdl_sub/config/plugin_mapping.py deleted file mode 100644 index 5a446a26..00000000 --- a/src/ytdl_sub/config/plugin_mapping.py +++ /dev/null @@ -1,79 +0,0 @@ -from typing import Dict -from typing import List -from typing import Type - -from ytdl_sub.config.plugin import Plugin -from ytdl_sub.plugins.audio_extract import AudioExtractPlugin -from ytdl_sub.plugins.chapters import ChaptersPlugin -from ytdl_sub.plugins.date_range import DateRangePlugin -from ytdl_sub.plugins.embed_thumbnail import EmbedThumbnailPlugin -from ytdl_sub.plugins.file_convert import FileConvertPlugin -from ytdl_sub.plugins.format import FormatPlugin -from ytdl_sub.plugins.internal.view import ViewPlugin -from ytdl_sub.plugins.match_filters import MatchFiltersPlugin -from ytdl_sub.plugins.music_tags import MusicTagsPlugin -from ytdl_sub.plugins.nfo_tags import NfoTagsPlugin -from ytdl_sub.plugins.output_directory_nfo_tags import OutputDirectoryNfoTagsPlugin -from ytdl_sub.plugins.regex import RegexPlugin -from ytdl_sub.plugins.split_by_chapters import SplitByChaptersPlugin -from ytdl_sub.plugins.subtitles import SubtitlesPlugin -from ytdl_sub.plugins.throttle_protection import ThrottleProtectionPlugin -from ytdl_sub.plugins.video_tags import VideoTagsPlugin - - -class PluginMapping: - """ - Maps plugins defined in the preset to its respective plugin class - """ - - _MAPPING: Dict[str, Type[Plugin]] = { - "_view": ViewPlugin, - "audio_extract": AudioExtractPlugin, - "date_range": DateRangePlugin, - "embed_thumbnail": EmbedThumbnailPlugin, - "file_convert": FileConvertPlugin, - "format": FormatPlugin, - "match_filters": MatchFiltersPlugin, - "music_tags": MusicTagsPlugin, - "video_tags": VideoTagsPlugin, - "nfo_tags": NfoTagsPlugin, - "output_directory_nfo_tags": OutputDirectoryNfoTagsPlugin, - "regex": RegexPlugin, - "subtitles": SubtitlesPlugin, - "chapters": ChaptersPlugin, - "split_by_chapters": SplitByChaptersPlugin, - "throttle_protection": ThrottleProtectionPlugin, - } - - @classmethod - def plugins(cls) -> List[str]: - """ - Returns - ------- - Available download sources - """ - return sorted(list(cls._MAPPING.keys())) - - @classmethod - def get(cls, plugin: str) -> Type[Plugin]: - """ - Parameters - ---------- - plugin - Name of the plugin - - Returns - ------- - The plugin class - - Raises - ------ - ValueError - Raised if the plugin does not exist - """ - if plugin not in cls.plugins(): - raise ValueError( - f"Tried to use plugin '{plugin}' that does not exist. Available plugins: " - f"{', '.join(cls.plugins())}" - ) - return cls._MAPPING[plugin] diff --git a/src/ytdl_sub/config/preset.py b/src/ytdl_sub/config/preset.py index e88e4fc0..2e5d9fb6 100644 --- a/src/ytdl_sub/config/preset.py +++ b/src/ytdl_sub/config/preset.py @@ -1,39 +1,25 @@ import copy from typing import Any from typing import Dict -from typing import Iterable from typing import List -from typing import Optional -from typing import Tuple -from typing import Type -from typing import Union from mergedeep import mergedeep from ytdl_sub.config.config_validator import ConfigValidator -from ytdl_sub.config.plugin import Plugin -from ytdl_sub.config.plugin_mapping import PluginMapping -from ytdl_sub.config.preset_options import OptionsValidator +from ytdl_sub.config.overrides import Overrides +from ytdl_sub.config.plugin.plugin_mapping import PluginMapping +from ytdl_sub.config.plugin.preset_plugins import PresetPlugins from ytdl_sub.config.preset_options import OutputOptions -from ytdl_sub.config.preset_options import Overrides -from ytdl_sub.config.preset_options import TOptionsValidator from ytdl_sub.config.preset_options import YTDLOptions +from ytdl_sub.config.validators.variable_validation import VariableValidation from ytdl_sub.downloaders.url.validators import MultiUrlValidator -from ytdl_sub.entries.entry import Entry from ytdl_sub.prebuilt_presets import PREBUILT_PRESET_NAMES from ytdl_sub.prebuilt_presets import PUBLISHED_PRESET_NAMES from ytdl_sub.utils.exceptions import ValidationException from ytdl_sub.utils.logger import Logger from ytdl_sub.utils.yaml import dump_yaml from ytdl_sub.validators.strict_dict_validator import StrictDictValidator -from ytdl_sub.validators.string_formatter_validators import DictFormatterValidator -from ytdl_sub.validators.string_formatter_validators import OverridesDictFormatterValidator -from ytdl_sub.validators.string_formatter_validators import OverridesStringFormatterValidator -from ytdl_sub.validators.string_formatter_validators import StringFormatterValidator -from ytdl_sub.validators.validators import DictValidator -from ytdl_sub.validators.validators import ListValidator from ytdl_sub.validators.validators import StringListValidator -from ytdl_sub.validators.validators import Validator from ytdl_sub.validators.validators import validation_exception PRESET_KEYS = { @@ -61,44 +47,6 @@ def _parent_preset_error_message( ) -class PresetPlugins: - def __init__(self): - self.plugin_types: List[Type[Plugin]] = [] - self.plugin_options: List[OptionsValidator] = [] - - def add(self, plugin_type: Type[Plugin], plugin_options: OptionsValidator) -> "PresetPlugins": - """ - Add a pair of plugin type and options to the list - """ - self.plugin_types.append(plugin_type) - self.plugin_options.append(plugin_options) - return self - - def zipped(self) -> Iterable[Tuple[Type[Plugin], OptionsValidator]]: - """ - Returns - ------- - Plugin and PluginOptions zipped - """ - return zip(self.plugin_types, self.plugin_options) - - def get(self, plugin_type: Type[TOptionsValidator]) -> Optional[TOptionsValidator]: - """ - Parameters - ---------- - plugin_type - Fetch the plugin options for this type - - Returns - ------- - Options of this plugin if they exit. Otherwise, return None. - """ - plugin_option_types = [type(plugin_options) for plugin_options in self.plugin_options] - if plugin_type in plugin_option_types: - return self.plugin_options[plugin_option_types.index(plugin_type)] - return None - - class _PresetShell(StrictDictValidator): # Have all present keys optional since parent presets could not have all the # required keys. They will get validated in the init after the mergedeep of dicts @@ -154,11 +102,7 @@ class Preset(_PresetShell): validator=PluginMapping.get(plugin_name).plugin_options_type, ) - @property - def _source_variables(self) -> List[str]: - return Entry.source_variables() - - def __validate_and_get_plugins(self) -> PresetPlugins: + def _validate_and_get_plugins(self) -> PresetPlugins: preset_plugins = PresetPlugins() for key in self._keys: @@ -172,88 +116,6 @@ class Preset(_PresetShell): return preset_plugins - def __validate_added_variables(self): - source_variables = copy.deepcopy(self._source_variables) - - # Validate added download option variables here since plugins could subsequently use them - self.downloader_options.validate_with_variables( - source_variables=source_variables, - override_variables=self.overrides.dict_with_format_strings, - ) - source_variables.extend(self.downloader_options.added_source_variables()) - - for _, plugin_options in sorted( - self.plugins.zipped(), key=lambda pl: pl[0].priority.modify_entry - ): - # Validate current plugin using source + added plugin variables - plugin_options.validate_with_variables( - source_variables=source_variables, - override_variables=self.overrides.dict_with_format_strings, - ) - - # Extend existing source variables with ones created from this plugin - source_variables.extend(plugin_options.added_source_variables()) - - def __validate_override_string_formatter_validator( - self, - formatter_validator: Union[StringFormatterValidator, OverridesStringFormatterValidator], - ): - # Set the formatter variables to be the overrides - variable_dict = copy.deepcopy(self.overrides.dict_with_format_strings) - - # If the formatter supports source variables, set the formatter variables to include - # both source and override variables - if not isinstance(formatter_validator, OverridesStringFormatterValidator): - source_variables = { - source_var: "dummy_string" - for source_var in self._source_variables - + self.downloader_options.added_source_variables() - } - variable_dict = dict(source_variables, **variable_dict) - - # For all plugins, add in any extra added source variables - for plugin_options in self.plugins.plugin_options: - added_plugin_variables = { - source_var: "dummy_string" - for source_var in plugin_options.added_source_variables() - } - # sanity check plugin variables do not override source variables - expected_len = len(variable_dict) + len(added_plugin_variables) - variable_dict = dict(variable_dict, **added_plugin_variables) - - assert ( - len(variable_dict) == expected_len - ), "plugin variables overwrote source variables" - - _ = formatter_validator.apply_formatter(variable_dict=variable_dict) - - def __recursive_preset_validate( - self, - validator: Optional[Validator] = None, - ) -> None: - """ - Ensure all OverridesStringFormatterValidator's only contain variables from the overrides - and resolve. - """ - if validator is None: - validator = self - - if isinstance(validator, DictValidator): - # pylint: disable=protected-access - # Usage of protected variables in other validators is fine. The reason to keep - # them protected is for readability when using them in subscriptions. - for validator_value in validator._validator_dict.values(): - self.__recursive_preset_validate(validator_value) - # pylint: enable=protected-access - elif isinstance(validator, ListValidator): - for list_value in validator.list: - self.__recursive_preset_validate(list_value) - elif isinstance(validator, (StringFormatterValidator, OverridesStringFormatterValidator)): - self.__validate_override_string_formatter_validator(validator) - elif isinstance(validator, (DictFormatterValidator, OverridesDictFormatterValidator)): - for validator_value in validator.dict.values(): - self.__validate_override_string_formatter_validator(validator_value) - def _get_presets_to_merge( self, parent_presets: str | List[str], seen_presets: List[str], config: ConfigValidator ) -> List[Dict]: @@ -291,7 +153,7 @@ class Preset(_PresetShell): return presets_to_merge - def __merge_parent_preset_dicts_if_present(self, config: ConfigValidator): + def _merge_parent_preset_dicts_if_present(self, config: ConfigValidator): parent_preset_validator = self._validate_key_if_present( key="preset", validator=StringListValidator ) @@ -314,7 +176,7 @@ class Preset(_PresetShell): super().__init__(name=name, value=value) # Perform the merge of parent presets before validating any keys - self.__merge_parent_preset_dicts_if_present(config=config) + self._merge_parent_preset_dicts_if_present(config=config) self.downloader_options: MultiUrlValidator = self._validate_key( key="download", validator=MultiUrlValidator @@ -329,13 +191,14 @@ class Preset(_PresetShell): key="ytdl_options", validator=YTDLOptions, default={} ) + self.plugins: PresetPlugins = self._validate_and_get_plugins() self.overrides = self._validate_key(key="overrides", validator=Overrides, default={}) - self.plugins: PresetPlugins = self.__validate_and_get_plugins() - self.__validate_added_variables() - # After all options are initialized, perform a recursive post-validate that requires - # values from multiple validators - self.__recursive_preset_validate() + VariableValidation( + downloader_options=self.downloader_options, + output_options=self.output_options, + plugins=self.plugins, + ).initialize_overrides(overrides=self.overrides).ensure_proper_usage() @property def name(self) -> str: diff --git a/src/ytdl_sub/config/preset_options.py b/src/ytdl_sub/config/preset_options.py index b4738a8c..d6b2bc2e 100644 --- a/src/ytdl_sub/config/preset_options.py +++ b/src/ytdl_sub/config/preset_options.py @@ -1,88 +1,16 @@ -from abc import ABC from typing import Any -from typing import Dict -from typing import List from typing import Optional -from typing import TypeVar - -from yt_dlp.utils import sanitize_filename from ytdl_sub.config.defaults import DEFAULT_DOWNLOAD_ARCHIVE_NAME -from ytdl_sub.entries.entry import Entry -from ytdl_sub.entries.variables.override_variables import SUBSCRIPTION_NAME -from ytdl_sub.utils.exceptions import ValidationException from ytdl_sub.validators.file_path_validators import OverridesStringFormatterFilePathValidator from ytdl_sub.validators.file_path_validators import StringFormatterFileNameValidator from ytdl_sub.validators.strict_dict_validator import StrictDictValidator from ytdl_sub.validators.string_datetime import StringDatetimeValidator -from ytdl_sub.validators.string_formatter_validators import DictFormatterValidator from ytdl_sub.validators.string_formatter_validators import OverridesIntegerFormatterValidator from ytdl_sub.validators.string_formatter_validators import OverridesStringFormatterValidator from ytdl_sub.validators.string_formatter_validators import StringFormatterValidator from ytdl_sub.validators.validators import BoolValidator from ytdl_sub.validators.validators import LiteralDictValidator -from ytdl_sub.validators.validators import Validator - - -# pylint: disable=no-self-use -# pylint: disable=unused-argument -class OptionsValidator(Validator, ABC): - """ - Abstract class that validates options for preset sections (plugins, downloaders) - """ - - def validation_exception( - self, - error_message: str | Exception, - ) -> ValidationException: - """ - Parameters - ---------- - error_message - Error message to include in the validation exception - - Returns - ------- - Validation exception that points to the location in the config. To be used to throw good - validation exceptions at runtime from code outside this class. - """ - return self._validation_exception(error_message=error_message) - - def added_source_variables(self) -> List[str]: - """ - If the plugin adds source variables, list them here. - - Returns - ------- - List of added source variables this plugin creates - """ - return [] - - def validate_with_variables( - self, source_variables: List[str], override_variables: Dict[str, str] - ) -> None: - """ - Optional validation after init with the session's source and override variables. - - Parameters - ---------- - source_variables - Available source variables when running the plugin - override_variables - Available override variables when running the plugin - """ - return None - - -TOptionsValidator = TypeVar("TOptionsValidator", bound=OptionsValidator) - - -class OptionsDictValidator(StrictDictValidator, OptionsValidator, ABC): - pass - - -# pylint: enable=no-self-use -# pylint: enable=unused-argument class YTDLOptions(LiteralDictValidator): @@ -124,101 +52,6 @@ class YTDLOptions(LiteralDictValidator): # Disable for proper docstring formatting # pylint: disable=line-too-long -class Overrides(DictFormatterValidator): - """ - Optional. This section allows you to define variables that can be used in any string formatter. - For example, if you want your file and thumbnail files to match without copy-pasting a large - format string, you can define something like: - - .. code-block:: yaml - - presets: - my_example_preset: - overrides: - output_directory: "/path/to/media" - custom_file_name: "{upload_year}.{upload_month_padded}.{upload_day_padded}.{title_sanitized}" - - # Then use the override variables in the output options - output_options: - output_directory: "{output_directory}" - file_name: "{custom_file_name}.{ext}" - thumbnail_name: "{custom_file_name}.{thumbnail_ext}" - - Override variables can contain explicit values and other variables, including both override - and source variables. - - In addition, any override variable defined will automatically create a ``sanitized`` variable - for use. In the example above, ``output_directory_sanitized`` will exist and perform - sanitization on the value when used. - """ - - # pylint: enable=line-too-long - - def _add_override_variable(self, key_name: str, format_string: str, sanitize: bool = False): - if sanitize: - key_name = f"{key_name}_sanitized" - format_string = sanitize_filename(format_string) - - self._value[key_name] = StringFormatterValidator( - name="__should_never_fail__", - value=format_string, - ) - - def __init__(self, name, value): - super().__init__(name, value) - - # Add sanitized overrides - for key in self._keys: - self._add_override_variable( - key_name=key, - format_string=self._value[key].format_string, - sanitize=True, - ) - - if SUBSCRIPTION_NAME not in self._value: - for sanitized in [True, False]: - self._add_override_variable( - key_name=SUBSCRIPTION_NAME, - format_string=self.subscription_name, - sanitize=sanitized, - ) - - @property - def subscription_name(self) -> str: - """ - Returns - ------- - Name of the subscription - """ - return self._root_name - - def apply_formatter( - self, - formatter: StringFormatterValidator, - entry: Optional[Entry] = None, - function_overrides: Dict[str, str] = None, - ) -> str: - """ - Parameters - ---------- - formatter - Formatter to apply - entry - Optional. Entry to add source variables to the formatter - function_overrides - Optional. Explicit values to override the overrides themselves and source variables - - Returns - ------- - The format_string after .format has been called - """ - variable_dict = self.dict_with_format_strings - if entry: - variable_dict = dict(entry.to_dict(), **variable_dict) - if function_overrides: - variable_dict = dict(variable_dict, **function_overrides) - - return formatter.apply_formatter(variable_dict) class OutputOptions(StrictDictValidator): diff --git a/src/ytdl_sub/config/validators/__init__.py b/src/ytdl_sub/config/validators/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/ytdl_sub/config/validators/options.py b/src/ytdl_sub/config/validators/options.py new file mode 100644 index 00000000..c35defa2 --- /dev/null +++ b/src/ytdl_sub/config/validators/options.py @@ -0,0 +1,59 @@ +from abc import ABC +from typing import Dict +from typing import Set +from typing import TypeVar + +from ytdl_sub.config.plugin.plugin_operation import PluginOperation +from ytdl_sub.utils.exceptions import ValidationException +from ytdl_sub.validators.strict_dict_validator import StrictDictValidator +from ytdl_sub.validators.validators import Validator + +# pylint: disable=no-self-use +# pylint: disable=unused-argument + + +class OptionsValidator(Validator, ABC): + """ + Abstract class that validates options for preset sections (plugins, downloaders) + """ + + def validation_exception( + self, + error_message: str | Exception, + ) -> ValidationException: + """ + Parameters + ---------- + error_message + Error message to include in the validation exception + + Returns + ------- + Validation exception that points to the location in the config. To be used to throw good + validation exceptions at runtime from code outside this class. + """ + return self._validation_exception(error_message=error_message) + + def modified_variables(self) -> Dict[PluginOperation, Set[str]]: + """ + If the plugin modifies existing variables, define them here + """ + return {} + + def added_variables( + self, + resolved_variables: Set[str], + unresolved_variables: Set[str], + plugin_op: PluginOperation, + ) -> Dict[PluginOperation, Set[str]]: + """ + If the plugin adds source variables, list them here. + """ + return {} + + +TOptionsValidator = TypeVar("TOptionsValidator", bound=OptionsValidator) + + +class OptionsDictValidator(StrictDictValidator, OptionsValidator, ABC): + pass diff --git a/src/ytdl_sub/config/validators/variable_validation.py b/src/ytdl_sub/config/validators/variable_validation.py new file mode 100644 index 00000000..a0aa180d --- /dev/null +++ b/src/ytdl_sub/config/validators/variable_validation.py @@ -0,0 +1,178 @@ +import copy +from typing import Dict +from typing import Iterable +from typing import List +from typing import Optional +from typing import Set +from typing import Tuple + +from ytdl_sub.config.overrides import Overrides +from ytdl_sub.config.plugin.plugin_mapping import PluginMapping +from ytdl_sub.config.plugin.plugin_operation import PluginOperation +from ytdl_sub.config.plugin.preset_plugins import PresetPlugins +from ytdl_sub.config.preset_options import OutputOptions +from ytdl_sub.config.validators.options import OptionsValidator +from ytdl_sub.downloaders.url.validators import MultiUrlValidator +from ytdl_sub.entries.script.variable_scripts import VARIABLE_SCRIPTS +from ytdl_sub.script.script import Script +from ytdl_sub.validators.string_formatter_validators import validate_formatters + + +def _add_dummy_variables(variables: Iterable[str]) -> Dict[str, str]: + dummy_variables: Dict[str, str] = {} + for var in variables: + dummy_variables[var] = "" + dummy_variables[f"{var}_sanitized"] = "" + + return dummy_variables + + +def _get_added_and_modified_variables( + plugins: PresetPlugins, downloader_options: MultiUrlValidator +) -> Iterable[Tuple[OptionsValidator, Set[str], Set[str]]]: + """ + Iterates and returns the plugin options, added variables, modified variables + """ + options: List[OptionsValidator] = plugins.plugin_options + options.append(downloader_options) + + for plugin_options in options: + added_variables: Set[str] = set() + modified_variables: Set[str] = set() + + for plugin_added_variables in plugin_options.added_variables( + resolved_variables=set(), + unresolved_variables=set(), + plugin_op=PluginOperation.ANY, + ).values(): + added_variables |= set(plugin_added_variables) + + for plugin_modified_variables in plugin_options.modified_variables().values(): + modified_variables = plugin_modified_variables + + yield plugin_options, added_variables, modified_variables + + +def _override_variables(overrides: Overrides) -> Set[str]: + return set(list(overrides.initial_variables().keys())) + + +def _entry_variables() -> Set[str]: + return set(list(VARIABLE_SCRIPTS.keys())) + + +class VariableValidation: + def __init__( + self, + downloader_options: MultiUrlValidator, + output_options: OutputOptions, + plugins: PresetPlugins, + ): + self.downloader_options = downloader_options + self.output_options = output_options + self.plugins = plugins + + self.script: Optional[Script] = None + self.resolved_variables: Set[str] = set() + self.unresolved_variables: Set[str] = set() + + def initialize_overrides(self, overrides: Overrides) -> "VariableValidation": + """ + Do some gymnastics to initialize the Overrides script. + """ + entry_variables = _entry_variables() + override_variables = _override_variables(overrides) + + # Set resolved variables as all entry + override variables + # at this point to generate every possible added/modified variable + self.resolved_variables = entry_variables | override_variables + + for ( + plugin_options, + added_variables, + modified_variables, + ) in _get_added_and_modified_variables( + plugins=self.plugins, + downloader_options=self.downloader_options, + ): + + for added_variable in added_variables: + if not overrides.ensure_added_plugin_variable_valid(added_variable=added_variable): + # pylint: disable=protected-access + raise plugin_options._validation_exception( + f"Cannot use the variable name {added_variable} because it exists as a" + " built-in ytdl-sub variable name." + ) + # pylint: enable=protected-access + + # Set unresolved as variables that are added but do not exist as + # entry/override variables since they are created at run-time + self.unresolved_variables |= added_variables | modified_variables + + # Then update resolved variables to reflect that + self.resolved_variables -= self.unresolved_variables + + # Initialize overrides with unresolved variables + modified variables to throw an error. + # For modified variables, this is to prevent a resolve(update=True) to setting any + # dependencies until it has been explicitly added + overrides = overrides.initialize_script(unresolved_variables=self.unresolved_variables) + + # copy the script and mock entry variables + self.script = copy.deepcopy(overrides.script).add(_add_dummy_variables(entry_variables)) + + return self + + def _update_script(self) -> None: + _ = self.script.resolve(unresolvable=self.unresolved_variables, update=True) + + def _add_variables(self, plugin_op: PluginOperation, options: OptionsValidator) -> Set[str]: + added_variables = options.added_variables( + resolved_variables=self.resolved_variables, + unresolved_variables=self.unresolved_variables, + plugin_op=plugin_op, + ).get(plugin_op, set()) + modified_variables = options.modified_variables().get(plugin_op, set()) + + resolved_variables = added_variables | modified_variables + + self.script.add(_add_dummy_variables(resolved_variables)) + self.resolved_variables |= resolved_variables + self.unresolved_variables -= resolved_variables + + return added_variables + + def ensure_proper_usage(self) -> None: + """ + Validate variables resolve as plugins are executed, and return + a mock script which contains actualized added variables from the plugins + """ + self._add_variables(PluginOperation.DOWNLOADER, options=self.downloader_options) + + # Metadata variables to be added + for plugin_options in PluginMapping.order_options_by( + self.plugins.zipped(), PluginOperation.MODIFY_ENTRY_METADATA + ): + self._add_variables(PluginOperation.MODIFY_ENTRY_METADATA, options=plugin_options) + + self._update_script() + for plugin_options in PluginMapping.order_options_by( + self.plugins.zipped(), PluginOperation.MODIFY_ENTRY + ): + added = self._add_variables(PluginOperation.MODIFY_ENTRY, options=plugin_options) + if added: + self._update_script() + + # Validate that any formatter in the plugin options can resolve + validate_formatters( + script=self.script, + unresolved_variables=self.unresolved_variables, + validator=plugin_options, + ) + + validate_formatters( + script=self.script, + unresolved_variables=self.unresolved_variables, + validator=self.output_options, + ) + + assert not self.unresolved_variables diff --git a/src/ytdl_sub/downloaders/info_json/info_json_downloader.py b/src/ytdl_sub/downloaders/info_json/info_json_downloader.py index a7fac294..f1b29aa4 100644 --- a/src/ytdl_sub/downloaders/info_json/info_json_downloader.py +++ b/src/ytdl_sub/downloaders/info_json/info_json_downloader.py @@ -6,17 +6,23 @@ from typing import Iterable from typing import List from typing import Optional -from ytdl_sub.config.preset_options import OptionsDictValidator -from ytdl_sub.config.preset_options import Overrides +from ytdl_sub.config.overrides import Overrides +from ytdl_sub.config.validators.options import OptionsDictValidator from ytdl_sub.downloaders.source_plugin import SourcePlugin from ytdl_sub.downloaders.ytdl_options_builder import YTDLOptionsBuilder from ytdl_sub.entries.entry import Entry +from ytdl_sub.entries.script.variable_definitions import VARIABLES +from ytdl_sub.entries.script.variable_definitions import VariableDefinitions +from ytdl_sub.entries.script.variable_scripts import DOWNLOADER_INJECTED_VARIABLES +from ytdl_sub.entries.script.variable_scripts import VARIABLE_SCRIPTS from ytdl_sub.utils.exceptions import ValidationException from ytdl_sub.utils.file_handler import FileHandler from ytdl_sub.utils.file_handler import get_file_extension from ytdl_sub.ytdl_additions.enhanced_download_archive import DownloadMapping from ytdl_sub.ytdl_additions.enhanced_download_archive import EnhancedDownloadArchive +v: VariableDefinitions = VARIABLES + class InfoJsonDownloaderOptions(OptionsDictValidator): _optional_keys = {"no-op"} @@ -97,14 +103,26 @@ class InfoJsonDownloader(SourcePlugin[InfoJsonDownloaderOptions]): for download_mapping in self._enhanced_download_archive.mapping.entry_mappings.values(): entry = self._get_entry_from_download_mapping(download_mapping) + + # See if prior variables exist. If so, delete them from metadata + # to avoid saving them recursively on multiple updates + prior_variables = entry.maybe_get_prior_variables() + + entry.initialize_script(self.overrides).add( + { + inj: prior_variables.get( + inj.variable_name, + VARIABLE_SCRIPTS[inj.variable_name], + ) + for inj in DOWNLOADER_INJECTED_VARIABLES + } + ) entries.append(entry) - # Remove each entry from the live download archive since it will get re-added - # unless it is filtered - for entry in entries: + for entry in sorted(entries, key=lambda ent: ent.get(v.download_index, int)): + # Remove each entry from the live download archive since it will get re-added + # unless it is filtered self._enhanced_download_archive.mapping.remove_entry(entry.uid) - - for entry in sorted(entries, key=lambda ent: ent.download_index): yield entry # If the original entry file_path is no longer maintained in the new mapping, then diff --git a/src/ytdl_sub/downloaders/source_plugin.py b/src/ytdl_sub/downloaders/source_plugin.py index b11b1940..167f67fd 100644 --- a/src/ytdl_sub/downloaders/source_plugin.py +++ b/src/ytdl_sub/downloaders/source_plugin.py @@ -8,10 +8,10 @@ from typing import Optional from typing import Type from typing import final -from ytdl_sub.config.plugin import BasePlugin -from ytdl_sub.config.plugin import Plugin -from ytdl_sub.config.preset_options import Overrides -from ytdl_sub.config.preset_options import TOptionsValidator +from ytdl_sub.config.overrides import Overrides +from ytdl_sub.config.plugin.plugin import BasePlugin +from ytdl_sub.config.plugin.plugin import Plugin +from ytdl_sub.config.validators.options import TOptionsValidator from ytdl_sub.downloaders.ytdl_options_builder import YTDLOptionsBuilder from ytdl_sub.entries.entry import Entry from ytdl_sub.ytdl_additions.enhanced_download_archive import EnhancedDownloadArchive diff --git a/src/ytdl_sub/downloaders/url/downloader.py b/src/ytdl_sub/downloaders/url/downloader.py index 4fb8f273..a0262963 100644 --- a/src/ytdl_sub/downloaders/url/downloader.py +++ b/src/ytdl_sub/downloaders/url/downloader.py @@ -11,8 +11,7 @@ from typing import Tuple from yt_dlp.utils import RejectedVideoReached -from ytdl_sub.config.plugin import PluginPriority -from ytdl_sub.config.preset_options import Overrides +from ytdl_sub.config.overrides import Overrides from ytdl_sub.downloaders.source_plugin import SourcePlugin from ytdl_sub.downloaders.source_plugin import SourcePluginExtension from ytdl_sub.downloaders.url.validators import MultiUrlValidator @@ -22,14 +21,8 @@ from ytdl_sub.downloaders.ytdl_options_builder import YTDLOptionsBuilder from ytdl_sub.downloaders.ytdlp import YTDLP from ytdl_sub.entries.entry import Entry from ytdl_sub.entries.entry_parent import EntryParent -from ytdl_sub.entries.variables.kwargs import COLLECTION_URL -from ytdl_sub.entries.variables.kwargs import COMMENTS -from ytdl_sub.entries.variables.kwargs import DOWNLOAD_INDEX -from ytdl_sub.entries.variables.kwargs import PLAYLIST_ENTRY -from ytdl_sub.entries.variables.kwargs import REQUESTED_SUBTITLES -from ytdl_sub.entries.variables.kwargs import SOURCE_ENTRY -from ytdl_sub.entries.variables.kwargs import SPONSORBLOCK_CHAPTERS -from ytdl_sub.entries.variables.kwargs import UPLOAD_DATE_INDEX +from ytdl_sub.entries.script.variable_definitions import VARIABLES +from ytdl_sub.entries.script.variable_definitions import VariableDefinitions from ytdl_sub.utils.file_handler import FileHandler from ytdl_sub.utils.logger import Logger from ytdl_sub.utils.thumbnail import ThumbnailTypes @@ -37,6 +30,8 @@ from ytdl_sub.utils.thumbnail import download_and_convert_url_thumbnail from ytdl_sub.utils.thumbnail import try_convert_download_thumbnail from ytdl_sub.ytdl_additions.enhanced_download_archive import EnhancedDownloadArchive +v: VariableDefinitions = VARIABLES + download_logger = Logger.get(name="downloader") @@ -47,8 +42,6 @@ class URLDownloadState: class UrlDownloaderThumbnailPlugin(SourcePluginExtension): - priority = PluginPriority(modify_entry=0) - def __init__( self, options: MultiUrlValidator, @@ -119,22 +112,18 @@ class UrlDownloaderThumbnailPlugin(SourcePluginExtension): directory, run this function. This lets the downloader add any extra files directly to the output directory, for things like YT channel image, banner. """ - if entry.kwargs_contains(PLAYLIST_ENTRY): + if playlist_metadata := entry.get(v.playlist_metadata, dict): self._download_parent_thumbnails( thumbnail_list_info=collection_url.playlist_thumbnails, entry=entry, - parent=EntryParent( - entry.kwargs(PLAYLIST_ENTRY), working_directory=self.working_directory - ), + parent=EntryParent(playlist_metadata, working_directory=self.working_directory), ) - if entry.kwargs_contains(SOURCE_ENTRY): + if source_metadata := entry.get(v.source_metadata, dict): self._download_parent_thumbnails( thumbnail_list_info=collection_url.source_thumbnails, entry=entry, - parent=EntryParent( - entry.kwargs(SOURCE_ENTRY), working_directory=self.working_directory - ), + parent=EntryParent(source_metadata, working_directory=self.working_directory), ) def modify_entry(self, entry: Entry) -> Optional[Entry]: @@ -147,17 +136,15 @@ class UrlDownloaderThumbnailPlugin(SourcePluginExtension): if not self.is_dry_run: try_convert_download_thumbnail(entry=entry) - if entry.kwargs_get(COLLECTION_URL) in self._collection_url_mapping: + if (input_url := entry.get(v.ytdl_sub_input_url, str)) in self._collection_url_mapping: self._download_url_thumbnails( - collection_url=self._collection_url_mapping[entry.kwargs(COLLECTION_URL)], + collection_url=self._collection_url_mapping[input_url], entry=entry, ) return entry class UrlDownloaderCollectionVariablePlugin(SourcePluginExtension): - priority = PluginPriority(modify_entry_metadata=0) - def __init__( self, options: MultiUrlValidator, @@ -181,7 +168,7 @@ class UrlDownloaderCollectionVariablePlugin(SourcePluginExtension): """ # COLLECTION_URL is a recent variable that may not exist for old entries when updating. # Try to use source_webpage_url if it does not exist - entry_collection_url = entry.kwargs_get(COLLECTION_URL, entry.source_webpage_url) + entry_collection_url = entry.get(v.ytdl_sub_input_url, str) # If the collection URL cannot find its mapping, use the last URL collection_url = ( @@ -189,7 +176,7 @@ class UrlDownloaderCollectionVariablePlugin(SourcePluginExtension): or list(self._collection_url_mapping.values())[-1] ) - entry.add_variables(variables_to_add=collection_url.variables.dict_with_format_strings) + entry.add(collection_url.variables.dict_with_format_strings) return entry @@ -356,7 +343,10 @@ class MultiUrlDownloader(SourcePlugin[MultiUrlValidator]): else entry.is_thumbnail_downloaded_via_ytdlp, url=entry.webpage_url, ) - return Entry(download_entry_dict, working_directory=self.working_directory) + return Entry( + download_entry_dict, + working_directory=self.working_directory, + ) def _iterate_child_entries( self, url_validator: UrlValidator, entries: List[Entry] @@ -413,7 +403,9 @@ class MultiUrlDownloader(SourcePlugin[MultiUrlValidator]): working_directory=self.working_directory, ) orphans = EntryParent.from_entry_dicts_with_no_parents( - parents=parents, entry_dicts=entry_dicts, working_directory=self.working_directory + parents=parents, + entry_dicts=entry_dicts, + working_directory=self.working_directory, ) return parents, orphans @@ -459,9 +451,8 @@ class MultiUrlDownloader(SourcePlugin[MultiUrlValidator]): for entry in self._iterate_entries( url_validator=collection_url, parents=parents, orphans=orphan_entries ): - # Add the collection URL to the info_dict to trace where it came from - entry.add_kwargs( - {COLLECTION_URL: self.overrides.apply_formatter(collection_url.url)} + entry.initialize_script(self.overrides).add( + {v.ytdl_sub_input_url: self.overrides.apply_formatter(collection_url.url)} ) yield entry @@ -497,22 +488,12 @@ class MultiUrlDownloader(SourcePlugin[MultiUrlValidator]): return None upload_date_idx = self._enhanced_download_archive.mapping.get_num_entries_with_upload_date( - upload_date_standardized=entry.upload_date_standardized + upload_date_standardized=entry.get(v.upload_date_standardized, str) ) download_idx = self._enhanced_download_archive.num_entries - entry.add_kwargs( - { - # Subtitles are not downloaded in metadata run, only here, so move over - REQUESTED_SUBTITLES: download_entry.kwargs_get(REQUESTED_SUBTITLES), - # Same with sponsorblock chapters - SPONSORBLOCK_CHAPTERS: download_entry.kwargs_get(SPONSORBLOCK_CHAPTERS), - COMMENTS: download_entry.kwargs_get(COMMENTS), - # Tracks number of entries downloaded - DOWNLOAD_INDEX: download_idx, - # Tracks number of entries with the same upload date to make them unique - UPLOAD_DATE_INDEX: upload_date_idx, - } + return entry.add_injected_variables( + download_entry=download_entry, + download_idx=download_idx, + upload_date_idx=upload_date_idx, ) - - return entry diff --git a/src/ytdl_sub/downloaders/url/validators.py b/src/ytdl_sub/downloaders/url/validators.py index 2f799873..bafe846a 100644 --- a/src/ytdl_sub/downloaders/url/validators.py +++ b/src/ytdl_sub/downloaders/url/validators.py @@ -1,10 +1,12 @@ import copy from typing import Any from typing import Dict -from typing import List from typing import Optional +from typing import Set -from ytdl_sub.config.preset_options import OptionsValidator +from ytdl_sub.config.plugin.plugin_operation import PluginOperation +from ytdl_sub.config.validators.options import OptionsValidator +from ytdl_sub.script.parser import parse from ytdl_sub.validators.strict_dict_validator import StrictDictValidator from ytdl_sub.validators.string_formatter_validators import DictFormatterValidator from ytdl_sub.validators.string_formatter_validators import OverridesStringFormatterValidator @@ -243,45 +245,26 @@ class MultiUrlValidator(OptionsValidator): # keep for readthedocs documentation return self._urls.list[0].variables - def added_source_variables(self) -> List[str]: + def added_variables( + self, + resolved_variables: Set[str], + unresolved_variables: Set[str], + plugin_op: PluginOperation, + ) -> Dict[PluginOperation, Set[str]]: """ Returns ------- List of variables added. The first collection url always contains all the variables. """ - return list(self._urls.list[0].variables.keys) + if plugin_op != PluginOperation.ANY: + for url in self._urls.list: + for variable_name, definition in url.variables.dict_with_format_strings.items(): + used_variables = set(var.name for var in parse(definition).variables) + if unresolved := used_variables & unresolved_variables: + raise self._validation_exception( + f"variable {variable_name} cannot use the variables " + f"{', '.join(sorted(list(unresolved)))} because it depends on other" + " variables that are computed later in execution" + ) - def validate_with_variables( - self, source_variables: List[str], override_variables: Dict[str, str] - ) -> None: - """ - Ensures new variables added are not existing variables - """ - for source_var_name in self.added_source_variables(): - if source_var_name in source_variables: - raise self._validation_exception( - f"'{source_var_name}' cannot be used as a variable name because it " - f"is an existing source variable" - ) - - base_variables = dict( - override_variables, **{source_var: "dummy_string" for source_var in source_variables} - ) - - # Apply formatting to each new source variable, ensure it resolves - for collection_url in self.urls.list: - for ( - source_var_name, - source_var_formatter_str, - ) in collection_url.variables.dict_with_format_strings.items(): - _ = StringFormatterValidator( - name=f"{self._name}.{source_var_name}", value=source_var_formatter_str - ).apply_formatter(base_variables) - - # Ensure at least URL is non-empty - has_non_empty_url = False - for url_validator in self.urls.list: - has_non_empty_url |= bool(url_validator.url.apply_formatter(override_variables)) - - if not has_non_empty_url: - raise self._validation_exception("Must contain at least one url that is non-empty") + return {PluginOperation.DOWNLOADER: set(self._urls.list[0].variables.keys)} diff --git a/src/ytdl_sub/downloaders/ytdl_options_builder.py b/src/ytdl_sub/downloaders/ytdl_options_builder.py index b159da42..9f389b51 100644 --- a/src/ytdl_sub/downloaders/ytdl_options_builder.py +++ b/src/ytdl_sub/downloaders/ytdl_options_builder.py @@ -17,7 +17,7 @@ class YTDLOptionsBuilder: self, *ytdl_option_dicts: Optional[Dict], before: bool = False, - strategy: mergedeep.Strategy = mergedeep.Strategy.TYPESAFE_ADDITIVE + strategy: mergedeep.Strategy = mergedeep.Strategy.TYPESAFE_ADDITIVE, ) -> "YTDLOptionsBuilder": """ Parameters diff --git a/src/ytdl_sub/entries/base_entry.py b/src/ytdl_sub/entries/base_entry.py index e5e25151..c4243d63 100644 --- a/src/ytdl_sub/entries/base_entry.py +++ b/src/ytdl_sub/entries/base_entry.py @@ -1,240 +1,22 @@ +# pylint: disable=protected-access from abc import ABC -from datetime import datetime from pathlib import Path from typing import Any from typing import Dict -from typing import List from typing import Optional from typing import Type from typing import TypeVar from typing import final -from yt_dlp.utils import sanitize_filename - -from ytdl_sub.entries.variables.kwargs import DESCRIPTION -from ytdl_sub.entries.variables.kwargs import EPOCH -from ytdl_sub.entries.variables.kwargs import IE_KEY -from ytdl_sub.entries.variables.kwargs import TITLE -from ytdl_sub.entries.variables.kwargs import UID -from ytdl_sub.entries.variables.kwargs import UPLOADER -from ytdl_sub.entries.variables.kwargs import UPLOADER_ID -from ytdl_sub.entries.variables.kwargs import UPLOADER_URL -from ytdl_sub.entries.variables.kwargs import WEBPAGE_URL - -# pylint: disable=no-member - - -def _sanitize_plex(string: str) -> str: - out = "" - for char in string: - match char: - case "0": - out += "０" - case "1": - out += "１" - case "2": - out += "２" - case "3": - out += "３" - case "4": - out += "４" - case "5": - out += "５" - case "6": - out += "６" - case "7": - out += "７" - case "8": - out += "８" - case "9": - out += "９" - case _: - out += char - return out - - -class BaseEntryVariables: - """ - Source variables are ``{variables}`` that contain metadata from downloaded media. - These variables can be used with fields that expect - :class:`~ytdl_sub.validators.string_formatter_validators.StringFormatterValidator`, - but not - :class:`~ytdl_sub.validators.string_formatter_validators.OverridesStringFormatterValidator`. - """ - - @property - def uid(self: "BaseEntry") -> str: - """ - Returns - ------- - str - The entry's unique ID - """ - return str(self.kwargs(UID)) - - @property - def uid_sanitized(self: "BaseEntry") -> str: - """ - Returns - ------- - str - The sanitized uid of the entry, which is safe to use for Unix and Windows file names. - """ - return sanitize_filename(self.uid) - - @property - def uid_sanitized_plex(self: "BaseEntry") -> str: - """ - Returns - ------- - str - The sanitized uid with additional sanitizing for Plex. Replaces numbers with - fixed-width numbers so Plex does not recognize them as season or episode numbers. - """ - return _sanitize_plex(self.uid_sanitized) - - @property - def extractor(self: "BaseEntry") -> str: - """ - Returns - ------- - str - The ytdl extractor name used in the download archive - """ - # pylint: disable=line-too-long - # Taken from https://github.com/yt-dlp/yt-dlp/blob/e6ab678e36c40ded0aae305bbb866cdab554d417/yt_dlp/YoutubeDL.py#L3514 - # pylint: enable=line-too-long - return str(self.kwargs_get("extractor_key") or self.kwargs(IE_KEY)).lower() - - @property - def epoch(self: "BaseEntry") -> int: - """ - Returns - ------- - int - The unix epoch of when the metadata was scraped by yt-dlp. - """ - return self.kwargs(EPOCH) - - @property - def epoch_date(self: "BaseEntry") -> str: - """ - Returns - ------- - str - The epoch's date, in YYYYMMDD format. - """ - return datetime.utcfromtimestamp(self.epoch).strftime("%Y%m%d") - - @property - def epoch_hour(self: "BaseEntry") -> str: - """ - Returns - ------- - str - The epoch's hour, padded - """ - return datetime.utcfromtimestamp(self.epoch).strftime("%H") - - @property - def title(self: "BaseEntry") -> str: - """ - Returns - ------- - str - The title of the entry. If a title does not exist, returns its unique ID. - """ - return self.kwargs_get(TITLE, self.uid) - - @property - def title_sanitized(self) -> str: - """ - Returns - ------- - str - The sanitized title of the entry, which is safe to use for Unix and Windows file names. - """ - return sanitize_filename(self.title) - - @property - def title_sanitized_plex(self) -> str: - """ - Returns - ------- - str - The sanitized title with additional sanitizing for Plex. Replaces numbers with - fixed-width numbers so Plex does not recognize them as season or episode numbers. - """ - return _sanitize_plex(self.title_sanitized) - - @property - def webpage_url(self: "BaseEntry") -> str: - """ - Returns - ------- - str - The url to the webpage. - """ - return self.kwargs(WEBPAGE_URL) - - @property - def info_json_ext(self) -> str: - """ - Returns - ------- - str - The "info.json" extension - """ - return "info.json" - - @property - def description(self: "BaseEntry") -> str: - """ - Returns - ------- - str - The description if it exists. Otherwise, returns an emtpy string. - """ - return self.kwargs_get(DESCRIPTION, "") - - @property - def uploader_id(self: "BaseEntry") -> str: - """ - Returns - ------- - str - The uploader id if it exists, otherwise return the unique ID. - """ - return self.kwargs_get(UPLOADER_ID, self.uid) - - @property - def uploader(self: "BaseEntry") -> str: - """ - Returns - ------- - str - The uploader if it exists, otherwise return the uploader ID. - """ - return self.kwargs_get(UPLOADER, self.uploader_id) - - @property - def uploader_url(self: "BaseEntry") -> str: - """ - Returns - ------- - str - The uploader url if it exists, otherwise returns the webpage_url. - """ - return self.kwargs_get(UPLOADER_URL, self.webpage_url) - - -# pylint: enable=no-member +from ytdl_sub.entries.script.variable_definitions import VARIABLES +from ytdl_sub.entries.script.variable_definitions import VariableDefinitions +v: VariableDefinitions = VARIABLES TBaseEntry = TypeVar("TBaseEntry", bound="BaseEntry") -class BaseEntry(BaseEntryVariables, ABC): +class BaseEntry(ABC): """ Abstract entry object to represent anything download from ytdl (playlist metadata, media, etc). """ @@ -253,30 +35,63 @@ class BaseEntry(BaseEntryVariables, ABC): self._working_directory = working_directory self._kwargs = entry_dict - self._additional_variables: Dict[str, str | int] = {} + @property + def uid(self) -> str: + """ + Returns + ------- + str + The entry's unique ID + """ + return str(self._kwargs[v.uid.metadata_key]) - def kwargs_contains(self, key: str) -> bool: - """Returns whether internal kwargs contains the specified key""" - return key in self._kwargs + @property + def download_archive_extractor(self) -> str: + """ + The extractor name used in yt-dlp download archives + """ + # pylint: disable=line-too-long + # Taken from https://github.com/yt-dlp/yt-dlp/blob/e6ab678e36c40ded0aae305bbb866cdab554d417/yt_dlp/YoutubeDL.py#L3514 + # pylint: enable=line-too-long + return str( + self._kwargs_get(v.extractor_key.metadata_key) + or self._kwargs_get(v.ie_key.metadata_key) + or "NO_EXTRACTOR" + ).lower() - def kwargs(self, key) -> Any: - """Returns an internal kwarg value supplied from ytdl""" - if not self.kwargs_contains(key): - raise KeyError(f"Expected '{key}' in {self.__class__.__name__} but does not exist.") - output = self._kwargs[key] + @property + def title(self) -> str: + """ + The title of the entry. If a title does not exist, returns its unique ID. + """ + return self._kwargs_get(v.title.metadata_key, self.uid) - # Replace curly braces with unicode version to avoid variable shenanigans - if isinstance(output, str): - return output.replace("{", "｛").replace("}", "｝") - return output + @property + def webpage_url(self) -> str: + """ + The url to the webpage. + """ + return self._kwargs[v.webpage_url.metadata_key] - def kwargs_get(self, key: str, default: Optional[Any] = None) -> Any: + @property + def info_json_ext(self) -> str: + """The "info.json" extension""" + return "info.json" + + @property + def uploader_id(self) -> str: + """ + The uploader id if it exists, otherwise return the unique ID. + """ + return self._kwargs_get(v.uploader_id.metadata_key, self.uid) + + def _kwargs_get(self, key: str, default: Optional[Any] = None) -> Any: """ Dict get on kwargs """ - if not self.kwargs_contains(key) or self.kwargs(key) is None: + if (out := self._kwargs.get(key)) is None: return default - return self.kwargs(key) + return out def working_directory(self) -> str: """ @@ -303,31 +118,6 @@ class BaseEntry(BaseEntryVariables, ABC): self._kwargs = dict(self._kwargs, **variables_to_add) return self - def add_variables(self, variables_to_add: Dict[str, str]) -> "BaseEntry": - """ - Parameters - ---------- - variables_to_add - Variables to add to this entry - - Returns - ------- - self - - Raises - ------ - ValueError - If a variable trying to be added already exists as a source variable - """ - for variable_name in variables_to_add.keys(): - if self.kwargs_contains(variable_name): - raise ValueError( - f"Cannot add variable '{variable_name}': already exists in the kwargs" - ) - - self._additional_variables = dict(self._additional_variables, **variables_to_add) - return self - def get_download_info_json_name(self) -> str: """ Returns @@ -344,36 +134,6 @@ class BaseEntry(BaseEntryVariables, ABC): """ return str(Path(self.working_directory()) / self.get_download_info_json_name()) - def _added_variables(self) -> Dict[str, str]: - """ - Returns - ------- - Dict of variables added to this entry - """ - return self._additional_variables - - @classmethod - def source_variables(cls) -> List[str]: - """ - Returns - ------- - List of all source variables - """ - property_names = [prop for prop in dir(cls) if isinstance(getattr(cls, prop), property)] - return property_names - - @final - def to_dict(self) -> Dict[str, str]: - """ - Returns - ------- - Dictionary containing all variables - """ - source_variable_dict = { - source_var: getattr(self, source_var) for source_var in self.source_variables() - } - return dict(source_variable_dict, **self._added_variables()) - @final def to_type(self, entry_type: Type[TBaseEntry]) -> TBaseEntry: """ @@ -392,7 +152,7 @@ class BaseEntry(BaseEntryVariables, ABC): """ entry_type: Optional[str] = None if isinstance(entry_dict, cls): - entry_type = entry_dict.kwargs_get("_type") + entry_type = entry_dict._kwargs_get("_type") if isinstance(entry_dict, dict): entry_type = entry_dict.get("_type") @@ -407,7 +167,7 @@ class BaseEntry(BaseEntryVariables, ABC): """ entry_ext: Optional[str] = None if isinstance(entry_dict, cls): - entry_ext = entry_dict.kwargs_get("ext") + entry_ext = entry_dict._kwargs_get("ext") if isinstance(entry_dict, dict): entry_ext = entry_dict.get("ext") @@ -419,4 +179,4 @@ class BaseEntry(BaseEntryVariables, ABC): ------- extractor + uid, making this a unique hash for any entry """ - return self.extractor + self.uid + return self.download_archive_extractor + self.uid diff --git a/src/ytdl_sub/entries/entry.py b/src/ytdl_sub/entries/entry.py index 0538c66c..b996c20b 100644 --- a/src/ytdl_sub/entries/entry.py +++ b/src/ytdl_sub/entries/entry.py @@ -1,21 +1,103 @@ +# pylint: disable=protected-access import copy import json import os from pathlib import Path +from typing import Any +from typing import Dict from typing import Optional +from typing import Type +from typing import TypeVar from typing import final from ytdl_sub.entries.base_entry import BaseEntry -from ytdl_sub.entries.variables.entry_variables import EntryVariables +from ytdl_sub.entries.script.variable_definitions import VARIABLES +from ytdl_sub.entries.script.variable_definitions import Variable +from ytdl_sub.entries.script.variable_definitions import VariableDefinitions +from ytdl_sub.script.utils.exceptions import ScriptVariableNotResolved +from ytdl_sub.utils.script import ScriptUtils +from ytdl_sub.utils.scriptable import Scriptable from ytdl_sub.validators.audo_codec_validator import AUDIO_CODEC_EXTS from ytdl_sub.validators.audo_codec_validator import VIDEO_CODEC_EXTS +v: VariableDefinitions = VARIABLES -class Entry(EntryVariables, BaseEntry): +_YTDL_SUB_ENTRY_VARIABLES_KWARG_KEY: str = "ytdl_sub_entry_variables" +ytdl_sub_chapters_from_comments = Variable("ytdl_sub_chapters_from_comments") +ytdl_sub_split_by_chapters_parent_uid = Variable("ytdl_sub_split_by_chapters_parent_uid") + +TypeT = TypeVar("TypeT") + + +class Entry(BaseEntry, Scriptable): """ Entry object to represent a single media object returned from yt-dlp. """ + def __init__(self, entry_dict: Dict, working_directory: str): + BaseEntry.__init__(self, entry_dict=entry_dict, working_directory=working_directory) + Scriptable.__init__(self) + + def _add_entry_kwargs_to_script(self) -> None: + # Add entry metadata, but avoid the `.add()` helper since it also adds sanitized + self.unresolvable.remove(v.entry_metadata.variable_name) + self.script.add({v.entry_metadata.variable_name: ScriptUtils.to_script(self._kwargs)}) + self.update_script() + + def initialize_script(self, other: Optional[Scriptable] = None) -> "Entry": + """ + Initializes the entry script using the Overrides script, then adding + its kwargs to the entry metadata variable + """ + # Overrides contains added variables that are unresolvable, add them here + if other: + self.script = copy.deepcopy(other.script) + self.unresolvable = copy.deepcopy(other.unresolvable) + + self._add_entry_kwargs_to_script() + return self + + def get(self, variable: Variable, expected_type: Type[TypeT]) -> TypeT: + """ + Gets a variable of an expected type. Will error if it does not exist or is not resolved. + """ + out = self.script.resolve(unresolvable=self.unresolvable).get_native(variable.variable_name) + return expected_type(out) + + def try_get(self, variable: Variable, expected_type: Type[TypeT]) -> Optional[TypeT]: + """ + Gets a variable of an expected type. Returns None if it does not exist or is not resolved. + """ + try: + return self.get(variable=variable, expected_type=expected_type) + except ScriptVariableNotResolved: + return None + + def add_injected_variables( + self, download_entry: "Entry", download_idx: int, upload_date_idx: int + ) -> "Entry": + """ + Adds variables that get injected into the Entry script that aren't available at + metadata scrape time (only after the actual download). + """ + self.add( + { + # Tracks number of entries downloaded + v.download_index: download_idx + 1, + # Tracks number of entries with the same upload date to make them unique + v.upload_date_index: upload_date_idx + 1, + v.requested_subtitles: download_entry._kwargs_get( + v.requested_subtitles.metadata_key, [] + ), + v.chapters: download_entry._kwargs_get(v.chapters.metadata_key, []), + v.sponsorblock_chapters: download_entry._kwargs_get( + v.sponsorblock_chapters.metadata_key, [] + ), + v.comments: download_entry._kwargs_get(v.comments.metadata_key, []), + } + ) + return self + @property def ext(self) -> str: """ @@ -23,12 +105,13 @@ class Entry(EntryVariables, BaseEntry): This is not reflected in the entry. See if the mkv file exists and return "mkv" if so, otherwise, return the original extension. """ - for possible_ext in [super().ext, "mkv"]: + ext = self.try_get(v.ext, str) or self._kwargs[v.ext.metadata_key] + for possible_ext in [ext, "mkv"]: file_path = str(Path(self.working_directory()) / f"{self.uid}.{possible_ext}") if os.path.isfile(file_path): return possible_ext - return super().ext + return ext def get_download_file_name(self) -> str: """ @@ -48,7 +131,7 @@ class Entry(EntryVariables, BaseEntry): ------- The download thumbnail's file name """ - return f"{self.uid}.{self.thumbnail_ext}" + return f"{self.uid}.{self.get(v.thumbnail_ext, str)}" def get_download_thumbnail_path(self) -> str: """Returns the entry's thumbnail's file path to where it was downloaded""" @@ -59,7 +142,7 @@ class Entry(EntryVariables, BaseEntry): The source `thumbnail` value and the actual downloaded thumbnail extension sometimes do not match. Return the actual downloaded thumbnail path. """ - thumbnails = self.kwargs_get("thumbnails", []) + thumbnails = self._kwargs_get("thumbnails", []) possible_thumbnail_exts = {"jpg", "webp"} # Always check for jpg and webp thumbs for thumbnail in thumbnails: @@ -77,7 +160,7 @@ class Entry(EntryVariables, BaseEntry): Write the entry's _kwargs back into the info.json file as well as its source variables """ kwargs_dict = copy.deepcopy(self._kwargs) - kwargs_dict["ytdl_sub_entry_variables"] = self.to_dict() + kwargs_dict[_YTDL_SUB_ENTRY_VARIABLES_KWARG_KEY] = self.to_dict() kwargs_json = json.dumps(kwargs_dict, ensure_ascii=False, sort_keys=True, indent=2) with open(self.get_download_info_json_path(), "w", encoding="utf-8") as file: @@ -119,3 +202,39 @@ class Entry(EntryVariables, BaseEntry): break return file_exists + + def maybe_get_prior_variables(self) -> Dict[str, Any]: + """ + If variables exist in the .info.json from a prior run, delete them + from kwargs (to prevent nested writes) and return them + """ + maybe_prior_variables: Dict[str, Any] = {} + if _YTDL_SUB_ENTRY_VARIABLES_KWARG_KEY in self._kwargs: + maybe_prior_variables = self._kwargs[_YTDL_SUB_ENTRY_VARIABLES_KWARG_KEY] + del self._kwargs[_YTDL_SUB_ENTRY_VARIABLES_KWARG_KEY] + + return maybe_prior_variables + + @final + def to_dict(self) -> Dict[str, Any]: + """ + Returns + ------- + Dictionary containing all variables + """ + return self.script.resolve().as_native() + + @classmethod + def create_split_entry(cls, entry: "Entry", new_uid: str) -> "Entry": + """ + Creates a copy of an entry with a new uid to use as the starting point for a split entry + """ + new_entry = copy.deepcopy(entry) + new_entry._kwargs[v.uid.metadata_key] = new_uid + new_entry.add( + { + v.uid.variable_name: new_uid, + ytdl_sub_split_by_chapters_parent_uid.variable_name: entry.uid, + } + ) + return new_entry diff --git a/src/ytdl_sub/entries/entry_parent.py b/src/ytdl_sub/entries/entry_parent.py index 24e184af..4f2669e4 100644 --- a/src/ytdl_sub/entries/entry_parent.py +++ b/src/ytdl_sub/entries/entry_parent.py @@ -1,55 +1,33 @@ import math +from typing import Any from typing import Dict from typing import List from typing import Optional -import mergedeep - from ytdl_sub.entries.base_entry import BaseEntry from ytdl_sub.entries.base_entry import TBaseEntry from ytdl_sub.entries.entry import Entry -from ytdl_sub.entries.variables.kwargs import DESCRIPTION -from ytdl_sub.entries.variables.kwargs import PLAYLIST_COUNT -from ytdl_sub.entries.variables.kwargs import PLAYLIST_DESCRIPTION -from ytdl_sub.entries.variables.kwargs import PLAYLIST_ENTRY -from ytdl_sub.entries.variables.kwargs import PLAYLIST_INDEX -from ytdl_sub.entries.variables.kwargs import PLAYLIST_MAX_UPLOAD_YEAR -from ytdl_sub.entries.variables.kwargs import PLAYLIST_MAX_UPLOAD_YEAR_TRUNCATED -from ytdl_sub.entries.variables.kwargs import PLAYLIST_TITLE -from ytdl_sub.entries.variables.kwargs import PLAYLIST_UID -from ytdl_sub.entries.variables.kwargs import PLAYLIST_UPLOADER -from ytdl_sub.entries.variables.kwargs import PLAYLIST_UPLOADER_ID -from ytdl_sub.entries.variables.kwargs import PLAYLIST_UPLOADER_URL -from ytdl_sub.entries.variables.kwargs import PLAYLIST_WEBPAGE_URL -from ytdl_sub.entries.variables.kwargs import SOURCE_COUNT -from ytdl_sub.entries.variables.kwargs import SOURCE_DESCRIPTION -from ytdl_sub.entries.variables.kwargs import SOURCE_ENTRY -from ytdl_sub.entries.variables.kwargs import SOURCE_INDEX -from ytdl_sub.entries.variables.kwargs import SOURCE_TITLE -from ytdl_sub.entries.variables.kwargs import SOURCE_UID -from ytdl_sub.entries.variables.kwargs import SOURCE_UPLOADER -from ytdl_sub.entries.variables.kwargs import SOURCE_UPLOADER_ID -from ytdl_sub.entries.variables.kwargs import SOURCE_UPLOADER_URL -from ytdl_sub.entries.variables.kwargs import SOURCE_WEBPAGE_URL -from ytdl_sub.entries.variables.kwargs import TITLE -from ytdl_sub.entries.variables.kwargs import UID -from ytdl_sub.entries.variables.kwargs import UPLOADER -from ytdl_sub.entries.variables.kwargs import UPLOADER_ID -from ytdl_sub.entries.variables.kwargs import UPLOADER_URL -from ytdl_sub.entries.variables.kwargs import WEBPAGE_URL +from ytdl_sub.entries.script.variable_definitions import VARIABLES +from ytdl_sub.entries.script.variable_definitions import MetadataVariable +from ytdl_sub.entries.script.variable_definitions import VariableDefinitions +from ytdl_sub.entries.script.variable_scripts import ENTRY_DEFAULT_VARIABLES +from ytdl_sub.entries.script.variable_scripts import ENTRY_REQUIRED_VARIABLES + +v: VariableDefinitions = VARIABLES -class ParentType: - PLAYLIST = "playlist" - SOURCE = "source" - - -def _sort_entries(entries: List[TBaseEntry]) -> List[TBaseEntry]: - """Try sorting by playlist_id first, then fall back to uid""" - return sorted(entries, key=lambda ent: (ent.kwargs_get(PLAYLIST_INDEX, math.inf), ent.uid)) +# pylint: disable=protected-access class EntryParent(BaseEntry): + @classmethod + def _sort_entries(cls, entries: List[TBaseEntry]) -> List[TBaseEntry]: + """Try sorting by playlist_id first, then fall back to uid""" + return sorted( + entries, + key=lambda ent: (ent._kwargs_get(v.playlist_index.metadata_key, math.inf), ent.uid), + ) + def __init__(self, entry_dict: Dict, working_directory: str): super().__init__(entry_dict=entry_dict, working_directory=working_directory) self._parent_children: List["EntryParent"] = [] @@ -73,86 +51,38 @@ class EntryParent(BaseEntry): self.entry_children() ) - def _playlist_variables(self, idx: int, children: List[TBaseEntry], parent_type: str) -> Dict: - _count = self.kwargs_get(PLAYLIST_COUNT, len(children)) - _index = children[idx].kwargs_get(PLAYLIST_INDEX, idx + 1) - - if parent_type == ParentType.SOURCE: - return {SOURCE_INDEX: _index, SOURCE_COUNT: _count} - return { - SOURCE_INDEX: self.kwargs_get(SOURCE_INDEX, 1), - SOURCE_COUNT: self.kwargs_get(SOURCE_INDEX, 1), - PLAYLIST_INDEX: _index, - PLAYLIST_COUNT: _count, - } - - def _parent_variables(self, parent_type: str) -> Dict: - def _(source_key: str, playlist_key: str) -> str: - return playlist_key if parent_type == ParentType.PLAYLIST else source_key - - def __(key: str) -> Optional[str]: - return self.kwargs_get(key=key) - - return { - _(SOURCE_ENTRY, PLAYLIST_ENTRY): self._kwargs, - _(SOURCE_TITLE, PLAYLIST_TITLE): __(TITLE), - _(SOURCE_WEBPAGE_URL, PLAYLIST_WEBPAGE_URL): __(WEBPAGE_URL), - _(SOURCE_UID, PLAYLIST_UID): __(UID), - _(SOURCE_DESCRIPTION, PLAYLIST_DESCRIPTION): __(DESCRIPTION), - _(SOURCE_UPLOADER, PLAYLIST_UPLOADER): __(UPLOADER), - _(SOURCE_UPLOADER_ID, PLAYLIST_UPLOADER_ID): __(UPLOADER_ID), - _(SOURCE_UPLOADER_URL, PLAYLIST_UPLOADER_URL): __(UPLOADER_URL), - } - - def _get_entry_children_variable_list(self, variable_name: str) -> List[str | int]: - return [getattr(entry_child, variable_name) for entry_child in self.entry_children()] - - def _entry_aggregate_variables(self) -> Dict: - if not self.entry_children(): - return {} - - return { - PLAYLIST_MAX_UPLOAD_YEAR: max(self._get_entry_children_variable_list("upload_year")), - PLAYLIST_MAX_UPLOAD_YEAR_TRUNCATED: max( - self._get_entry_children_variable_list("upload_year_truncated") - ), - } - - # pylint: disable=protected-access + def _sibling_entry_metadata(self) -> List[Dict[str, Any]]: + sibling_entry_metadata: List[Dict[str, Any]] = [] + variable_filter: List[MetadataVariable] = list(ENTRY_REQUIRED_VARIABLES.keys()) + list( + ENTRY_DEFAULT_VARIABLES.keys() + ) + for entry in self.entry_children(): + sibling_entry_metadata.append( + {var.metadata_key: entry._kwargs_get(var.metadata_key) for var in variable_filter} + ) + return sibling_entry_metadata def _set_child_variables(self, parents: Optional[List["EntryParent"]] = None) -> "EntryParent": if parents is None: parents = [self] - self.add_kwargs( - self._playlist_variables(idx=0, children=parents, parent_type=ParentType.SOURCE) - ) - kwargs_to_add: Dict = {} + kwargs_to_add: Dict[str, Any] = { + v.sibling_metadata.metadata_key: self._sibling_entry_metadata() + } if len(parents) >= 1: - mergedeep.merge(kwargs_to_add, parents[-1]._parent_variables(ParentType.PLAYLIST)) + kwargs_to_add[v.playlist_metadata.metadata_key] = parents[-1]._kwargs if len(parents) >= 2: - mergedeep.merge(kwargs_to_add, parents[-2]._parent_variables(ParentType.SOURCE)) + kwargs_to_add[v.source_metadata.metadata_key] = parents[-2]._kwargs if len(parents) >= 3: raise ValueError( "ytdl-sub currently does support more than 3 layers of playlists/entries. " "If you encounter this error, please file a ticket with the URLs used." ) - mergedeep.merge(kwargs_to_add, self._entry_aggregate_variables()) - for idx, entry_child in enumerate(self.entry_children()): - entry_child.add_kwargs( - self._playlist_variables( - idx=idx, children=self.entry_children(), parent_type=ParentType.PLAYLIST - ) - ) - entry_child.add_kwargs(kwargs_to_add) + for entry_child in self.entry_children(): + entry_child._kwargs = dict(entry_child._kwargs, **kwargs_to_add) - for idx, parent_child in enumerate(self.parent_children()): - parent_child.add_kwargs( - self._playlist_variables( - idx=idx, children=self.parent_children(), parent_type=ParentType.SOURCE - ) - ) + for parent_child in self.parent_children(): parent_child._set_child_variables(parents=parents + [parent_child]) return self @@ -170,8 +100,10 @@ class EntryParent(BaseEntry): if entry_dict in self ] - self._parent_children = _sort_entries([ent for ent in entries if self.is_entry_parent(ent)]) - self._entry_children = _sort_entries( + self._parent_children = self._sort_entries( + [ent for ent in entries if self.is_entry_parent(ent)] + ) + self._entry_children = self._sort_entries( [ent.to_type(Entry) for ent in entries if self.is_entry(ent)] ) @@ -190,7 +122,7 @@ class EntryParent(BaseEntry): ------- Desired thumbnail url if it exists. None if it does not. """ - for thumbnail in self.kwargs_get("thumbnails", []): + for thumbnail in self._kwargs_get("thumbnails", []): if thumbnail["id"] == thumbnail_id: return thumbnail["url"] return None @@ -200,7 +132,7 @@ class EntryParent(BaseEntry): if isinstance(item, dict): playlist_id = item.get("playlist_id") elif isinstance(item, BaseEntry): - playlist_id = item.kwargs_get("playlist_id") + playlist_id = item._kwargs_get("playlist_id") if not playlist_id: return False @@ -275,11 +207,12 @@ class EntryParent(BaseEntry): return parents - # pylint: enable=protected-access - @classmethod def from_entry_dicts_with_no_parents( - cls, parents: List["EntryParent"], entry_dicts: List[Dict], working_directory: str + cls, + parents: List["EntryParent"], + entry_dicts: List[Dict], + working_directory: str, ) -> List[Entry]: """ Reads all entries that do not have any parents @@ -289,7 +222,10 @@ class EntryParent(BaseEntry): return any(entry_dict in parent for parent in parents) return [ - Entry(entry_dict=entry_dict, working_directory=working_directory) + Entry( + entry_dict=entry_dict, + working_directory=working_directory, + ) for entry_dict in entry_dicts if cls.is_entry(entry_dict) and not _in_any_parents(entry_dict) ] diff --git a/src/ytdl_sub/entries/script/__init__.py b/src/ytdl_sub/entries/script/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/ytdl_sub/entries/script/custom_functions.py b/src/ytdl_sub/entries/script/custom_functions.py new file mode 100644 index 00000000..a7440b9d --- /dev/null +++ b/src/ytdl_sub/entries/script/custom_functions.py @@ -0,0 +1,178 @@ +import os +import posixpath + +from yt_dlp.utils import sanitize_filename + +from ytdl_sub.script.functions import Functions +from ytdl_sub.script.types.map import Map +from ytdl_sub.script.types.resolvable import AnyArgument +from ytdl_sub.script.types.resolvable import Integer +from ytdl_sub.script.types.resolvable import ReturnableArgument +from ytdl_sub.script.types.resolvable import String +from ytdl_sub.script.utils.exceptions import RuntimeException +from ytdl_sub.utils.file_path import FilePathTruncater + + +def _pad(num: int, width: int): + return str(num).zfill(width) + + +_days_in_month = [0, 31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31] + + +class CustomFunctions: + @staticmethod + def legacy_bracket_safety(value: ReturnableArgument) -> ReturnableArgument: + """ + ytdl-sub used to replace brackets ('{', '}') with unicode brackets ('｛', '｝') to not + interfere with its legacy variable scripting system. This function replicates that + behavior. + """ + if isinstance(value, String): + value = String(value.value.replace("{", "｛").replace("}", "｝")) + return value + + @staticmethod + def to_native_filepath(filepath: String) -> String: + """ + Convert any unix-based path separators ('/') with the OS's native + separator. + """ + return String(filepath.value.replace(posixpath.sep, os.sep)) + + @staticmethod + def truncate_filepath_if_too_long(filepath: String) -> String: + """ + If a file-path is too long for the OS, this function will truncate it while preserving + the extension. + """ + return String(FilePathTruncater.maybe_truncate_file_path(filepath.value)) + + @staticmethod + def sanitize(value: AnyArgument) -> String: + """ + Sanitize a string using yt-dlp's ``sanitize_filename`` method to ensure it's safe to use + for file/directory names on any OS. + """ + return String(sanitize_filename(str(value))) + + @staticmethod + def sanitize_plex_episode(string: String) -> String: + """ + Sanitize a string using ``sanitize`` and replace numerics with their respective fixed-width + numbers. This is used to have Plex avoid scraping numbers like ``4x4`` as the + season and/or episode. + """ + sanitized_string = CustomFunctions.sanitize(string).value + out = "" + for char in sanitized_string: + match char: + case "0": + out += "０" + case "1": + out += "１" + case "2": + out += "２" + case "3": + out += "３" + case "4": + out += "４" + case "5": + out += "５" + case "6": + out += "６" + case "7": + out += "７" + case "8": + out += "８" + case "9": + out += "９" + case _: + out += char + return String(out) + + @staticmethod + def to_date_metadata(yyyymmdd: String) -> Map: + """ + Takes a date in the form of YYYYMMDD and returns a Map containing: + + - date (String, YYYYMMDD) + - date_standardized (String, YYYY-MM-DD) + - year (Integer) + - month (Integer) + - day (Integer) + - year_truncated (String, YY from YY[YY]) + - month_padded (String) + - day_padded (String) + - year_truncated_reversed (Integer, 100 - year_truncated) + - month_reversed (Integer, 13 - month) + - month_reversed_padded (String) + - day_reversed (Integer, total_days_in_month + 1 - day) + - day_reversed_padded (String) + - day_of_year (Integer) + - day_of_year_padded (String, padded 3) + - day_of_year_reversed (Integer, total_days_in_year + 1 - day_of_year) + - day_of_year_reversed_padded (String, padded 3) + """ + date_str = yyyymmdd.value + if not (date_str.isnumeric() and len(date_str) == 8): + raise RuntimeException( + f"Expected input of date_metadata to be YYYYMMDD, but received {date_str}" + ) + + year: int = int(date_str[:4]) + month_padded: str = date_str[4:6] + day_padded: str = date_str[6:8] + + month: int = int(month_padded) + day: int = int(day_padded) + year_truncated: int = int(str(year)[-2:]) + + day_of_year: int = sum(_days_in_month[:month]) + day + total_days_in_month: int = _days_in_month[month] + total_days_in_year: int = 365 + if year % 4 == 0: + total_days_in_year += 1 + if month == 2: + total_days_in_month += 1 + if month > 2: + day_of_year += 1 + + day_of_year_reversed: int = total_days_in_year + 1 - day_of_year + month_reversed: int = 13 - month + day_reversed: int = total_days_in_month + 1 - day + + return Map( + { + String("date"): yyyymmdd, + String("date_standardized"): String(f"{year}-{month_padded}-{day_padded}"), + String("year"): Integer(year), + String("month"): Integer(month), + String("day"): Integer(day), + String("year_truncated"): String(year_truncated), + String("month_padded"): String(month_padded), + String("day_padded"): String(day_padded), + String("year_truncated_reversed"): Integer(100 - year_truncated), + String("month_reversed"): Integer(month_reversed), + String("month_reversed_padded"): String(_pad(month_reversed, width=2)), + String("day_reversed"): Integer(day_reversed), + String("day_reversed_padded"): String(_pad(day_reversed, width=2)), + String("day_of_year"): Integer(day_of_year), + String("day_of_year_padded"): String(_pad(day_of_year, width=3)), + String("day_of_year_reversed"): Integer(day_of_year_reversed), + String("day_of_year_reversed_padded"): String(_pad(day_of_year_reversed, width=3)), + } + ) + + @staticmethod + def register(): + """ + Register Custom functions once and only once + """ + if not Functions.is_built_in("sanitize"): + Functions.register_function(CustomFunctions.legacy_bracket_safety) + Functions.register_function(CustomFunctions.truncate_filepath_if_too_long) + Functions.register_function(CustomFunctions.to_native_filepath) + Functions.register_function(CustomFunctions.sanitize) + Functions.register_function(CustomFunctions.sanitize_plex_episode) + Functions.register_function(CustomFunctions.to_date_metadata) diff --git a/src/ytdl_sub/entries/script/function_scripts.py b/src/ytdl_sub/entries/script/function_scripts.py new file mode 100644 index 00000000..85622187 --- /dev/null +++ b/src/ytdl_sub/entries/script/function_scripts.py @@ -0,0 +1,37 @@ +from typing import Dict + +from ytdl_sub.entries.script.variable_definitions import VARIABLES +from ytdl_sub.entries.script.variable_definitions import VariableDefinitions + +v: VariableDefinitions = VARIABLES + +CUSTOM_FUNCTION_SCRIPTS: Dict[str, str] = { + "%extract_field_from_metadata_array_getter": """{ + %map_get( %map(%array_at($0, 0)), %array_at($0, 1) ) + }""", + "%extract_field_from_metadata_array": """{ + %if( + %bool($0), + %array_extend( + %array_apply( + %array_product( + %array($0), + [ %string($1) ] + ), + %extract_field_from_metadata_array_getter + ) + ), + [] + ) + }""", + "%extract_field_from_siblings": f"""{{ + %if( + %bool({v.sibling_metadata.variable_name}), + %extract_field_from_metadata_array( + {v.sibling_metadata.variable_name}, + $0 + ), + [] + ) + }}""", +} diff --git a/src/ytdl_sub/entries/script/variable_definitions.py b/src/ytdl_sub/entries/script/variable_definitions.py new file mode 100644 index 00000000..6c576db4 --- /dev/null +++ b/src/ytdl_sub/entries/script/variable_definitions.py @@ -0,0 +1,991 @@ +from dataclasses import dataclass + +# This file contains mixins to a BaseEntry subclass. Ignore pylint's "no kwargs member" suggestion +# pylint: disable=no-member +# pylint: disable=too-many-public-methods + + +@dataclass(frozen=True) +class Variable: + variable_name: str + + +@dataclass(frozen=True) +class InternalVariable(Variable): + pass + + +@dataclass(frozen=True) +class Metadata(Variable): + pass + + +@dataclass(frozen=True) +class MetadataVariable(Variable): + metadata_key: str + + +@dataclass(frozen=True) +class RelativeMetadata(MetadataVariable, Metadata): + pass + + +@dataclass(frozen=True) +class SiblingMetadata(MetadataVariable): + pass + + +class VariableDefinitions: + @property + def entry_metadata(self) -> Metadata: + """ + The entry's info.json + """ + return Metadata("entry_metadata") + + @property + def playlist_metadata(self) -> RelativeMetadata: + """ + Metadata from the playlist (i.e. the parent metadata, like playlist -> entry) + """ + return RelativeMetadata("playlist_metadata", metadata_key="playlist_metadata") + + @property + def source_metadata(self) -> RelativeMetadata: + """ + Metadata from the source (i.e. the grandparent metadata, like channel -> playlist -> entry) + """ + return RelativeMetadata("source_metadata", metadata_key="source_metadata") + + @property + def sibling_metadata(self) -> SiblingMetadata: + """ + Metadata from any sibling entries that reside in the same playlist as this entry. + """ + return SiblingMetadata("sibling_metadata", metadata_key="sibling_metadata") + + @property + def uid(self) -> MetadataVariable: + """ + The entry's unique ID + """ + return MetadataVariable(metadata_key="id", variable_name="uid") + + @property + def duration(self) -> MetadataVariable: + """ + The duration of the entry in seconds + """ + return MetadataVariable("duration", metadata_key="duration") + + @property + def uid_sanitized_plex(self) -> Variable: + """ + The sanitized uid with additional sanitizing for Plex. Replaces numbers with + fixed-width numbers so Plex does not recognize them as season or episode numbers. + """ + return Variable("uid_sanitized_plex") + + @property + def ie_key(self) -> MetadataVariable: + """ + The ie_key, used in legacy yt-dlp things as the 'info-extractor key' + """ + return MetadataVariable(metadata_key="ie_key", variable_name="ie_key") + + @property + def extractor_key(self) -> MetadataVariable: + """ + The yt-dlp extractor key + """ + return MetadataVariable(metadata_key="extractor_key", variable_name="extractor_key") + + @property + def extractor(self) -> MetadataVariable: + """ + The yt-dlp extractor name + """ + return MetadataVariable(variable_name="extractor", metadata_key="extractor") + + @property + def epoch(self) -> MetadataVariable: + """ + The unix epoch of when the metadata was scraped by yt-dlp. + """ + return MetadataVariable(metadata_key="epoch", variable_name="epoch") + + @property + def epoch_date(self) -> Variable: + """ + The epoch's date, in YYYYMMDD format. + """ + return Variable("epoch_date") + + @property + def epoch_hour(self) -> Variable: + """ + The epoch's hour + """ + return Variable("epoch_hour") + + @property + def title(self) -> MetadataVariable: + """ + The title of the entry. If a title does not exist, returns its unique ID. + """ + return MetadataVariable(variable_name="title", metadata_key="title") + + @property + def title_sanitized_plex(self) -> Variable: + """ + The sanitized title with additional sanitizing for Plex. It replaces numbers with + fixed-width numbers so Plex does not recognize them as season or episode numbers. + """ + return Variable("title_sanitized_plex") + + @property + def webpage_url(self) -> MetadataVariable: + """ + Returns + ------- + str + The url to the webpage. + """ + return MetadataVariable(metadata_key="webpage_url", variable_name="webpage_url") + + @property + def info_json_ext(self) -> Variable: + """ + Returns + ------- + str + The "info.json" extension + """ + return Variable("info_json_ext") + + @property + def description(self) -> MetadataVariable: + """ + Returns + ------- + str + The description if it exists. Otherwise, returns an emtpy string. + """ + return MetadataVariable(variable_name="description", metadata_key="description") + + @property + def uploader_id(self) -> MetadataVariable: + """ + Returns + ------- + str + The uploader id if it exists, otherwise return the unique ID. + """ + return MetadataVariable(variable_name="uploader_id", metadata_key="uploader_id") + + @property + def uploader(self) -> MetadataVariable: + """ + Returns + ------- + str + The uploader if it exists, otherwise return the uploader ID. + """ + return MetadataVariable(variable_name="uploader", metadata_key="uploader") + + @property + def uploader_url(self) -> MetadataVariable: + """ + Returns + ------- + str + The uploader url if it exists, otherwise returns the webpage_url. + """ + return MetadataVariable("uploader_url", metadata_key="uploader_url") + + @property + def source_title(self) -> MetadataVariable: + """ + Returns + ------- + str + Name of the source (i.e. channel with multiple playlists) if it exists, otherwise + returns its playlist_title. + """ + return MetadataVariable("source_title", metadata_key=self.title.metadata_key) + + @property + def source_uid(self) -> MetadataVariable: + """ + Returns + ------- + str + The source unique id if it exists, otherwise returns the playlist unique ID. + """ + return MetadataVariable("source_uid", metadata_key=self.uid.metadata_key) + + @property + def source_index(self) -> MetadataVariable: + """ + Returns + ------- + int + Source index if it exists, otherwise returns ``1``. + + It is recommended to not use this unless you know the source will never add new content + (it is easy for this value to change). + """ + return MetadataVariable("source_index", metadata_key=self.playlist_index.metadata_key) + + @property + def source_index_padded(self) -> Variable: + """ + Returns + ------- + int + The source index, padded. + """ + return Variable("source_index_padded") + + @property + def source_count(self) -> MetadataVariable: + """ + Returns + ------- + int + The source count if it exists, otherwise returns the playlist count. + """ + return MetadataVariable("source_count", metadata_key=self.playlist_count.metadata_key) + + @property + def source_webpage_url(self) -> MetadataVariable: + """ + Returns + ------- + str + The source webpage url if it exists, otherwise returns the playlist webpage url. + """ + return MetadataVariable("source_webpage_url", metadata_key=self.webpage_url.metadata_key) + + @property + def source_description(self) -> MetadataVariable: + """ + Returns + ------- + str + The source description if it exists, otherwise returns the playlist description. + """ + return MetadataVariable("source_description", metadata_key=self.description.metadata_key) + + @property + def playlist_uid(self) -> MetadataVariable: + """ + Returns + ------- + str + The playlist unique ID if it exists, otherwise return the entry unique ID. + """ + return MetadataVariable(variable_name="playlist_uid", metadata_key="playlist_id") + + @property + def playlist_title(self) -> MetadataVariable: + """ + Returns + ------- + str + Name of its parent playlist/channel if it exists, otherwise returns its title. + """ + return MetadataVariable(variable_name="playlist_title", metadata_key="playlist_title") + + @property + def playlist_index(self) -> MetadataVariable: + """ + Returns + ------- + int + Playlist index if it exists, otherwise returns ``1``. + + Note that for channels/playlists, any change (i.e. adding or removing a video) will make + this value change. Use with caution. + """ + return MetadataVariable(metadata_key="playlist_index", variable_name="playlist_index") + + @property + def playlist_index_reversed(self) -> Variable: + """ + Returns + ------- + int + Playlist index reversed via ``playlist_count - playlist_index + 1`` + """ + return Variable("playlist_index_reversed") + + @property + def playlist_index_padded(self) -> Variable: + """ + Returns + ------- + str + playlist_index padded two digits + """ + return Variable("playlist_index_padded") + + @property + def playlist_index_reversed_padded(self) -> Variable: + """ + Returns + ------- + str + playlist_index_reversed padded two digits + """ + return Variable("playlist_index_reversed_padded") + + @property + def playlist_index_padded6(self) -> Variable: + """ + Returns + ------- + str + playlist_index padded six digits. + """ + return Variable("playlist_index_padded6") + + @property + def playlist_index_reversed_padded6(self) -> Variable: + """ + Returns + ------- + str + playlist_index_reversed padded six digits. + """ + return Variable("playlist_index_reversed_padded6") + + @property + def playlist_count(self) -> MetadataVariable: + """ + Returns + ------- + int + Playlist count if it exists, otherwise returns ``1``. + + Note that for channels/playlists, any change (i.e. adding or removing a video) will make + this value change. Use with caution. + """ + return MetadataVariable(variable_name="playlist_count", metadata_key="playlist_count") + + @property + def playlist_description(self) -> MetadataVariable: + """ + Returns + ------- + str + The playlist description if it exists, otherwise returns the entry's description. + """ + return MetadataVariable( + variable_name="playlist_description", metadata_key=self.description.metadata_key + ) + + @property + def playlist_webpage_url(self) -> MetadataVariable: + """ + Returns + ------- + str + The playlist webpage url if it exists. Otherwise, returns the entry webpage url. + """ + return MetadataVariable( + variable_name="playlist_webpage_url", metadata_key=self.webpage_url.metadata_key + ) + + @property + def playlist_max_upload_date(self) -> Variable: + """ + Returns + ------- + Max upload_date for all entries in this entry's playlist if it exists, otherwise returns + ``upload_date`` + """ + return Variable("playlist_max_upload_date") + + @property + def playlist_max_upload_year(self) -> Variable: + """ + Returns + ------- + int + Max upload_year for all entries in this entry's playlist if it exists, otherwise returns + ``upload_year`` + """ + # override in EntryParent + return Variable("playlist_max_upload_year") + + @property + def playlist_max_upload_year_truncated(self) -> Variable: + """ + Returns + ------- + int + The max playlist truncated upload year for all entries in this entry's playlist if it + exists, otherwise returns ``upload_year_truncated``. + """ + return Variable("playlist_max_upload_year_truncated") + + @property + def playlist_uploader_id(self) -> MetadataVariable: + """ + Returns + ------- + str + The playlist uploader id if it exists, otherwise returns the entry uploader ID. + """ + return MetadataVariable("playlist_uploader_id", metadata_key="playlist_uploader_id") + + @property + def playlist_uploader(self) -> MetadataVariable: + """ + Returns + ------- + str + The playlist uploader if it exists, otherwise return the entry uploader. + """ + return MetadataVariable("playlist_uploader", metadata_key=self.uploader.metadata_key) + + @property + def playlist_uploader_url(self) -> MetadataVariable: + """ + Returns + ------- + str + The playlist uploader url if it exists, otherwise returns the playlist webpage_url. + """ + return MetadataVariable( + "playlist_uploader_url", metadata_key=self.uploader_url.metadata_key + ) + + @property + def source_uploader_id(self) -> MetadataVariable: + """ + Returns + ------- + str + The source uploader id if it exists, otherwise returns the playlist_uploader_id + """ + return MetadataVariable("source_uploader_id", metadata_key=self.uploader_id.metadata_key) + + @property + def source_uploader(self) -> MetadataVariable: + """ + Returns + ------- + str + The source uploader if it exists, otherwise return the playlist_uploader + """ + return MetadataVariable("source_uploader", metadata_key=self.uploader.metadata_key) + + @property + def source_uploader_url(self) -> MetadataVariable: + """ + Returns + ------- + str + The source uploader url if it exists, otherwise returns the source webpage_url. + """ + return MetadataVariable("source_uploader_url", metadata_key=self.uploader_url.metadata_key) + + @property + def creator(self) -> MetadataVariable: + """ + Returns + ------- + str + The creator name if it exists, otherwise returns the channel. + """ + return MetadataVariable(variable_name="creator", metadata_key="creator") + + @property + def channel(self) -> MetadataVariable: + """ + Returns + ------- + str + The channel name if it exists, otherwise returns the uploader. + """ + return MetadataVariable(variable_name="channel", metadata_key="channel") + + @property + def channel_id(self) -> MetadataVariable: + """ + Returns + ------- + str + The channel id if it exists, otherwise returns the entry uploader ID. + """ + return MetadataVariable(variable_name="channel_id", metadata_key="channel_id") + + @property + def ext(self) -> MetadataVariable: + """ + Returns + ------- + str + The downloaded entry's file extension + """ + return MetadataVariable(variable_name="ext", metadata_key="ext") + + @property + def thumbnail_ext(self) -> Variable: + """ + Returns + ------- + str + The download entry's thumbnail extension. Will always return 'jpg'. Until there is a + need to support other image types, we always convert to jpg. + """ + return Variable("thumbnail_ext") + + @property + def comments(self) -> MetadataVariable: + """ + Comments if they are requested + """ + return MetadataVariable("comments", "comments") + + @property + def chapters(self) -> MetadataVariable: + """ + Chapters if they exist + """ + return MetadataVariable("chapters", "chapters") + + @property + def sponsorblock_chapters(self) -> MetadataVariable: + """ + Sponsorblock Chapters if they are requested and exist + """ + return MetadataVariable("sponsorblock_chapters", "sponsorblock_chapters") + + @property + def requested_subtitles(self) -> MetadataVariable: + """ + Subtitles if they are requested and exist + """ + return MetadataVariable("requested_subtitles", "requested_subtitles") + + @property + def ytdl_sub_input_url(self) -> Variable: + """ + The input URL used in ytdl-sub to create this entry. + """ + return Variable("ytdl_sub_input_url") + + @property + def download_index(self) -> Variable: + """ + Returns + ------- + int + The i'th entry downloaded. NOTE that this is fetched dynamically from the download + archive. + """ + return Variable(variable_name="download_index") + + @property + def download_index_padded6(self) -> Variable: + """ + Returns + ------- + str + The download_index padded six digits + """ + return Variable("download_index_padded6") + + @property + def upload_date_index(self) -> Variable: + """ + Returns + ------- + int + The i'th entry downloaded with this upload date. + """ + return Variable(variable_name="upload_date_index") + + @property + def upload_date_index_padded(self) -> Variable: + """ + Returns + ------- + int + The upload_date_index padded two digits + """ + return Variable("upload_date_index_padded") + + @property + def upload_date_index_reversed(self) -> Variable: + """ + Returns + ------- + int + 100 - upload_date_index + """ + return Variable("upload_date_index_reversed") + + @property + def upload_date_index_reversed_padded(self) -> Variable: + """ + Returns + ------- + int + The upload_date_index padded two digits + """ + return Variable("upload_date_index_reversed_padded") + + @property + def upload_date(self) -> MetadataVariable: + """ + Returns + ------- + str + The entry’s uploaded date, in YYYYMMDD format. If not present, return today’s date. + """ + return MetadataVariable(variable_name="upload_date", metadata_key="upload_date") + + @property + def upload_year(self) -> Variable: + """ + Returns + ------- + int + The entry's upload year + """ + return Variable("upload_year") + + @property + def upload_year_truncated(self) -> Variable: + """ + Returns + ------- + int + The last two digits of the upload year, i.e. 22 in 2022 + """ + return Variable("upload_year_truncated") + + @property + def upload_year_truncated_reversed(self) -> Variable: + """ + Returns + ------- + int + The upload year truncated, but reversed using ``100 - {upload_year_truncated}``, i.e. + 2022 returns ``100 - 22`` = ``78`` + """ + return Variable("upload_year_truncated_reversed") + + @property + def upload_month_reversed(self) -> Variable: + """ + Returns + ------- + int + The upload month, but reversed using ``13 - {upload_month}``, i.e. March returns ``10`` + """ + return Variable("upload_month_reversed") + + @property + def upload_month_reversed_padded(self) -> Variable: + """ + Returns + ------- + str + The reversed upload month, but padded. i.e. November returns "02" + """ + return Variable("upload_month_reversed_padded") + + @property + def upload_month_padded(self) -> Variable: + """ + Returns + ------- + str + The entry's upload month padded to two digits, i.e. March returns "03" + """ + return Variable("upload_month_padded") + + @property + def upload_day_padded(self) -> Variable: + """ + Returns + ------- + str + The entry's upload day padded to two digits, i.e. the fifth returns "05" + """ + return Variable("upload_day_padded") + + @property + def upload_month(self) -> Variable: + """ + Returns + ------- + int + The upload month as an integer (no padding). + """ + return Variable("upload_month") + + @property + def upload_day(self) -> Variable: + """ + Returns + ------- + int + The upload day as an integer (no padding). + """ + return Variable("upload_day") + + @property + def upload_day_reversed(self) -> Variable: + """ + Returns + ------- + int + The upload day, but reversed using ``{total_days_in_month} + 1 - {upload_day}``, + i.e. August 8th would have upload_day_reversed of ``31 + 1 - 8`` = ``24`` + """ + return Variable("upload_day_reversed") + + @property + def upload_day_reversed_padded(self) -> Variable: + """ + Returns + ------- + str + The reversed upload day, but padded. i.e. August 30th returns "02". + """ + return Variable("upload_day_reversed_padded") + + @property + def upload_day_of_year(self) -> Variable: + """ + Returns + ------- + int + The day of the year, i.e. February 1st returns ``32`` + """ + return Variable("upload_day_of_year") + + @property + def upload_day_of_year_padded(self) -> Variable: + """ + Returns + ------- + str + The upload day of year, but padded i.e. February 1st returns "032" + """ + return Variable("upload_day_of_year_padded") + + @property + def upload_day_of_year_reversed(self) -> Variable: + """ + Returns + ------- + int + The upload day, but reversed using ``{total_days_in_year} + 1 - {upload_day}``, + i.e. February 2nd would have upload_day_of_year_reversed of ``365 + 1 - 32`` = ``334`` + """ + return Variable("upload_day_of_year_reversed") + + @property + def upload_day_of_year_reversed_padded(self) -> Variable: + """ + Returns + ------- + str + The reversed upload day of year, but padded i.e. December 31st returns "001" + """ + return Variable("upload_day_of_year_reversed_padded") + + @property + def upload_date_standardized(self) -> Variable: + """ + Returns + ------- + str + The uploaded date formatted as YYYY-MM-DD + """ + return Variable("upload_date_standardized") + + @property + def release_date(self) -> MetadataVariable: + """ + Returns + ------- + str + The entry’s release date, in YYYYMMDD format. If not present, return the upload date. + """ + return MetadataVariable(variable_name="release_date", metadata_key="release_date") + + @property + def release_year(self) -> Variable: + """ + Returns + ------- + int + The entry's release year + """ + return Variable("release_year") + + @property + def release_year_truncated(self) -> Variable: + """ + Returns + ------- + int + The last two digits of the release year, i.e. 22 in 2022 + """ + return Variable("release_year_truncated") + + @property + def release_year_truncated_reversed(self) -> Variable: + """ + Returns + ------- + int + The release year truncated, but reversed using ``100 - {release_year_truncated}``, i.e. + 2022 returns ``100 - 22`` = ``78`` + """ + return Variable("release_year_truncated_reversed") + + @property + def release_month_reversed(self) -> Variable: + """ + Returns + ------- + int + The release month, but reversed + using ``13 - {release_month}``, i.e. March returns ``10`` + """ + return Variable("release_month_reversed") + + @property + def release_month_reversed_padded(self) -> Variable: + """ + Returns + ------- + str + The reversed release month, but padded. i.e. November returns "02" + """ + return Variable("release_month_reversed_padded") + + @property + def release_month_padded(self) -> Variable: + """ + Returns + ------- + str + The entry's release month padded to two digits, i.e. March returns "03" + """ + return Variable("release_month_padded") + + @property + def release_day_padded(self) -> Variable: + """ + Returns + ------- + str + The entry's release day padded to two digits, i.e. the fifth returns "05" + """ + return Variable("release_day_padded") + + @property + def release_month(self) -> Variable: + """ + Returns + ------- + int + The release month as an integer (no padding). + """ + return Variable("release_month") + + @property + def release_day(self) -> Variable: + """ + Returns + ------- + int + The release day as an integer (no padding). + """ + return Variable("release_day") + + @property + def release_day_reversed(self) -> Variable: + """ + Returns + ------- + int + The release day, but reversed using ``{total_days_in_month} + 1 - {release_day}``, + i.e. August 8th would have release_day_reversed of ``31 + 1 - 8`` = ``24`` + """ + return Variable("release_day_reversed") + + @property + def release_day_reversed_padded(self) -> Variable: + """ + Returns + ------- + str + The reversed release day, but padded. i.e. August 30th returns "02". + """ + return Variable("release_day_reversed_padded") + + @property + def release_day_of_year(self) -> Variable: + """ + Returns + ------- + int + The day of the year, i.e. February 1st returns ``32`` + """ + return Variable("release_day_of_year") + + @property + def release_day_of_year_padded(self) -> Variable: + """ + Returns + ------- + str + The release day of year, but padded i.e. February 1st returns "032" + """ + return Variable("release_day_of_year_padded") + + @property + def release_day_of_year_reversed(self) -> Variable: + """ + Returns + ------- + int + The release day, but reversed using ``{total_days_in_year} + 1 - {release_day}``, + i.e. February 2nd would have release_day_of_year_reversed of ``365 + 1 - 32`` = ``334`` + """ + return Variable("release_day_of_year_reversed") + + @property + def release_day_of_year_reversed_padded(self) -> Variable: + """ + Returns + ------- + str + The reversed release day of year, but padded i.e. December 31st returns "001" + """ + return Variable("release_day_of_year_reversed_padded") + + @property + def release_date_standardized(self) -> Variable: + """ + Returns + ------- + str + The release date formatted as YYYY-MM-DD + """ + return Variable("release_date_standardized") + + +# Singleton to use externally +VARIABLES: VariableDefinitions = VariableDefinitions() diff --git a/src/ytdl_sub/entries/script/variable_scripts.py b/src/ytdl_sub/entries/script/variable_scripts.py new file mode 100644 index 00000000..90808c37 --- /dev/null +++ b/src/ytdl_sub/entries/script/variable_scripts.py @@ -0,0 +1,296 @@ +from typing import Dict +from typing import List +from typing import Optional +from typing import Set + +import mergedeep + +from ytdl_sub.entries.script.custom_functions import CustomFunctions +from ytdl_sub.entries.script.variable_definitions import VARIABLES +from ytdl_sub.entries.script.variable_definitions import Metadata +from ytdl_sub.entries.script.variable_definitions import MetadataVariable +from ytdl_sub.entries.script.variable_definitions import Variable +from ytdl_sub.entries.script.variable_definitions import VariableDefinitions + +############################################################################################### +# Helpers + +v: VariableDefinitions = VARIABLES + + +def _pad_int(key: Variable, pad: int) -> str: + return f"{{%pad_zero({key.variable_name}, {pad})}}" + + +def _sanitized_plex(key: Variable) -> str: + return f"{{%sanitize_plex_episode({key.variable_name})}}" + + +def _date_metadata(date_key: Variable, metadata_key: str) -> str: + return f"{{%map_get(%to_date_metadata({date_key.variable_name}), '{metadata_key}')}}" + + +############################################################################################### +# Metadata Getters + + +def _get( + cast: str, + metadata: Metadata, + key: MetadataVariable, + default: Optional[Variable | str | int | Dict | List], +) -> str: + if default is None: + # TODO: assert with good error message if key DNE + out = f"%map_get({metadata.variable_name}, '{key.metadata_key}')" + elif isinstance(default, Variable): + args = f"{metadata.variable_name}, '{key.metadata_key}', {default.variable_name}" + out = f"%map_get_non_empty({args})" + elif isinstance(default, str): + out = f"%map_get_non_empty({metadata.variable_name}, '{key.metadata_key}', '{default}')" + elif isinstance(default, dict): + out = f"%map_get_non_empty({metadata.variable_name}, '{key.metadata_key}', {{}})" + elif isinstance(default, list): + out = f"%map_get_non_empty({metadata.variable_name}, '{key.metadata_key}', [])" + else: + out = f"%map_get_non_empty({metadata.variable_name}, '{key.metadata_key}', {default})" + + return f"{{ %legacy_bracket_safety(%{cast}({out})) }}" + + +############################################################################################### +# Entry Getters + + +def _entry_get_str(key: MetadataVariable, default: Optional[Variable | str] = None) -> str: + return _get("string", metadata=v.entry_metadata, key=key, default=default) + + +def _entry_get_int(key: MetadataVariable, default: Optional[Variable | int] = None) -> str: + return _get("int", metadata=v.entry_metadata, key=key, default=default) + + +def _entry_get_map(key: MetadataVariable, default: Optional[Variable | Dict] = None): + return _get("map", metadata=v.entry_metadata, key=key, default=default) + + +def _entry_get_array(key: MetadataVariable, default: Optional[Variable | List] = None): + return _get("array", metadata=v.entry_metadata, key=key, default=default) + + +############################################################################################### +# Playlist Getters + + +def _playlist_get_str(key: MetadataVariable, default: Optional[Variable | str] = None) -> str: + return _get("string", metadata=v.playlist_metadata, key=key, default=default) + + +def _playlist_get_int(key: MetadataVariable, default: Optional[Variable | int] = None) -> str: + return _get("int", metadata=v.playlist_metadata, key=key, default=default) + + +############################################################################################### +# Source Getters + + +def _source_get_str(key: MetadataVariable, default: Optional[Variable | str] = None) -> str: + return _get("string", metadata=v.source_metadata, key=key, default=default) + + +############################################################################################### +# Scripts + +ENTRY_EMPTY_METADATA: Dict[Variable, str] = {v.entry_metadata: "{ {} }"} + +ENTRY_HARDCODED_VARIABLES: Dict[Variable, str] = { + v.info_json_ext: "info.json", + v.thumbnail_ext: "jpg", +} + +ENTRY_RELATIVE_VARIABLES: Dict[MetadataVariable, str] = { + v.playlist_metadata: _entry_get_map(v.playlist_metadata, {}), + v.source_metadata: _entry_get_map(v.source_metadata, {}), + v.sibling_metadata: _entry_get_array(v.sibling_metadata, []), +} + +ENTRY_REQUIRED_VARIABLES: Dict[MetadataVariable, str] = { + v.uid: _entry_get_str(v.uid), + v.extractor_key: _entry_get_str(v.extractor_key), + v.epoch: _entry_get_int(v.epoch), + v.webpage_url: _entry_get_str(v.webpage_url), + v.ext: _entry_get_str(v.ext), +} + +ENTRY_DEFAULT_VARIABLES: Dict[MetadataVariable, str] = { + v.title: _entry_get_str(v.title, v.uid), + v.extractor: _entry_get_str(v.extractor, v.extractor_key), + v.description: _entry_get_str(v.description, ""), + v.ie_key: _entry_get_str(v.ie_key, v.extractor_key), + v.uploader_id: _entry_get_str(v.uploader_id, v.uid), + v.uploader: _entry_get_str(v.uploader, v.uploader_id), + v.uploader_url: _entry_get_str(v.uploader_url, v.webpage_url), + v.upload_date: _entry_get_str(v.upload_date, v.epoch_date), + v.release_date: _entry_get_str(v.release_date, v.upload_date), + v.channel: _entry_get_str(v.channel, v.uploader), + v.creator: _entry_get_str(v.creator, v.channel), + v.channel_id: _entry_get_str(v.channel_id, v.uploader_id), + v.duration: _entry_get_int(v.duration, 0), + v.playlist_index: _entry_get_int(v.playlist_index, 1), + v.playlist_count: _entry_get_int(v.playlist_count, 1), + v.playlist_uid: _entry_get_str(v.playlist_uid, v.uid), + v.playlist_title: _entry_get_str(v.playlist_title, v.title), + v.playlist_uploader_id: _entry_get_str(v.playlist_uploader_id, v.uploader_id), +} + +# MARK AS UNRESOLVABLE UNTIL THEY ARE ADDED IN THE DOWNLOADER +DOWNLOADER_INJECTED_VARIABLES: Dict[Variable, str] = { + v.download_index: "{%int(1)}", + v.upload_date_index: "{%int(1)}", + v.comments: "{ [] }", + v.requested_subtitles: "{ {} }", + v.chapters: "{ [] }", + v.sponsorblock_chapters: "{ [] }", + v.ytdl_sub_input_url: f"{{{v.source_webpage_url.variable_name}}}", +} + +ENTRY_DERIVED_VARIABLES: Dict[Variable, str] = { + v.uid_sanitized_plex: _sanitized_plex(v.uid), + v.title_sanitized_plex: _sanitized_plex(v.title), + v.epoch_date: f"{{%datetime_strftime({v.epoch.variable_name}, '%Y%m%d')}}", + v.epoch_hour: f"{{%datetime_strftime({v.epoch.variable_name}, '%H')}}", + v.download_index_padded6: _pad_int(v.download_index, 6), + v.upload_date_index_padded: _pad_int(v.upload_date_index, 2), + v.upload_date_index_reversed: f"{{%sub(100, {v.upload_date_index.variable_name})}}", + v.upload_date_index_reversed_padded: _pad_int(v.upload_date_index_reversed, 2), + v.playlist_index_reversed: ( + f"{{%sub({v.playlist_count.variable_name}, {v.playlist_index.variable_name}, -1)}}" + ), + v.playlist_index_padded: _pad_int(v.playlist_index, 2), + v.playlist_index_reversed_padded: _pad_int(v.playlist_index_reversed, 2), + v.playlist_index_padded6: _pad_int(v.playlist_index, 6), + v.playlist_index_reversed_padded6: _pad_int(v.playlist_index_reversed, 6), +} + +ENTRY_UPLOAD_DATE_VARIABLES: Dict[Variable, str] = { + v.upload_year: _date_metadata(v.upload_date, "year"), + v.upload_year_truncated: _date_metadata(v.upload_date, "year_truncated"), + v.upload_year_truncated_reversed: _date_metadata(v.upload_date, "year_truncated_reversed"), + v.upload_month_reversed: _date_metadata(v.upload_date, "month_reversed"), + v.upload_month_reversed_padded: _date_metadata(v.upload_date, "month_reversed_padded"), + v.upload_month_padded: _date_metadata(v.upload_date, "month_padded"), + v.upload_day_padded: _date_metadata(v.upload_date, "day_padded"), + v.upload_month: _date_metadata(v.upload_date, "month"), + v.upload_day: _date_metadata(v.upload_date, "day"), + v.upload_day_reversed: _date_metadata(v.upload_date, "day_reversed"), + v.upload_day_reversed_padded: _date_metadata(v.upload_date, "day_reversed_padded"), + v.upload_day_of_year: _date_metadata(v.upload_date, "day_of_year"), + v.upload_day_of_year_padded: _date_metadata(v.upload_date, "day_of_year_padded"), + v.upload_day_of_year_reversed: _date_metadata(v.upload_date, "day_of_year_reversed"), + v.upload_day_of_year_reversed_padded: _date_metadata( + v.upload_date, "day_of_year_reversed_padded" + ), + v.upload_date_standardized: _date_metadata(v.upload_date, "date_standardized"), +} + +ENTRY_RELEASE_DATE_VARIABLES: Dict[Variable, str] = { + v.release_year: _date_metadata(v.release_date, "year"), + v.release_year_truncated: _date_metadata(v.release_date, "year_truncated"), + v.release_year_truncated_reversed: _date_metadata(v.release_date, "year_truncated_reversed"), + v.release_month_reversed: _date_metadata(v.release_date, "month_reversed"), + v.release_month_reversed_padded: _date_metadata(v.release_date, "month_reversed_padded"), + v.release_month_padded: _date_metadata(v.release_date, "month_padded"), + v.release_day_padded: _date_metadata(v.release_date, "day_padded"), + v.release_month: _date_metadata(v.release_date, "month"), + v.release_day: _date_metadata(v.release_date, "day"), + v.release_day_reversed: _date_metadata(v.release_date, "day_reversed"), + v.release_day_reversed_padded: _date_metadata(v.release_date, "day_reversed_padded"), + v.release_day_of_year: _date_metadata(v.release_date, "day_of_year"), + v.release_day_of_year_padded: _date_metadata(v.release_date, "day_of_year_padded"), + v.release_day_of_year_reversed: _date_metadata(v.release_date, "day_of_year_reversed"), + v.release_day_of_year_reversed_padded: _date_metadata( + v.release_date, "day_of_year_reversed_padded" + ), + v.release_date_standardized: _date_metadata(v.release_date, "date_standardized"), +} + +PLAYLIST_VARIABLES: Dict[Variable, str] = { + v.playlist_webpage_url: _playlist_get_str(v.playlist_webpage_url, v.webpage_url), + v.playlist_description: _playlist_get_str(v.playlist_description, v.description), + v.playlist_uploader: _playlist_get_str(v.playlist_uploader, v.uploader), + v.playlist_uploader_url: _playlist_get_str(v.playlist_uploader_url, v.playlist_webpage_url), + v.source_index: _playlist_get_int(v.source_index, 1), + v.source_count: _playlist_get_int(v.source_count, 1), +} + + +SOURCE_VARIABLES: Dict[Variable, str] = { + v.source_uid: _source_get_str(v.source_uid, v.playlist_uid), + v.source_title: _source_get_str(v.source_title, v.playlist_title), + v.source_webpage_url: _source_get_str(v.source_webpage_url, v.playlist_webpage_url), + v.source_description: _source_get_str(v.source_description, v.playlist_description), + v.source_uploader_id: _source_get_str(v.source_uploader_id, v.playlist_uploader_id), + v.source_uploader: _source_get_str(v.source_uploader, v.playlist_uploader), + v.source_uploader_url: _source_get_str(v.source_uploader_url, v.source_webpage_url), +} + +SOURCE_DERIVED_VARIABLES: Dict[Variable, str] = { + v.source_index_padded: _pad_int(v.source_index, 2), +} + +SIBLING_VARIABLES: Dict[Variable, str] = { + v.playlist_max_upload_date: f"""{{ + %array_reduce( + %if_passthrough( + %extract_field_from_siblings('{v.upload_date.variable_name}'), + [{v.upload_date.variable_name}] + ), + %max + ) + }}""" +} + +SIBLING_DERIVED_VARIABLES: Dict[Variable, str] = { + v.playlist_max_upload_year: _date_metadata(v.playlist_max_upload_date, "year"), + v.playlist_max_upload_year_truncated: _date_metadata( + v.playlist_max_upload_date, "year_truncated" + ), +} + +_VARIABLE_SCRIPTS: Dict[Variable, str] = {} +mergedeep.merge( + _VARIABLE_SCRIPTS, + ENTRY_EMPTY_METADATA, + ENTRY_HARDCODED_VARIABLES, + ENTRY_RELATIVE_VARIABLES, + ENTRY_REQUIRED_VARIABLES, + ENTRY_DEFAULT_VARIABLES, + DOWNLOADER_INJECTED_VARIABLES, + ENTRY_DERIVED_VARIABLES, + ENTRY_UPLOAD_DATE_VARIABLES, + ENTRY_RELEASE_DATE_VARIABLES, + SIBLING_VARIABLES, + SIBLING_DERIVED_VARIABLES, + PLAYLIST_VARIABLES, + SOURCE_VARIABLES, + SOURCE_DERIVED_VARIABLES, +) + +VARIABLE_SCRIPTS: Dict[str, str] = { + var.variable_name: script for var, script in _VARIABLE_SCRIPTS.items() +} + + +def _keys(*variables: Dict[Variable, str]) -> Set[str]: + keys: Set[str] = set() + for variable_set in variables: + keys.update(set(var.variable_name for var in variable_set.keys())) + return keys + + +UNRESOLVED_VARIABLES: Set[str] = _keys( + ENTRY_EMPTY_METADATA, + DOWNLOADER_INJECTED_VARIABLES, +) + +CustomFunctions.register() diff --git a/src/ytdl_sub/entries/variables/entry_variables.py b/src/ytdl_sub/entries/variables/entry_variables.py deleted file mode 100644 index 8405b060..00000000 --- a/src/ytdl_sub/entries/variables/entry_variables.py +++ /dev/null @@ -1,857 +0,0 @@ -from datetime import datetime -from typing import Union - -from yt_dlp.utils import sanitize_filename - -from ytdl_sub.entries.base_entry import BaseEntry -from ytdl_sub.entries.base_entry import BaseEntryVariables -from ytdl_sub.entries.variables.kwargs import CHANNEL -from ytdl_sub.entries.variables.kwargs import CHANNEL_ID -from ytdl_sub.entries.variables.kwargs import CREATOR -from ytdl_sub.entries.variables.kwargs import DOWNLOAD_INDEX -from ytdl_sub.entries.variables.kwargs import EXT -from ytdl_sub.entries.variables.kwargs import PLAYLIST_COUNT -from ytdl_sub.entries.variables.kwargs import PLAYLIST_DESCRIPTION -from ytdl_sub.entries.variables.kwargs import PLAYLIST_INDEX -from ytdl_sub.entries.variables.kwargs import PLAYLIST_MAX_UPLOAD_YEAR -from ytdl_sub.entries.variables.kwargs import PLAYLIST_MAX_UPLOAD_YEAR_TRUNCATED -from ytdl_sub.entries.variables.kwargs import PLAYLIST_TITLE -from ytdl_sub.entries.variables.kwargs import PLAYLIST_UID -from ytdl_sub.entries.variables.kwargs import PLAYLIST_UPLOADER -from ytdl_sub.entries.variables.kwargs import PLAYLIST_UPLOADER_ID -from ytdl_sub.entries.variables.kwargs import PLAYLIST_UPLOADER_URL -from ytdl_sub.entries.variables.kwargs import PLAYLIST_WEBPAGE_URL -from ytdl_sub.entries.variables.kwargs import RELEASE_DATE -from ytdl_sub.entries.variables.kwargs import SOURCE_COUNT -from ytdl_sub.entries.variables.kwargs import SOURCE_DESCRIPTION -from ytdl_sub.entries.variables.kwargs import SOURCE_INDEX -from ytdl_sub.entries.variables.kwargs import SOURCE_TITLE -from ytdl_sub.entries.variables.kwargs import SOURCE_UID -from ytdl_sub.entries.variables.kwargs import SOURCE_UPLOADER -from ytdl_sub.entries.variables.kwargs import SOURCE_UPLOADER_ID -from ytdl_sub.entries.variables.kwargs import SOURCE_UPLOADER_URL -from ytdl_sub.entries.variables.kwargs import SOURCE_WEBPAGE_URL -from ytdl_sub.entries.variables.kwargs import UPLOAD_DATE -from ytdl_sub.entries.variables.kwargs import UPLOAD_DATE_INDEX - -# This file contains mixins to a BaseEntry subclass. Ignore pylint's "no kwargs member" suggestion -# pylint: disable=no-member -# pylint: disable=too-many-public-methods - - -def pad(num: int, width: int = 2): - """Pad integers""" - return str(num).zfill(width) - - -_days_in_month = [0, 31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31] - -Self = Union[BaseEntry, "EntryVariables"] - - -class EntryVariables(BaseEntryVariables): - @property - def source_title(self: Self) -> str: - """ - Returns - ------- - str - Name of the source (i.e. channel with multiple playlists) if it exists, otherwise - returns its playlist_title. - """ - return self.kwargs_get(SOURCE_TITLE, self.playlist_title) - - @property - def source_title_sanitized(self: Self) -> str: - """ - Returns - ------- - str - The source title, sanitized - """ - return sanitize_filename(self.source_title) - - @property - def source_uid(self: Self) -> str: - """ - Returns - ------- - str - The source unique id if it exists, otherwise returns the playlist unique ID. - """ - return self.kwargs_get(SOURCE_UID, self.playlist_uid) - - @property - def source_index(self: Self) -> int: - """ - Returns - ------- - int - Source index if it exists, otherwise returns ``1``. - - It is recommended to not use this unless you know the source will never add new content - (it is easy for this value to change). - """ - return self.kwargs_get(SOURCE_INDEX, self.playlist_index) - - @property - def source_index_padded(self: Self) -> str: - """ - Returns - ------- - int - The source index, padded. - """ - return pad(self.source_index, 2) - - @property - def source_count(self: Self) -> int: - """ - Returns - ------- - int - The source count if it exists, otherwise returns the playlist count. - """ - return self.kwargs_get(SOURCE_COUNT, self.playlist_count) - - @property - def source_webpage_url(self: Self) -> str: - """ - Returns - ------- - str - The source webpage url if it exists, otherwise returns the playlist webpage url. - """ - return self.kwargs_get(SOURCE_WEBPAGE_URL, self.playlist_webpage_url) - - @property - def source_description(self: Self) -> str: - """ - Returns - ------- - str - The source description if it exists, otherwise returns the playlist description. - """ - return self.kwargs_get(SOURCE_DESCRIPTION, self.playlist_description) - - @property - def playlist_uid(self: Self) -> str: - """ - Returns - ------- - str - The playlist unique ID if it exists, otherwise return the entry unique ID. - """ - return self.kwargs_get(PLAYLIST_UID, self.uid) - - @property - def playlist_title(self: Self) -> str: - """ - Returns - ------- - str - Name of its parent playlist/channel if it exists, otherwise returns its title. - """ - return self.kwargs_get(PLAYLIST_TITLE, self.title) - - @property - def playlist_title_sanitized(self: Self) -> str: - """ - Returns - ------- - str - The playlist name, sanitized - """ - return sanitize_filename(self.playlist_title) - - @property - def playlist_index(self: Self) -> int: - """ - Returns - ------- - int - Playlist index if it exists, otherwise returns ``1``. - - Note that for channels/playlists, any change (i.e. adding or removing a video) will make - this value change. Use with caution. - """ - return self.kwargs_get(PLAYLIST_INDEX, 1) - - @property - def playlist_index_reversed(self: Self) -> int: - """ - Returns - ------- - int - Playlist index reversed via ``playlist_count - playlist_index + 1`` - """ - return self.playlist_count - self.playlist_index + 1 - - @property - def playlist_index_padded(self: Self) -> str: - """ - Returns - ------- - str - playlist_index padded two digits - """ - return pad(self.playlist_index, width=2) - - @property - def playlist_index_reversed_padded(self: Self) -> str: - """ - Returns - ------- - str - playlist_index_reversed padded two digits - """ - return pad(self.playlist_index_reversed, width=2) - - @property - def playlist_index_padded6(self: Self) -> str: - """ - Returns - ------- - str - playlist_index padded six digits. - """ - return pad(self.playlist_index, width=6) - - @property - def playlist_index_reversed_padded6(self: Self) -> str: - """ - Returns - ------- - str - playlist_index_reversed padded six digits. - """ - return pad(self.playlist_index_reversed, width=6) - - @property - def playlist_count(self: Self) -> int: - """ - Returns - ------- - int - Playlist count if it exists, otherwise returns ``1``. - - Note that for channels/playlists, any change (i.e. adding or removing a video) will make - this value change. Use with caution. - """ - return self.kwargs_get(PLAYLIST_COUNT, 1) - - @property - def playlist_description(self: Self) -> str: - """ - Returns - ------- - str - The playlist description if it exists, otherwise returns the entry's description. - """ - return self.kwargs_get(PLAYLIST_DESCRIPTION, self.description) - - @property - def playlist_webpage_url(self: Self) -> str: - """ - Returns - ------- - str - The playlist webpage url if it exists. Otherwise, returns the entry webpage url. - """ - return self.kwargs_get(PLAYLIST_WEBPAGE_URL, self.webpage_url) - - @property - def playlist_max_upload_year(self: Self) -> int: - """ - Returns - ------- - int - Max upload_year for all entries in this entry's playlist if it exists, otherwise returns - ``upload_year`` - """ - # override in EntryParent - return self.kwargs_get(PLAYLIST_MAX_UPLOAD_YEAR, self.upload_year) - - @property - def playlist_max_upload_year_truncated(self: Self) -> int: - """ - Returns - ------- - int - The max playlist truncated upload year for all entries in this entry's playlist if it - exists, otherwise returns ``upload_year_truncated``. - """ - return self.kwargs_get(PLAYLIST_MAX_UPLOAD_YEAR_TRUNCATED, self.upload_year_truncated) - - @property - def playlist_uploader_id(self: Self) -> str: - """ - Returns - ------- - str - The playlist uploader id if it exists, otherwise returns the entry uploader ID. - """ - return self.kwargs_get(PLAYLIST_UPLOADER_ID, self.uploader_id) - - @property - def playlist_uploader(self: Self) -> str: - """ - Returns - ------- - str - The playlist uploader if it exists, otherwise return the entry uploader. - """ - return self.kwargs_get(PLAYLIST_UPLOADER, self.uploader) - - @property - def playlist_uploader_sanitized(self: Self) -> str: - """ - Returns - ------- - str - The playlist uploader, sanitized. - """ - return sanitize_filename(self.playlist_uploader) - - @property - def playlist_uploader_url(self: Self) -> str: - """ - Returns - ------- - str - The playlist uploader url if it exists, otherwise returns the playlist webpage_url. - """ - return self.kwargs_get(PLAYLIST_UPLOADER_URL, self.playlist_webpage_url) - - @property - def source_uploader_id(self: Self) -> str: - """ - Returns - ------- - str - The source uploader id if it exists, otherwise returns the playlist_uploader_id - """ - return self.kwargs_get(SOURCE_UPLOADER_ID, self.playlist_uploader_id) - - @property - def source_uploader(self: Self) -> str: - """ - Returns - ------- - str - The source uploader if it exists, otherwise return the playlist_uploader - """ - return self.kwargs_get(SOURCE_UPLOADER, self.playlist_uploader) - - @property - def source_uploader_url(self: Self) -> str: - """ - Returns - ------- - str - The source uploader url if it exists, otherwise returns the source webpage_url. - """ - return self.kwargs_get(SOURCE_UPLOADER_URL, self.source_webpage_url) - - @property - def creator(self: Self) -> str: - """ - Returns - ------- - str - The creator name if it exists, otherwise returns the channel. - """ - return self.kwargs_get(CREATOR, self.channel) - - @property - def creator_sanitized(self: Self) -> str: - """ - Returns - ------- - str - The creator name, sanitized - """ - return sanitize_filename(self.creator) - - @property - def channel(self: Self) -> str: - """ - Returns - ------- - str - The channel name if it exists, otherwise returns the uploader. - """ - return self.kwargs_get(CHANNEL, self.uploader) - - @property - def channel_sanitized(self: Self) -> str: - """ - Returns - ------- - str - The channel name, sanitized. - """ - return sanitize_filename(self.channel) - - @property - def channel_id(self: Self) -> str: - """ - Returns - ------- - str - The channel id if it exists, otherwise returns the entry uploader ID. - """ - return self.kwargs_get(CHANNEL_ID, self.uploader_id) - - @property - def ext(self: Self) -> str: - """ - Returns - ------- - str - The downloaded entry's file extension - """ - return self.kwargs(EXT) - - @property - def thumbnail_ext(self: Self) -> str: - """ - Returns - ------- - str - The download entry's thumbnail extension. Will always return 'jpg'. Until there is a - need to support other image types, we always convert to jpg. - """ - return "jpg" - - @property - def download_index(self: Self) -> int: - """ - Returns - ------- - int - The i'th entry downloaded. NOTE that this is fetched dynamically from the download - archive. - """ - return self.kwargs_get(DOWNLOAD_INDEX, 0) + 1 - - @property - def download_index_padded6(self: Self) -> str: - """ - Returns - ------- - str - The download_index padded six digits - """ - return pad(self.download_index, 6) - - @property - def upload_date_index(self: Self) -> int: - """ - Returns - ------- - int - The i'th entry downloaded with this upload date. - """ - return self.kwargs_get(UPLOAD_DATE_INDEX, 0) + 1 - - @property - def upload_date_index_padded(self: Self) -> str: - """ - Returns - ------- - int - The upload_date_index padded two digits - """ - return pad(self.upload_date_index, 2) - - @property - def upload_date_index_reversed(self: Self) -> int: - """ - Returns - ------- - int - 100 - upload_date_index - """ - return 100 - self.upload_date_index - - @property - def upload_date_index_reversed_padded(self: Self) -> str: - """ - Returns - ------- - int - The upload_date_index padded two digits - """ - return pad(self.upload_date_index_reversed, 2) - - @property - def upload_date(self: Self) -> str: - """ - Returns - ------- - str - The entry's uploaded date, in YYYYMMDD format. If not present, return today's date. - """ - return self.kwargs_get(UPLOAD_DATE, datetime.now().strftime("%Y%m%d")) - - @property - def upload_year(self: Self) -> int: - """ - Returns - ------- - int - The entry's upload year - """ - return int(self.upload_date[:4]) - - @property - def upload_year_truncated(self: Self) -> int: - """ - Returns - ------- - int - The last two digits of the upload year, i.e. 22 in 2022 - """ - return int(str(self.upload_year)[-2:]) - - @property - def upload_year_truncated_reversed(self: Self) -> int: - """ - Returns - ------- - int - The upload year truncated, but reversed using ``100 - {upload_year_truncated}``, i.e. - 2022 returns ``100 - 22`` = ``78`` - """ - return 100 - self.upload_year_truncated - - @property - def upload_month_reversed(self: Self) -> int: - """ - Returns - ------- - int - The upload month, but reversed using ``13 - {upload_month}``, i.e. March returns ``10`` - """ - return 13 - self.upload_month - - @property - def upload_month_reversed_padded(self: Self) -> str: - """ - Returns - ------- - str - The reversed upload month, but padded. i.e. November returns "02" - """ - return pad(self.upload_month_reversed) - - @property - def upload_month_padded(self: Self) -> str: - """ - Returns - ------- - str - The entry's upload month padded to two digits, i.e. March returns "03" - """ - return self.upload_date[4:6] - - @property - def upload_day_padded(self: Self) -> str: - """ - Returns - ------- - str - The entry's upload day padded to two digits, i.e. the fifth returns "05" - """ - return self.upload_date[6:8] - - @property - def upload_month(self: Self) -> int: - """ - Returns - ------- - int - The upload month as an integer (no padding). - """ - return int(self.upload_month_padded.lstrip("0")) - - @property - def upload_day(self: Self) -> int: - """ - Returns - ------- - int - The upload day as an integer (no padding). - """ - return int(self.upload_day_padded.lstrip("0")) - - @property - def upload_day_reversed(self: Self) -> int: - """ - Returns - ------- - int - The upload day, but reversed using ``{total_days_in_month} + 1 - {upload_day}``, - i.e. August 8th would have upload_day_reversed of ``31 + 1 - 8`` = ``24`` - """ - total_days_in_month = _days_in_month[self.upload_month] - if self.upload_month == 2 and self.upload_year % 4 == 0: # leap year - total_days_in_month += 1 - - return total_days_in_month + 1 - self.upload_day - - @property - def upload_day_reversed_padded(self: Self) -> str: - """ - Returns - ------- - str - The reversed upload day, but padded. i.e. August 30th returns "02". - """ - return pad(self.upload_day_reversed) - - @property - def upload_day_of_year(self: Self) -> int: - """ - Returns - ------- - int - The day of the year, i.e. February 1st returns ``32`` - """ - output = sum(_days_in_month[: self.upload_month]) + self.upload_day - if self.upload_month > 2 and self.upload_year % 4 == 0: - output += 1 - - return output - - @property - def upload_day_of_year_padded(self: Self) -> str: - """ - Returns - ------- - str - The upload day of year, but padded i.e. February 1st returns "032" - """ - return pad(self.upload_day_of_year, width=3) - - @property - def upload_day_of_year_reversed(self: Self) -> int: - """ - Returns - ------- - int - The upload day, but reversed using ``{total_days_in_year} + 1 - {upload_day}``, - i.e. February 2nd would have upload_day_of_year_reversed of ``365 + 1 - 32`` = ``334`` - """ - total_days_in_year = 365 - if self.upload_year % 4 == 0: - total_days_in_year += 1 - - return total_days_in_year + 1 - self.upload_day_of_year - - @property - def upload_day_of_year_reversed_padded(self: Self) -> str: - """ - Returns - ------- - str - The reversed upload day of year, but padded i.e. December 31st returns "001" - """ - return pad(self.upload_day_of_year_reversed, width=3) - - @property - def upload_date_standardized(self: Self) -> str: - """ - Returns - ------- - str - The uploaded date formatted as YYYY-MM-DD - """ - return f"{self.upload_year}-{self.upload_month_padded}-{self.upload_day_padded}" - - @property - def release_date(self: Self) -> str: - """ - Returns - ------- - str - The entry's release date, in YYYYMMDD format. If not present, return the upload date. - """ - return self.kwargs_get(RELEASE_DATE, self.upload_date) - - @property - def release_year(self: Self) -> int: - """ - Returns - ------- - int - The entry's release year - """ - return int(self.release_date[:4]) - - @property - def release_year_truncated(self: Self) -> int: - """ - Returns - ------- - int - The last two digits of the release year, i.e. 22 in 2022 - """ - return int(str(self.release_year)[-2:]) - - @property - def release_year_truncated_reversed(self: Self) -> int: - """ - Returns - ------- - int - The release year truncated, but reversed using ``100 - {release_year_truncated}``, i.e. - 2022 returns ``100 - 22`` = ``78`` - """ - return 100 - self.release_year_truncated - - @property - def release_month_reversed(self: Self) -> int: - """ - Returns - ------- - int - The release month, but reversed - using ``13 - {release_month}``, i.e. March returns ``10`` - """ - return 13 - self.release_month - - @property - def release_month_reversed_padded(self: Self) -> str: - """ - Returns - ------- - str - The reversed release month, but padded. i.e. November returns "02" - """ - return pad(self.release_month_reversed) - - @property - def release_month_padded(self: Self) -> str: - """ - Returns - ------- - str - The entry's release month padded to two digits, i.e. March returns "03" - """ - return self.release_date[4:6] - - @property - def release_day_padded(self: Self) -> str: - """ - Returns - ------- - str - The entry's release day padded to two digits, i.e. the fifth returns "05" - """ - return self.release_date[6:8] - - @property - def release_month(self: Self) -> int: - """ - Returns - ------- - int - The release month as an integer (no padding). - """ - return int(self.release_month_padded.lstrip("0")) - - @property - def release_day(self: Self) -> int: - """ - Returns - ------- - int - The release day as an integer (no padding). - """ - return int(self.release_day_padded.lstrip("0")) - - @property - def release_day_reversed(self: Self) -> int: - """ - Returns - ------- - int - The release day, but reversed using ``{total_days_in_month} + 1 - {release_day}``, - i.e. August 8th would have release_day_reversed of ``31 + 1 - 8`` = ``24`` - """ - total_days_in_month = _days_in_month[self.release_month] - if self.release_month == 2 and self.release_year % 4 == 0: # leap year - total_days_in_month += 1 - - return total_days_in_month + 1 - self.release_day - - @property - def release_day_reversed_padded(self: Self) -> str: - """ - Returns - ------- - str - The reversed release day, but padded. i.e. August 30th returns "02". - """ - return pad(self.release_day_reversed) - - @property - def release_day_of_year(self: Self) -> int: - """ - Returns - ------- - int - The day of the year, i.e. February 1st returns ``32`` - """ - output = sum(_days_in_month[: self.release_month]) + self.release_day - if self.release_month > 2 and self.release_year % 4 == 0: - output += 1 - - return output - - @property - def release_day_of_year_padded(self: Self) -> str: - """ - Returns - ------- - str - The release day of year, but padded i.e. February 1st returns "032" - """ - return pad(self.release_day_of_year, width=3) - - @property - def release_day_of_year_reversed(self: Self) -> int: - """ - Returns - ------- - int - The release day, but reversed using ``{total_days_in_year} + 1 - {release_day}``, - i.e. February 2nd would have release_day_of_year_reversed of ``365 + 1 - 32`` = ``334`` - """ - total_days_in_year = 365 - if self.release_year % 4 == 0: - total_days_in_year += 1 - - return total_days_in_year + 1 - self.release_day_of_year - - @property - def release_day_of_year_reversed_padded(self: Self) -> str: - """ - Returns - ------- - str - The reversed release day of year, but padded i.e. December 31st returns "001" - """ - return pad(self.release_day_of_year_reversed, width=3) - - @property - def release_date_standardized(self: Self) -> str: - """ - Returns - ------- - str - The release date formatted as YYYY-MM-DD - """ - return f"{self.release_year}-{self.release_month_padded}-{self.release_day_padded}" diff --git a/src/ytdl_sub/entries/variables/kwargs.py b/src/ytdl_sub/entries/variables/kwargs.py deleted file mode 100644 index 84f729f9..00000000 --- a/src/ytdl_sub/entries/variables/kwargs.py +++ /dev/null @@ -1,68 +0,0 @@ -from typing import List - - -class KwargKeys: - keys: List[str] = [] - backend_keys: List[str] = [] - - -def _(key: str, backend: bool = False) -> str: - if backend: - assert key not in KwargKeys.backend_keys - KwargKeys.backend_keys.append(key) - else: - assert key not in KwargKeys.keys - KwargKeys.keys.append(key) - return key - - -SOURCE_ENTRY = _("source_entry", backend=True) -SOURCE_INDEX = _("source_index") -SOURCE_COUNT = _("source_count") -SOURCE_TITLE = _("source_title") -SOURCE_UID = _("source_uid") -SOURCE_DESCRIPTION = _("source_description") -SOURCE_WEBPAGE_URL = _("source_webpage_url") -SOURCE_UPLOADER = _("source_uploader") -SOURCE_UPLOADER_ID = _("source_uploader_id") -SOURCE_UPLOADER_URL = _("source_uploader_url") - -PLAYLIST_ENTRY = _("playlist_entry", backend=True) -PLAYLIST_WEBPAGE_URL = _("playlist_webpage_url") -PLAYLIST_INDEX = _("playlist_index") -PLAYLIST_COUNT = _("playlist_count") -PLAYLIST_MAX_UPLOAD_YEAR = _("playlist_max_upload_year") -PLAYLIST_MAX_UPLOAD_YEAR_TRUNCATED = _("playlist_max_upload_year_truncated") -PLAYLIST_TITLE = _("playlist_title") -PLAYLIST_DESCRIPTION = _("playlist_description") -PLAYLIST_UID = _("playlist_uid") -PLAYLIST_UPLOADER = _("playlist_uploader") -PLAYLIST_UPLOADER_ID = _("playlist_uploader_id") -PLAYLIST_UPLOADER_URL = _("playlist_uploader_url") - -COLLECTION_URL = _("collection_url", backend=True) -DOWNLOAD_INDEX = _("download_index", backend=True) -UPLOAD_DATE_INDEX = _("upload_date_index", backend=True) -REQUESTED_SUBTITLES = _("requested_subtitles", backend=True) -CHAPTERS = _("chapters", backend=True) -YTDL_SUB_CUSTOM_CHAPTERS = _("ytdl_sub_custom_chapters", backend=True) -YTDL_SUB_REGEX_SOURCE_VARS = _("ytdl_sub_regex_source_vars", backend=True) -SPONSORBLOCK_CHAPTERS = _("sponsorblock_chapters", backend=True) -SPLIT_BY_CHAPTERS_PARENT_ENTRY = _("split_by_chapters_parent_entry", backend=True) -COMMENTS = _("comments", backend=True) -UID = _("id") -EXTRACTOR = _("extractor") -IE_KEY = _("ie_key") -EPOCH = _("epoch") -CHANNEL = _("channel") -CHANNEL_ID = _("channel_id") -CREATOR = _("creator") -EXT = _("ext") -TITLE = _("title") -DESCRIPTION = _("description") -WEBPAGE_URL = _("webpage_url") -RELEASE_DATE = _("release_date") -UPLOAD_DATE = _("upload_date") -UPLOADER = _("uploader") -UPLOADER_ID = _("uploader_id") -UPLOADER_URL = _("uploader_url") diff --git a/src/ytdl_sub/entries/variables/override_variables.py b/src/ytdl_sub/entries/variables/override_variables.py index 8a885f83..b16bf68a 100644 --- a/src/ytdl_sub/entries/variables/override_variables.py +++ b/src/ytdl_sub/entries/variables/override_variables.py @@ -1,5 +1,11 @@ +from ytdl_sub.entries.script.function_scripts import CUSTOM_FUNCTION_SCRIPTS +from ytdl_sub.entries.script.variable_scripts import VARIABLE_SCRIPTS +from ytdl_sub.script.functions import Functions + SUBSCRIPTION_NAME = "subscription_name" SUBSCRIPTION_VALUE = "subscription_value" +SUBSCRIPTION_MAP = "subscription_map" +SUBSCRIPTION_ARRAY = "subscription_array" class OverrideVariables: @@ -55,3 +61,23 @@ class OverrideVariables: ``subscription_value``. """ return f"subscription_value_{index + 1}" + + @classmethod + def is_entry_variable_name(cls, name: str) -> bool: + """ + Returns + ------- + True if the name is an entry variable name. False otherwise. + """ + return name in VARIABLE_SCRIPTS + + @classmethod + def is_function_name(cls, name: str) -> bool: + """ + Returns + ------- + True if the name is a function name (either built-in or script). False otherwise. + """ + if name.startswith("%"): + return name in CUSTOM_FUNCTION_SCRIPTS or Functions.is_built_in(name[1:]) + return False diff --git a/src/ytdl_sub/plugins/audio_extract.py b/src/ytdl_sub/plugins/audio_extract.py index 750429de..e728f392 100644 --- a/src/ytdl_sub/plugins/audio_extract.py +++ b/src/ytdl_sub/plugins/audio_extract.py @@ -2,11 +2,15 @@ import os.path from typing import Any from typing import Dict from typing import Optional +from typing import Set -from ytdl_sub.config.plugin import Plugin -from ytdl_sub.config.preset_options import OptionsDictValidator +from ytdl_sub.config.plugin.plugin import Plugin +from ytdl_sub.config.plugin.plugin_operation import PluginOperation +from ytdl_sub.config.validators.options import OptionsDictValidator from ytdl_sub.downloaders.ytdl_options_builder import YTDLOptionsBuilder from ytdl_sub.entries.entry import Entry +from ytdl_sub.entries.script.variable_definitions import VARIABLES +from ytdl_sub.entries.script.variable_definitions import VariableDefinitions from ytdl_sub.utils.exceptions import FileNotDownloadedException from ytdl_sub.utils.file_handler import FileMetadata from ytdl_sub.validators.audo_codec_validator import AUDIO_CODEC_EXTS @@ -14,6 +18,8 @@ from ytdl_sub.validators.audo_codec_validator import AUDIO_CODEC_TYPES_EXTENSION from ytdl_sub.validators.audo_codec_validator import AudioTypeValidator from ytdl_sub.validators.validators import FloatValidator +v: VariableDefinitions = VARIABLES + class AudioExtractOptions(OptionsDictValidator): """ @@ -65,6 +71,12 @@ class AudioExtractOptions(OptionsDictValidator): return self._quality.value return None + def modified_variables(self) -> Dict[PluginOperation, Set[str]]: + """ + Possibly changes ``ext``, so do not resolve until this has run + """ + return {PluginOperation.MODIFY_ENTRY: {v.ext.variable_name}} + class AudioExtractPlugin(Plugin[AudioExtractOptions]): plugin_options_type = AudioExtractOptions @@ -125,7 +137,7 @@ class AudioExtractPlugin(Plugin[AudioExtractOptions]): new_ext = AUDIO_CODEC_TYPES_EXTENSION_MAPPING[self.plugin_options.codec] extracted_audio_file = entry.get_download_file_path().removesuffix(entry.ext) + new_ext - entry.add_kwargs({"ext": new_ext}) + entry.add({v.ext: new_ext}) if not self.is_dry_run: if not os.path.isfile(extracted_audio_file): diff --git a/src/ytdl_sub/plugins/chapters.py b/src/ytdl_sub/plugins/chapters.py index 456994f5..bbb10ddd 100644 --- a/src/ytdl_sub/plugins/chapters.py +++ b/src/ytdl_sub/plugins/chapters.py @@ -5,12 +5,15 @@ from typing import List from typing import Optional from typing import Set -from ytdl_sub.config.plugin import Plugin -from ytdl_sub.config.preset_options import OptionsDictValidator +from ytdl_sub.config.plugin.plugin import Plugin +from ytdl_sub.config.plugin.plugin_operation import PluginOperation +from ytdl_sub.config.validators.options import OptionsDictValidator from ytdl_sub.downloaders.ytdl_options_builder import YTDLOptionsBuilder from ytdl_sub.entries.entry import Entry -from ytdl_sub.entries.variables.kwargs import COMMENTS -from ytdl_sub.entries.variables.kwargs import YTDL_SUB_CUSTOM_CHAPTERS +from ytdl_sub.entries.entry import ytdl_sub_chapters_from_comments +from ytdl_sub.entries.entry import ytdl_sub_split_by_chapters_parent_uid +from ytdl_sub.entries.script.variable_definitions import VARIABLES +from ytdl_sub.entries.script.variable_definitions import VariableDefinitions from ytdl_sub.utils.chapters import Chapters from ytdl_sub.utils.ffmpeg import set_ffmpeg_metadata_chapters from ytdl_sub.utils.file_handler import FileMetadata @@ -19,6 +22,8 @@ from ytdl_sub.validators.string_select_validator import StringSelectValidator from ytdl_sub.validators.validators import BoolValidator from ytdl_sub.validators.validators import ListValidator +v: VariableDefinitions = VARIABLES + SPONSORBLOCK_HIGHLIGHT_CATEGORIES: Set[str] = {"poi_highlight"} SPONSORBLOCK_CATEGORIES: Set[str] = SPONSORBLOCK_HIGHLIGHT_CATEGORIES | { "sponsor", @@ -33,15 +38,11 @@ SPONSORBLOCK_CATEGORIES: Set[str] = SPONSORBLOCK_HIGHLIGHT_CATEGORIES | { def _chapters(entry: Entry) -> List[Dict]: - if entry.kwargs_contains("chapters"): - return entry.kwargs("chapters") or [] - return [] + return entry.get(v.chapters, list) def _sponsorblock_chapters(entry: Entry) -> List[Dict]: - if entry.kwargs_contains("sponsorblock_chapters"): - return entry.kwargs("sponsorblock_chapters") or [] - return [] + return entry.get(v.sponsorblock_chapters, list) def _contains_any_chapters(entry: Entry) -> bool: @@ -195,6 +196,14 @@ class ChaptersOptions(OptionsDictValidator): """ return self._force_key_frames + def added_variables( + self, + resolved_variables: Set[str], + unresolved_variables: Set[str], + plugin_op: PluginOperation, + ) -> Dict[PluginOperation, Set[str]]: + return {PluginOperation.MODIFY_ENTRY: {ytdl_sub_chapters_from_comments.variable_name}} + class ChaptersPlugin(Plugin[ChaptersOptions]): plugin_options_type = ChaptersOptions @@ -300,27 +309,33 @@ class ChaptersPlugin(Plugin[ChaptersOptions]): ------- entry """ - chapters = Chapters.from_empty() + has_chapters_from_comments = False # If there are no embedded chapters, and comment chapters are allowed... if not _contains_any_chapters(entry) and self.plugin_options.allow_chapters_from_comments: + chapters = Chapters.from_empty() + # Try to get chapters from comments - for comment in entry.kwargs_get(COMMENTS, []): + for comment in entry.get(v.comments, list): chapters = Chapters.from_string(comment.get("text", "")) if chapters.contains_any_chapters(): break # If some are actually found, add a special kwarg and embed them if chapters.contains_any_chapters(): - entry.add_kwargs({YTDL_SUB_CUSTOM_CHAPTERS: chapters.to_file_metadata_dict()}) + has_chapters_from_comments = True + entry.add({ytdl_sub_chapters_from_comments: chapters.to_yt_dlp_chapter_metadata()}) if not self.is_dry_run: set_ffmpeg_metadata_chapters( file_path=entry.get_download_file_path(), chapters=chapters, - file_duration_sec=entry.kwargs("duration"), + file_duration_sec=entry.get(v.duration, int), ) + if not has_chapters_from_comments: + entry.add({ytdl_sub_chapters_from_comments: []}) + return entry def post_process_entry(self, entry: Entry) -> Optional[FileMetadata]: @@ -334,12 +349,9 @@ class ChaptersPlugin(Plugin[ChaptersOptions]): ------- FileMetadata outlining which chapters/SponsorBlock segments got removed """ - if custom_chapters_metadata := entry.kwargs_get(YTDL_SUB_CUSTOM_CHAPTERS): - title: str = "Chapters from comments" - return FileMetadata.from_dict( - value_dict=custom_chapters_metadata, - title=title, - sort_dict=False, # timestamps + titles are already sorted + if custom_chapters := entry.get(ytdl_sub_chapters_from_comments, list): + return Chapters.from_yt_dlp_chapters(custom_chapters).to_file_metadata( + title="Chapters from comments" ) if self.plugin_options.embed_chapters: @@ -356,9 +368,10 @@ class ChaptersPlugin(Plugin[ChaptersOptions]): if removed_sponsorblock: metadata_dict["Removed SponsorBlock Category Count(s)"] = removed_sponsorblock - # TODO: check if file actually has embedded chapters - return FileMetadata.from_dict( - value_dict=metadata_dict, title="Embedded Chapters", sort_dict=False - ) + # If the entry wasn't split on embedded chapters, report it in the file metadata + if not entry.try_get(ytdl_sub_split_by_chapters_parent_uid, str): + return FileMetadata.from_dict( + value_dict=metadata_dict, title="Embedded Chapters", sort_dict=False + ) return None diff --git a/src/ytdl_sub/plugins/date_range.py b/src/ytdl_sub/plugins/date_range.py index 1d93a483..092d1054 100644 --- a/src/ytdl_sub/plugins/date_range.py +++ b/src/ytdl_sub/plugins/date_range.py @@ -2,8 +2,8 @@ from typing import List from typing import Optional from typing import Tuple -from ytdl_sub.config.plugin import Plugin -from ytdl_sub.config.preset_options import OptionsDictValidator +from ytdl_sub.config.plugin.plugin import Plugin +from ytdl_sub.config.validators.options import OptionsDictValidator from ytdl_sub.utils.datetime import to_date_str from ytdl_sub.validators.string_datetime import StringDatetimeValidator diff --git a/src/ytdl_sub/plugins/embed_thumbnail.py b/src/ytdl_sub/plugins/embed_thumbnail.py index 4886f707..64703870 100644 --- a/src/ytdl_sub/plugins/embed_thumbnail.py +++ b/src/ytdl_sub/plugins/embed_thumbnail.py @@ -3,9 +3,8 @@ from typing import Optional import mediafile -from ytdl_sub.config.plugin import Plugin -from ytdl_sub.config.plugin import PluginPriority -from ytdl_sub.config.preset_options import OptionsValidator +from ytdl_sub.config.plugin.plugin import Plugin +from ytdl_sub.config.validators.options import OptionsValidator from ytdl_sub.entries.entry import Entry from ytdl_sub.utils.ffmpeg import FFMPEG from ytdl_sub.utils.file_handler import FileHandler @@ -33,7 +32,6 @@ class EmbedThumbnailOptions(BoolValidator, OptionsValidator): class EmbedThumbnailPlugin(Plugin[EmbedThumbnailOptions]): plugin_options_type = EmbedThumbnailOptions - priority = PluginPriority(post_process=PluginPriority.POST_PROCESS_AFTER_FILE_CONVERT) @property def _embed_thumbnail(self) -> bool: diff --git a/src/ytdl_sub/plugins/file_convert.py b/src/ytdl_sub/plugins/file_convert.py index f99b374d..3b56e1c1 100644 --- a/src/ytdl_sub/plugins/file_convert.py +++ b/src/ytdl_sub/plugins/file_convert.py @@ -2,12 +2,15 @@ import os from typing import Any from typing import Dict from typing import Optional +from typing import Set -from ytdl_sub.config.plugin import Plugin -from ytdl_sub.config.plugin import PluginPriority -from ytdl_sub.config.preset_options import OptionsDictValidator +from ytdl_sub.config.overrides import Overrides +from ytdl_sub.config.plugin.plugin import Plugin +from ytdl_sub.config.plugin.plugin_operation import PluginOperation +from ytdl_sub.config.validators.options import OptionsDictValidator from ytdl_sub.entries.entry import Entry -from ytdl_sub.entries.variables.kwargs import EXT +from ytdl_sub.entries.script.variable_definitions import VARIABLES +from ytdl_sub.entries.script.variable_definitions import VariableDefinitions from ytdl_sub.utils.exceptions import FileNotDownloadedException from ytdl_sub.utils.exceptions import ValidationException from ytdl_sub.utils.ffmpeg import FFMPEG @@ -16,6 +19,9 @@ from ytdl_sub.utils.file_handler import FileMetadata from ytdl_sub.validators.audo_codec_validator import FileTypeValidator from ytdl_sub.validators.string_formatter_validators import OverridesStringFormatterValidator from ytdl_sub.validators.string_select_validator import StringSelectValidator +from ytdl_sub.ytdl_additions.enhanced_download_archive import EnhancedDownloadArchive + +v: VariableDefinitions = VARIABLES class FileConvertWithValidator(StringSelectValidator): @@ -115,13 +121,26 @@ class FileConvertOptions(OptionsDictValidator): """ return self._ffmpeg_post_process_args + def modified_variables(self) -> Dict[PluginOperation, Set[str]]: + return {PluginOperation.MODIFY_ENTRY: {v.ext.variable_name}} + class FileConvertPlugin(Plugin[FileConvertOptions]): plugin_options_type = FileConvertOptions - # Perform this after regex - priority: PluginPriority = PluginPriority( - modify_entry=PluginPriority.MODIFY_ENTRY_AFTER_SPLIT + 1 - ) + + def __init__( + self, + options: FileConvertOptions, + overrides: Overrides, + enhanced_download_archive: EnhancedDownloadArchive, + ): + super().__init__( + options=options, + overrides=overrides, + enhanced_download_archive=enhanced_download_archive, + ) + # Lookup of entry id to what it was converted from for logging + self._converted_from_lookup: Dict[str, str] = {} def ytdl_options(self) -> Optional[Dict]: """ @@ -198,13 +217,9 @@ class FileConvertPlugin(Plugin[FileConvertOptions]): FileHandler.delete(tmp_output_file) if original_ext != new_ext: - entry.add_kwargs( - { - "__converted_from": original_ext, - } - ) + self._converted_from_lookup[entry.ytdl_uid()] = original_ext - entry.add_kwargs({EXT: new_ext}) + entry.add({v.ext: new_ext}) return entry @@ -212,7 +227,7 @@ class FileConvertPlugin(Plugin[FileConvertOptions]): """ Add metadata about conversion if it happened """ - if converted_from := entry.kwargs_get("__converted_from"): + if converted_from := self._converted_from_lookup.get(entry.ytdl_uid()): return FileMetadata(f"Converted from {converted_from}") return None diff --git a/src/ytdl_sub/plugins/format.py b/src/ytdl_sub/plugins/format.py index 7f1ccc06..3b5c334f 100644 --- a/src/ytdl_sub/plugins/format.py +++ b/src/ytdl_sub/plugins/format.py @@ -1,8 +1,8 @@ from typing import Dict from typing import Optional -from ytdl_sub.config.plugin import Plugin -from ytdl_sub.config.preset_options import OptionsValidator +from ytdl_sub.config.plugin.plugin import Plugin +from ytdl_sub.config.validators.options import OptionsValidator from ytdl_sub.validators.validators import StringValidator diff --git a/src/ytdl_sub/plugins/internal/view.py b/src/ytdl_sub/plugins/internal/view.py index fa8d5471..39685ad3 100644 --- a/src/ytdl_sub/plugins/internal/view.py +++ b/src/ytdl_sub/plugins/internal/view.py @@ -1,10 +1,9 @@ import copy from typing import Optional -from ytdl_sub.config.plugin import Plugin -from ytdl_sub.config.plugin import PluginPriority -from ytdl_sub.config.preset_options import OptionsDictValidator -from ytdl_sub.config.preset_options import Overrides +from ytdl_sub.config.overrides import Overrides +from ytdl_sub.config.plugin.plugin import Plugin +from ytdl_sub.config.validators.options import OptionsDictValidator from ytdl_sub.entries.entry import Entry from ytdl_sub.utils.file_handler import FileMetadata from ytdl_sub.ytdl_additions.enhanced_download_archive import EnhancedDownloadArchive @@ -20,7 +19,6 @@ class ViewOptions(OptionsDictValidator): class ViewPlugin(Plugin[ViewOptions]): plugin_options_type = ViewOptions - priority: PluginPriority = PluginPriority(modify_entry=PluginPriority.MODIFY_ENTRY_AFTER_SPLIT) _MAX_LINE_WIDTH: int = 80 diff --git a/src/ytdl_sub/plugins/match_filters.py b/src/ytdl_sub/plugins/match_filters.py index ef4d6d4a..4e1ad498 100644 --- a/src/ytdl_sub/plugins/match_filters.py +++ b/src/ytdl_sub/plugins/match_filters.py @@ -3,8 +3,8 @@ from typing import Any from typing import List from typing import Tuple -from ytdl_sub.config.plugin import Plugin -from ytdl_sub.config.preset_options import OptionsDictValidator +from ytdl_sub.config.plugin.plugin import Plugin +from ytdl_sub.config.validators.options import OptionsDictValidator from ytdl_sub.utils.logger import Logger from ytdl_sub.validators.validators import StringListValidator diff --git a/src/ytdl_sub/plugins/music_tags.py b/src/ytdl_sub/plugins/music_tags.py index e7bb92e4..ec99d50f 100644 --- a/src/ytdl_sub/plugins/music_tags.py +++ b/src/ytdl_sub/plugins/music_tags.py @@ -6,9 +6,11 @@ from typing import List import mediafile -from ytdl_sub.config.plugin import Plugin -from ytdl_sub.config.preset_options import OptionsDictValidator +from ytdl_sub.config.plugin.plugin import Plugin +from ytdl_sub.config.validators.options import OptionsDictValidator from ytdl_sub.entries.entry import Entry +from ytdl_sub.entries.script.variable_definitions import VARIABLES +from ytdl_sub.entries.script.variable_definitions import VariableDefinitions from ytdl_sub.utils.file_handler import FileMetadata from ytdl_sub.utils.logger import Logger from ytdl_sub.validators.audo_codec_validator import AUDIO_CODEC_EXTS @@ -17,6 +19,8 @@ from ytdl_sub.validators.string_formatter_validators import ListFormatterValidat from ytdl_sub.validators.string_formatter_validators import StringFormatterValidator from ytdl_sub.validators.validators import BoolValidator +v: VariableDefinitions = VARIABLES + logger = Logger.get("music_tags") @@ -132,9 +136,9 @@ class MusicTagsPlugin(Plugin[MusicTagsOptions]): """ Tags the entry's audio file using values defined in the metadata options """ - if entry.ext not in AUDIO_CODEC_EXTS: + if (ext := entry.get(v.ext, str)) not in AUDIO_CODEC_EXTS: raise self.plugin_options.validation_exception( - f"music_tags plugin received a video with the extension '{entry.ext}'. Only audio " + f"music_tags plugin received a video with the extension '{ext}'. Only audio " f"files are supported for setting music tags. Ensure you are converting the video " f"to audio using the audio_extract plugin." ) diff --git a/src/ytdl_sub/plugins/nfo_tags.py b/src/ytdl_sub/plugins/nfo_tags.py index 2ff09d55..92e2b968 100644 --- a/src/ytdl_sub/plugins/nfo_tags.py +++ b/src/ytdl_sub/plugins/nfo_tags.py @@ -7,8 +7,8 @@ from typing import Dict from typing import List from typing import Optional -from ytdl_sub.config.plugin import Plugin -from ytdl_sub.config.preset_options import OptionsDictValidator +from ytdl_sub.config.plugin.plugin import Plugin +from ytdl_sub.config.validators.options import OptionsDictValidator from ytdl_sub.entries.entry import Entry from ytdl_sub.utils.file_handler import FileHandler from ytdl_sub.utils.file_handler import FileMetadata diff --git a/src/ytdl_sub/plugins/output_directory_nfo_tags.py b/src/ytdl_sub/plugins/output_directory_nfo_tags.py index 1244c870..72f1828d 100644 --- a/src/ytdl_sub/plugins/output_directory_nfo_tags.py +++ b/src/ytdl_sub/plugins/output_directory_nfo_tags.py @@ -1,4 +1,4 @@ -from ytdl_sub.config.preset_options import Overrides +from ytdl_sub.config.overrides import Overrides from ytdl_sub.entries.entry import Entry from ytdl_sub.plugins.nfo_tags import NfoTagsValidator from ytdl_sub.plugins.nfo_tags import SharedNfoTagsOptions diff --git a/src/ytdl_sub/plugins/regex.py b/src/ytdl_sub/plugins/regex.py index d2228190..325649ea 100644 --- a/src/ytdl_sub/plugins/regex.py +++ b/src/ytdl_sub/plugins/regex.py @@ -1,17 +1,18 @@ +from collections import defaultdict from typing import Any from typing import Dict from typing import List from typing import Optional +from typing import Set -from yt_dlp.utils import sanitize_filename - -from ytdl_sub.config.plugin import Plugin -from ytdl_sub.config.plugin import PluginPriority -from ytdl_sub.config.preset_options import OptionsDictValidator +from ytdl_sub.config.overrides import Overrides +from ytdl_sub.config.plugin.plugin import Plugin +from ytdl_sub.config.plugin.plugin_operation import PluginOperation +from ytdl_sub.config.validators.options import OptionsDictValidator from ytdl_sub.entries.entry import Entry -from ytdl_sub.entries.variables.kwargs import YTDL_SUB_REGEX_SOURCE_VARS +from ytdl_sub.script.parser import parse +from ytdl_sub.script.utils.exceptions import RuntimeException from ytdl_sub.utils.exceptions import RegexNoMatchException -from ytdl_sub.utils.exceptions import StringFormattingVariableNotFoundException from ytdl_sub.utils.logger import Logger from ytdl_sub.validators.regex_validator import RegexListValidator from ytdl_sub.validators.source_variable_validator import SourceVariableNameListValidator @@ -19,6 +20,8 @@ from ytdl_sub.validators.strict_dict_validator import StrictDictValidator from ytdl_sub.validators.string_formatter_validators import ListFormatterValidator from ytdl_sub.validators.string_formatter_validators import StringFormatterValidator from ytdl_sub.validators.validators import BoolValidator +from ytdl_sub.validators.validators import DictValidator +from ytdl_sub.ytdl_additions.enhanced_download_archive import EnhancedDownloadArchive logger = Logger.get(name="regex") @@ -109,11 +112,7 @@ class VariableRegex(StrictDictValidator): return self._capture_group_defaults is not None -class FromSourceVariablesRegex(StrictDictValidator): - - _optional_keys = Entry.source_variables() - _allow_extra_keys = True - +class FromSourceVariablesRegex(DictValidator): def __init__(self, name, value): super().__init__(name, value) self.variable_capture_dict: Dict[str, VariableRegex] = { @@ -204,6 +203,9 @@ class RegexOptions(OptionsDictValidator): key="skip_if_match_fails", validator=BoolValidator, default=True ).value + # Variables added by the regex plugin + self._added_variable_names: Set[str] = set() + @property def skip_if_match_fails(self) -> Optional[bool]: """ @@ -212,39 +214,6 @@ class RegexOptions(OptionsDictValidator): """ return self._skip_if_match_fails - def validate_with_variables( - self, source_variables: List[str], override_variables: Dict[str, str] - ) -> None: - """ - Ensures each source variable capture group is valid - - Parameters - ---------- - source_variables - Available source variables when running the plugin - override_variables - Available override variables when running the plugin - """ - for key, regex_options in self.source_variable_capture_dict.items(): - # Ensure each variable getting captured is a source variable - if key not in source_variables and key not in override_variables: - raise self._validation_exception( - f"cannot regex capture '{key}' because it is not a source or override variable" - ) - - # Ensure the capture group names are not existing source/override variables - for capture_group_name in regex_options.capture_group_names: - if capture_group_name in source_variables: - raise self._validation_exception( - f"'{capture_group_name}' cannot be used as a capture group name because it " - f"is a source variable" - ) - if capture_group_name in override_variables: - raise self._validation_exception( - f"'{capture_group_name}' cannot be used as a capture group name because it " - f"is an override variable" - ) - @property def source_variable_capture_dict(self) -> Dict[str, VariableRegex]: """ @@ -254,39 +223,91 @@ class RegexOptions(OptionsDictValidator): """ return self._from.variable_capture_dict - def added_source_variables(self) -> List[str]: + @classmethod + def _can_resolve( + cls, unresolved_variables: Set[str], input_variable_name: str, regex_options: VariableRegex + ) -> bool: + if input_variable_name in unresolved_variables: + return False + for capture_group_default in regex_options.capture_group_defaults or []: + parsed_default = parse(capture_group_default.format_string) + if parsed_default.variables and parsed_default.variables.issubset(unresolved_variables): + return False + return True + + def added_variables( + self, + resolved_variables: Set[str], + unresolved_variables: Set[str], + plugin_op: PluginOperation, + ) -> Dict[PluginOperation, Set[str]]: """ Returns ------- List of new source variables created via regex capture """ - added_source_vars: List[str] = [] - for regex_options in self.source_variable_capture_dict.values(): - added_source_vars.extend(regex_options.capture_group_names) - added_source_vars.extend( - f"{capture_group_name}_sanitized" - for capture_group_name in regex_options.capture_group_names - ) + added_source_vars: Dict[PluginOperation, Set[str]] = { + PluginOperation.MODIFY_ENTRY_METADATA: set(), + PluginOperation.MODIFY_ENTRY: set(), + } + for input_variable_name, regex_options in self.source_variable_capture_dict.items(): + variables_to_add = set(regex_options.capture_group_names) + + if plugin_op != PluginOperation.ANY and input_variable_name not in ( + resolved_variables | unresolved_variables + ): + raise self._validation_exception( + f"cannot regex capture '{input_variable_name}' because it is not a" + " defined variable." + ) + if ( + plugin_op.value >= PluginOperation.MODIFY_ENTRY.value + and input_variable_name in unresolved_variables + ): + raise self._validation_exception( + f"cannot regex capture '{input_variable_name}' because it is not " + f"computed until later in execution." + ) + + if plugin_op == PluginOperation.ANY: + added_source_vars[PluginOperation.MODIFY_ENTRY_METADATA] |= variables_to_add + self._added_variable_names |= variables_to_add + continue + + if not self._can_resolve( + unresolved_variables=unresolved_variables, + input_variable_name=input_variable_name, + regex_options=regex_options, + ): + continue + + for capture_group_name in regex_options.capture_group_names: + if capture_group_name in (resolved_variables - self._added_variable_names): + raise self._validation_exception( + f"cannot use '{capture_group_name}' as a capture group name because it is " + f"an already defined variable." + ) + added_source_vars[plugin_op] |= set(regex_options.capture_group_names) return added_source_vars class RegexPlugin(Plugin[RegexOptions]): plugin_options_type = RegexOptions - priority = PluginPriority( - modify_entry=PluginPriority.MODIFY_ENTRY_AFTER_SPLIT + 0, - ) - @classmethod - def _add_processed_regex_variable_name(cls, entry: Entry, source_var: str) -> None: - if not entry.kwargs_contains(YTDL_SUB_REGEX_SOURCE_VARS): - entry.add_kwargs({YTDL_SUB_REGEX_SOURCE_VARS: []}) - - entry.kwargs(YTDL_SUB_REGEX_SOURCE_VARS).append(source_var) - - @classmethod - def _contains_processed_regex_variable(cls, entry: Entry, variable_name: str) -> bool: - return variable_name in entry.kwargs_get(YTDL_SUB_REGEX_SOURCE_VARS, []) + def __init__( + self, + options: RegexOptions, + overrides: Overrides, + enhanced_download_archive: EnhancedDownloadArchive, + ): + super().__init__( + options=options, + overrides=overrides, + enhanced_download_archive=enhanced_download_archive, + ) + # Lookup of entry id to processed regex variables + self._processed_regex_vars: Dict[str, Set[str]] = defaultdict(set) def _try_skip_entry(self, entry: Entry, variable_name: str) -> None: # Skip the entry if toggled @@ -301,34 +322,16 @@ class RegexPlugin(Plugin[RegexOptions]): # Otherwise, error raise RegexNoMatchException(f"Regex failed to match '{variable_name}' from '{entry.title}'") - def _can_process_at_metadata_stage(self, entry: Entry, variable_name: str) -> bool: - # If the variable is an override... - if variable_name in self.overrides.dict: - # Try to see if it can resolve - try: - self.overrides.apply_formatter( - formatter=self.overrides.dict[variable_name], - entry=entry, - ) - # If it can not from missing variables (from post-metadata stage), return False - except StringFormattingVariableNotFoundException: - return False - # If it is a source variable and not present, return false - elif variable_name not in entry.to_dict(): + @classmethod + def _can_process_at_metadata_stage(cls, entry: Entry, variable_name: str) -> bool: + # Try to see if it can resolve + try: + _ = entry.script.get(variable_name) + return True + # If it can not from missing variables (from post-metadata stage), return False + except RuntimeException: return False - return True - - def _get_regex_input_string(self, entry: Entry, variable_name: str) -> str: - # Apply override formatter if it's an override - if variable_name in self.overrides.dict: - return self.overrides.apply_formatter( - formatter=self.overrides.dict[variable_name], - entry=entry, - ) - # Otherwise pluck from the entry's source variable - return entry.to_dict()[variable_name] - def _modify_entry_metadata(self, entry: Entry, is_metadata_stage: bool) -> Optional[Entry]: """ Parameters @@ -356,7 +359,7 @@ class RegexPlugin(Plugin[RegexOptions]): # Record which regex source variables are processed, to # process as many variables as possible in the metadata stage, then the rest # after the media file has been downloaded. - if self._contains_processed_regex_variable(entry, variable_name): + if variable_name in self._processed_regex_vars[entry.ytdl_uid()]: continue # If it's the metadata stage, and it can't be processed, skip until post-metadata @@ -365,12 +368,8 @@ class RegexPlugin(Plugin[RegexOptions]): ): continue - self._add_processed_regex_variable_name(entry, variable_name) - - regex_input_str = self._get_regex_input_string( - entry=entry, - variable_name=variable_name, - ) + self._processed_regex_vars[entry.ytdl_uid()].add(variable_name) + regex_input_str = str(entry.script.get(variable_name)) if ( regex_options.exclude is not None @@ -388,49 +387,23 @@ class RegexPlugin(Plugin[RegexOptions]): if not regex_options.has_defaults: return self._try_skip_entry(entry=entry, variable_name=variable_name) - # otherwise, use defaults (apply them using the original entry source dict) - source_variables_and_overrides_dict = dict( - entry.to_dict(), **self.overrides.dict_with_format_strings - ) - # add both the default... - entry.add_variables( - variables_to_add={ - regex_options.capture_group_names[i]: default.apply_formatter( - variable_dict=source_variables_and_overrides_dict + entry.add( + { + regex_options.capture_group_names[i]: self.overrides.apply_formatter( + formatter=default, entry=entry ) for i, default in enumerate(regex_options.capture_group_defaults) - }, - ) - # and sanitized default - entry.add_variables( - variables_to_add={ - f"{regex_options.capture_group_names[i]}_sanitized": sanitize_filename( - default.apply_formatter( - variable_dict=source_variables_and_overrides_dict - ) - ) - for i, default in enumerate(regex_options.capture_group_defaults) - }, + } ) # There is a capture, add the source variables to the entry as # {source_var}_capture_1, {source_var}_capture_2, ... else: - # Add the value... - entry.add_variables( - variables_to_add={ + entry.add( + { regex_options.capture_group_names[i]: capture for i, capture in enumerate(maybe_capture) - }, - ) - # And the sanitized value - entry.add_variables( - variables_to_add={ - f"{regex_options.capture_group_names[i]}_sanitized": sanitize_filename( - capture - ) - for i, capture in enumerate(maybe_capture) - }, + } ) return entry diff --git a/src/ytdl_sub/plugins/split_by_chapters.py b/src/ytdl_sub/plugins/split_by_chapters.py index 50717854..fc83a07c 100644 --- a/src/ytdl_sub/plugins/split_by_chapters.py +++ b/src/ytdl_sub/plugins/split_by_chapters.py @@ -1,19 +1,17 @@ -import copy -from pathlib import Path from typing import Any +from typing import Dict from typing import List from typing import Optional +from typing import Set from typing import Tuple -from yt_dlp.utils import sanitize_filename - -from ytdl_sub.config.plugin import SplitPlugin -from ytdl_sub.config.preset_options import OptionsDictValidator +from ytdl_sub.config.plugin.plugin import SplitPlugin +from ytdl_sub.config.plugin.plugin_operation import PluginOperation +from ytdl_sub.config.validators.options import OptionsDictValidator from ytdl_sub.entries.entry import Entry -from ytdl_sub.entries.variables.kwargs import CHAPTERS -from ytdl_sub.entries.variables.kwargs import SPLIT_BY_CHAPTERS_PARENT_ENTRY -from ytdl_sub.entries.variables.kwargs import SPONSORBLOCK_CHAPTERS -from ytdl_sub.entries.variables.kwargs import UID +from ytdl_sub.entries.entry import ytdl_sub_split_by_chapters_parent_uid +from ytdl_sub.entries.script.variable_definitions import VARIABLES +from ytdl_sub.entries.script.variable_definitions import VariableDefinitions from ytdl_sub.utils.chapters import Chapters from ytdl_sub.utils.chapters import Timestamp from ytdl_sub.utils.exceptions import ValidationException @@ -22,6 +20,8 @@ from ytdl_sub.utils.file_handler import FileHandler from ytdl_sub.utils.file_handler import FileMetadata from ytdl_sub.validators.string_select_validator import StringSelectValidator +v: VariableDefinitions = VARIABLES + def _split_video_ffmpeg_cmd( input_file: str, output_file: str, timestamps: List[Timestamp], idx: int @@ -81,14 +81,21 @@ class SplitByChaptersOptions(OptionsDictValidator): key="when_no_chapters", validator=WhenNoChaptersValidator ).value - def added_source_variables(self) -> List[str]: - return [ - "chapter_title", - "chapter_title_sanitized", - "chapter_index", - "chapter_index_padded", - "chapter_count", - ] + def added_variables( + self, + resolved_variables: Set[str], + unresolved_variables: Set[str], + plugin_op: PluginOperation, + ) -> Dict[PluginOperation, Set[str]]: + return { + PluginOperation.MODIFY_ENTRY: { + "chapter_title", + "chapter_title_sanitized", + "chapter_index", + "chapter_index_padded", + "chapter_count", + } + } @property def when_no_chapters(self) -> str: @@ -98,44 +105,49 @@ class SplitByChaptersOptions(OptionsDictValidator): """ return self._when_no_chapters + def modified_variables(self) -> Dict[PluginOperation, Set[str]]: + return { + PluginOperation.MODIFY_ENTRY: { + v.uid.variable_name, + ytdl_sub_split_by_chapters_parent_uid.variable_name, + } + } + class SplitByChaptersPlugin(SplitPlugin[SplitByChaptersOptions]): plugin_options_type = SplitByChaptersOptions + @classmethod + def _non_split_entry(cls, entry: Entry) -> Entry: + entry.add( + { + "chapter_title": f"{{ {v.title.variable_name} }}", + "chapter_index": 1, + "chapter_index_padded": "01", + "chapter_count": 1, + v.uid: entry.uid, + ytdl_sub_split_by_chapters_parent_uid: entry.uid, + } + ) + return entry + def _create_split_entry( - self, source_entry: Entry, title: str, idx: int, chapters: Chapters + self, new_entry: Entry, title: str, idx: int, chapters: Chapters ) -> Tuple[Entry, FileMetadata]: """ Runs ffmpeg to create the split video """ - entry = copy.deepcopy(source_entry) - - entry.add_variables( + new_entry.add( { "chapter_title": title, - "chapter_title_sanitized": sanitize_filename(title), "chapter_index": idx + 1, "chapter_index_padded": f"{(idx + 1):02d}", "chapter_count": len(chapters.timestamps), } ) - # pylint: disable=protected-access - entry.add_kwargs( - { - UID: _split_video_uid(source_uid=entry.uid, idx=idx), - SPLIT_BY_CHAPTERS_PARENT_ENTRY: source_entry._kwargs, - } - ) - - if entry.kwargs_contains(CHAPTERS): - del entry._kwargs[CHAPTERS] - if entry.kwargs_contains(SPONSORBLOCK_CHAPTERS): - del entry._kwargs[SPONSORBLOCK_CHAPTERS] - # pylint: enable=protected-access - timestamp_begin = chapters.timestamps[idx].readable_str - timestamp_end = Timestamp(entry.kwargs("duration")).readable_str + timestamp_end = Timestamp(new_entry.get(v.duration, int)).readable_str if idx + 1 < len(chapters.timestamps): timestamp_end = chapters.timestamps[idx + 1].readable_str @@ -145,7 +157,7 @@ class SplitByChaptersPlugin(SplitPlugin[SplitByChaptersOptions]): "Warning" ] = "Dry-run assumes embedded chapters with no modifications" - metadata_value_dict["Source Title"] = entry.title + metadata_value_dict["Source Title"] = new_entry.title metadata_value_dict["Segment"] = f"{timestamp_begin} - {timestamp_end}" metadata = FileMetadata.from_dict( @@ -154,7 +166,7 @@ class SplitByChaptersPlugin(SplitPlugin[SplitByChaptersOptions]): sort_dict=False, ) - return entry, metadata + return new_entry, metadata def split(self, entry: Entry) -> Optional[List[Tuple[Entry, FileMetadata]]]: """ @@ -166,15 +178,9 @@ class SplitByChaptersPlugin(SplitPlugin[SplitByChaptersOptions]): # If no chapters, do not split anything if not chapters.contains_any_chapters(): if self.plugin_options.when_no_chapters == "pass": - entry.add_variables( - { - "chapter_title": entry.title, - "chapter_index": 1, - "chapter_index_padded": "01", - "chapter_count": 1, - } - ) - return [(entry, FileMetadata())] + # Modify the entry t + return [(self._non_split_entry(entry), FileMetadata())] + if self.plugin_options.when_no_chapters == "drop": return [] @@ -183,18 +189,16 @@ class SplitByChaptersPlugin(SplitPlugin[SplitByChaptersOptions]): ) for idx, title in enumerate(chapters.titles): - new_uid = _split_video_uid(source_uid=entry.uid, idx=idx) + new_entry = Entry.create_split_entry( + entry=entry, new_uid=_split_video_uid(source_uid=entry.uid, idx=idx) + ) if not self.is_dry_run: - # Get the input/output file paths - input_file = entry.get_download_file_path() - output_file = str(Path(self.working_directory) / f"{new_uid}.{entry.ext}") - # Run ffmpeg to create the split the video FFMPEG.run( _split_video_ffmpeg_cmd( - input_file=input_file, - output_file=output_file, + input_file=entry.get_download_file_path(), + output_file=new_entry.get_download_file_path(), timestamps=chapters.timestamps, idx=idx, ) @@ -205,14 +209,13 @@ class SplitByChaptersPlugin(SplitPlugin[SplitByChaptersOptions]): if entry.is_thumbnail_downloaded(): FileHandler.copy( src_file_path=entry.get_download_thumbnail_path(), - dst_file_path=Path(self.working_directory) - / f"{new_uid}.{entry.thumbnail_ext}", + dst_file_path=new_entry.get_download_thumbnail_path(), ) # Format the split video split_videos_and_metadata.append( self._create_split_entry( - source_entry=entry, + new_entry=new_entry, title=title, idx=idx, chapters=chapters, diff --git a/src/ytdl_sub/plugins/subtitles.py b/src/ytdl_sub/plugins/subtitles.py index 4dff539d..445a00a9 100644 --- a/src/ytdl_sub/plugins/subtitles.py +++ b/src/ytdl_sub/plugins/subtitles.py @@ -2,11 +2,15 @@ from pathlib import Path from typing import Dict from typing import List from typing import Optional +from typing import Set -from ytdl_sub.config.plugin import Plugin -from ytdl_sub.config.preset_options import OptionsDictValidator +from ytdl_sub.config.plugin.plugin import Plugin +from ytdl_sub.config.plugin.plugin_operation import PluginOperation +from ytdl_sub.config.validators.options import OptionsDictValidator from ytdl_sub.downloaders.ytdl_options_builder import YTDLOptionsBuilder from ytdl_sub.entries.entry import Entry +from ytdl_sub.entries.script.variable_definitions import VARIABLES +from ytdl_sub.entries.script.variable_definitions import VariableDefinitions from ytdl_sub.utils.file_handler import FileHandler from ytdl_sub.utils.file_handler import FileMetadata from ytdl_sub.utils.logger import Logger @@ -17,6 +21,8 @@ from ytdl_sub.validators.string_select_validator import StringSelectValidator from ytdl_sub.validators.validators import BoolValidator from ytdl_sub.validators.validators import StringListValidator +v: VariableDefinitions = VARIABLES + logger = Logger.get(name="subtitles") @@ -113,13 +119,18 @@ class SubtitleOptions(OptionsDictValidator): """ return self._allow_auto_generated_subtitles - def added_source_variables(self) -> List[str]: + def added_variables( + self, + resolved_variables: Set[str], + unresolved_variables: Set[str], + plugin_op: PluginOperation, + ) -> Dict[PluginOperation, Set[str]]: """ Returns ------- List of new source variables created by using the subtitles plugin """ - return ["lang", "subtitles_ext"] + return {PluginOperation.MODIFY_ENTRY_METADATA: {"lang", "subtitles_ext"}} class SubtitlesPlugin(Plugin[SubtitleOptions]): @@ -158,18 +169,8 @@ class SubtitlesPlugin(Plugin[SubtitleOptions]): return builder.to_dict() - def modify_entry(self, entry: Entry) -> Optional[Entry]: - if not (requested_subtitles := entry.kwargs_get("requested_subtitles", None)): - return entry - - languages = sorted(requested_subtitles.keys()) - entry.add_variables( - variables_to_add={ - "subtitles_ext": self.plugin_options.subtitles_type, - "lang": ",".join(languages), - } - ) - + def modify_entry_metadata(self, entry: Entry) -> Optional[Entry]: + entry.add({"subtitles_ext": self.plugin_options.subtitles_type, "lang": ""}) return entry def post_process_entry(self, entry: Entry) -> Optional[FileMetadata]: @@ -181,7 +182,7 @@ class SubtitlesPlugin(Plugin[SubtitleOptions]): entry: Entry to create subtitles for """ - requested_subtitles = entry.kwargs("requested_subtitles") + requested_subtitles = entry.get(v.requested_subtitles, expected_type=dict) if not requested_subtitles: logger.debug("subtitles not found for %s", entry.title) return None @@ -189,6 +190,10 @@ class SubtitlesPlugin(Plugin[SubtitleOptions]): file_metadata: Optional[FileMetadata] = None langs = list(requested_subtitles.keys()) + # HACK to maintain order of languages for fixtures + if len(langs) == len(self.plugin_options.languages): + langs = self.plugin_options.languages + if self.plugin_options.embed_subtitles: file_metadata = FileMetadata(f"Embedded subtitles with lang(s) {', '.join(langs)}") if self.plugin_options.subtitles_name: diff --git a/src/ytdl_sub/plugins/throttle_protection.py b/src/ytdl_sub/plugins/throttle_protection.py index 25b5791b..5bab172c 100644 --- a/src/ytdl_sub/plugins/throttle_protection.py +++ b/src/ytdl_sub/plugins/throttle_protection.py @@ -4,9 +4,9 @@ from typing import List from typing import Optional from typing import Tuple -from ytdl_sub.config.plugin import Plugin -from ytdl_sub.config.preset_options import OptionsDictValidator -from ytdl_sub.config.preset_options import Overrides +from ytdl_sub.config.overrides import Overrides +from ytdl_sub.config.plugin.plugin import Plugin +from ytdl_sub.config.validators.options import OptionsDictValidator from ytdl_sub.entries.entry import Entry from ytdl_sub.utils.file_handler import FileMetadata from ytdl_sub.utils.logger import Logger diff --git a/src/ytdl_sub/plugins/video_tags.py b/src/ytdl_sub/plugins/video_tags.py index e27af4d8..9ce3d8fd 100644 --- a/src/ytdl_sub/plugins/video_tags.py +++ b/src/ytdl_sub/plugins/video_tags.py @@ -2,8 +2,8 @@ import copy from typing import Any from typing import Dict -from ytdl_sub.config.plugin import Plugin -from ytdl_sub.config.preset_options import OptionsDictValidator +from ytdl_sub.config.plugin.plugin import Plugin +from ytdl_sub.config.validators.options import OptionsDictValidator from ytdl_sub.entries.entry import Entry from ytdl_sub.utils.ffmpeg import add_ffmpeg_metadata_key_values from ytdl_sub.utils.file_handler import FileMetadata diff --git a/src/ytdl_sub/script/__init__.py b/src/ytdl_sub/script/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/ytdl_sub/script/functions/__init__.py b/src/ytdl_sub/script/functions/__init__.py new file mode 100644 index 00000000..5b98d24b --- /dev/null +++ b/src/ytdl_sub/script/functions/__init__.py @@ -0,0 +1,82 @@ +from typing import Callable +from typing import Dict + +from ytdl_sub.script.functions.array_functions import ArrayFunctions +from ytdl_sub.script.functions.boolean_functions import BooleanFunctions +from ytdl_sub.script.functions.conditional_functions import ConditionalFunctions +from ytdl_sub.script.functions.date_functions import DateFunctions +from ytdl_sub.script.functions.error_functions import ErrorFunctions +from ytdl_sub.script.functions.json_functions import JsonFunctions +from ytdl_sub.script.functions.map_functions import MapFunctions +from ytdl_sub.script.functions.numeric_functions import NumericFunctions +from ytdl_sub.script.functions.regex_functions import RegexFunctions +from ytdl_sub.script.functions.string_functions import StringFunctions +from ytdl_sub.script.types.resolvable import Resolvable +from ytdl_sub.script.utils.exceptions import FunctionDoesNotExistRuntimeException + + +class Functions( + StringFunctions, + NumericFunctions, + ConditionalFunctions, + ArrayFunctions, + MapFunctions, + BooleanFunctions, + ErrorFunctions, + RegexFunctions, + DateFunctions, + JsonFunctions, +): + _custom_functions: Dict[str, Callable[..., Resolvable]] = {} + + @classmethod + def is_built_in(cls, name: str) -> bool: + """ + Returns + ------- + True if the name exists as a built-in function or custom function. False otherwise. + """ + return hasattr(cls, name) or hasattr(cls, f"{name}_") or name in cls._custom_functions + + @classmethod + def get(cls, name: str) -> Callable[..., Resolvable]: + """ + Returns + ------- + The actual Python callable for the function of the given name. + + Raises + ------ + FunctionDoesNotExistRuntimeException + If the function does not exist. + """ + if hasattr(cls, name): + return getattr(cls, name) + if hasattr(cls, f"{name}_"): + return getattr(cls, f"{name}_") + if name in cls._custom_functions: + return cls._custom_functions[name] + + raise FunctionDoesNotExistRuntimeException(f"The function {name} does not exist") + + @classmethod + def register_function(cls, function: Callable[..., Resolvable]) -> None: + """ + Adds a function to the suite of offered functions. + + Parameters + ---------- + function + A static function whose name will be used as the offered function name. + + Raises + ------ + ValueError + If the name already exists as a function. + """ + if cls.is_built_in(function.__name__): + raise ValueError( + f"Cannot register a function with name {function.__name__} " + f"because it already exists" + ) + cls._custom_functions[function.__name__] = function diff --git a/src/ytdl_sub/script/functions/array_functions.py b/src/ytdl_sub/script/functions/array_functions.py new file mode 100644 index 00000000..ffff0da7 --- /dev/null +++ b/src/ytdl_sub/script/functions/array_functions.py @@ -0,0 +1,147 @@ +import itertools +from typing import List +from typing import Optional + +from ytdl_sub.script.types.array import Array +from ytdl_sub.script.types.resolvable import AnyArgument +from ytdl_sub.script.types.resolvable import Boolean +from ytdl_sub.script.types.resolvable import Integer +from ytdl_sub.script.types.resolvable import Lambda +from ytdl_sub.script.types.resolvable import LambdaReduce +from ytdl_sub.script.types.resolvable import LambdaTwo +from ytdl_sub.script.types.resolvable import Resolvable +from ytdl_sub.script.utils.exceptions import UNREACHABLE +from ytdl_sub.script.utils.exceptions import ArrayValueDoesNotExist +from ytdl_sub.script.utils.exceptions import FunctionRuntimeException + + +class ArrayFunctions: + @staticmethod + def array(maybe_array: AnyArgument) -> Array: + """ + Tries to cast an unknown variable type to an Array. + + Raises + ------ + FunctionRuntimeException + If the input type is not actually an Array. + """ + if not isinstance(maybe_array, Array): + raise FunctionRuntimeException( + f"Tried and failed to cast {maybe_array.type_name()} as an Array" + ) + return maybe_array + + @staticmethod + def array_size(array: Array) -> Integer: + """ + Returns the size of an Array. + """ + return Integer(len(array.value)) + + @staticmethod + def array_extend(*arrays: Array) -> Array: + """ + Combine multiple Arrays into a single Array. + """ + output: List[Resolvable] = [] + for array in arrays: + output.extend(array.value) + + return Array(output) + + @staticmethod + def array_at(array: Array, idx: Integer) -> AnyArgument: + """ + Return the element in the Array at index ``idx``. + """ + return array.value[idx.value] + + @staticmethod + def array_contains(array: Array, value: AnyArgument) -> Boolean: + """ + Return True if the value exists in the Array. False otherwise. + """ + return Boolean(value in array.value) + + @staticmethod + def array_index(array: Array, value: AnyArgument) -> Integer: + """ + Return the index of the value within the Array if it exists. If it does not, it will + throw an error. + """ + if not ArrayFunctions.array_contains(array=array, value=value): + raise ArrayValueDoesNotExist( + "Tried to get the index of a value in an Array that does not exist" + ) + + if isinstance(value, Resolvable): + return Integer(array.value.index(value)) + + raise UNREACHABLE + + @staticmethod + def array_slice(array: Array, start: Integer, end: Optional[Integer] = None) -> Array: + """ + Returns the slice of the Array. + """ + if end is not None: + return Array(array.value[start.value : end.value]) + return Array(array.value[start.value :]) + + @staticmethod + def array_flatten(array: Array) -> Array: + """ + Flatten any nested Arrays into a single-dimensional Array. + """ + output: List[Resolvable] = [] + for elem in array.value: + if isinstance(elem, Array): + output.extend(ArrayFunctions.array_flatten(elem).value) + else: + output.append(elem) + + return Array(output) + + @staticmethod + def array_reverse(array: Array) -> Array: + """ + Reverse an Array. + """ + return Array(list(reversed(array.value))) + + @staticmethod + def array_product(*arrays: Array) -> Array: + """ + Returns the Cartesian product of elements from different arrays + """ + out: List[Resolvable] = [] + for combo in itertools.product(*[arr.value for arr in arrays]): + out.append(Array(combo)) + + return Array(out) + + # pylint: disable=unused-argument + + @staticmethod + def array_apply(array: Array, lambda_function: Lambda) -> Array: + """ + Apply a lambda function on every element in the Array. + """ + return Array([Array([val]) for val in array.value]) + + @staticmethod + def array_enumerate(array: Array, lambda_function: LambdaTwo) -> Array: + """ + Apply a lambda function on every element in the Array, where each arg + passed to the lambda function is ``idx, element`` as two separate args. + """ + return Array([Array([Integer(idx), val]) for idx, val in enumerate(array.value)]) + + @staticmethod + def array_reduce(array: Array, lambda_reduce_function: LambdaReduce) -> AnyArgument: + """ + Apply a reduce function on pairs of elements in the Array, until one element remains. + Executes using the left-most and reduces in the right direction. + """ + return array diff --git a/src/ytdl_sub/script/functions/boolean_functions.py b/src/ytdl_sub/script/functions/boolean_functions.py new file mode 100644 index 00000000..0b45a547 --- /dev/null +++ b/src/ytdl_sub/script/functions/boolean_functions.py @@ -0,0 +1,89 @@ +from ytdl_sub.script.types.resolvable import AnyArgument +from ytdl_sub.script.types.resolvable import Boolean + +# pylint: disable=invalid-name + + +class BooleanFunctions: + """ + Comparison functions that output Booleans. + """ + + @staticmethod + def bool(value: AnyArgument) -> Boolean: + """ + Cast any type to a Boolean. + """ + return Boolean(bool(value.value)) + + @staticmethod + def eq(left: AnyArgument, right: AnyArgument) -> Boolean: + """ + ``==`` operator. Returns True if left == right. False otherwise. + """ + return Boolean(left.value == right.value) + + @staticmethod + def ne(left: AnyArgument, right: AnyArgument) -> Boolean: + """ + ``!=`` operator. Returns True if left != right. False otherwise. + """ + return Boolean(left.value != right.value) + + @staticmethod + def lt(left: AnyArgument, right: AnyArgument) -> Boolean: + """ + ``<`` operator. Returns True if left < right. False otherwise. + """ + return Boolean(left.value < right.value) + + @staticmethod + def lte(left: AnyArgument, right: AnyArgument) -> Boolean: + """ + ``<=`` operator. Returns True if left <= right. False otherwise. + """ + return Boolean(left.value <= right.value) + + @staticmethod + def gt(left: AnyArgument, right: AnyArgument) -> Boolean: + """ + ``>`` operator. Returns True if left > right. False otherwise. + """ + return Boolean(left.value > right.value) + + @staticmethod + def gte(left: AnyArgument, right: AnyArgument) -> Boolean: + """ + ``>=`` operator. Returns True if left >= right. False otherwise. + """ + return Boolean(left.value >= right.value) + + @staticmethod + def and_(*values: AnyArgument) -> Boolean: + """ + ``and`` operator. Returns True if all values evaluate to True. False otherwise. + """ + return Boolean(all(bool(val.value) for val in values)) + + @staticmethod + def or_(*values: AnyArgument) -> Boolean: + """ + ``or`` operator. Returns True if any value evaluates to True. False otherwise. + """ + return Boolean(any(bool(val.value) for val in values)) + + @staticmethod + def xor(*values: AnyArgument) -> Boolean: + """ + ``^`` operator. Returns True if exactly one value is set to True. False otherwise. + """ + bit_array = [bool(val.value) for val in values] + + return Boolean(sum(bit_array) == 1) + + @staticmethod + def not_(value: Boolean) -> Boolean: + """ + ``not`` operator. Returns the opposite of value. + """ + return Boolean(not value.value) diff --git a/src/ytdl_sub/script/functions/conditional_functions.py b/src/ytdl_sub/script/functions/conditional_functions.py new file mode 100644 index 00000000..8caaae02 --- /dev/null +++ b/src/ytdl_sub/script/functions/conditional_functions.py @@ -0,0 +1,31 @@ +from typing import Union + +from ytdl_sub.script.types.resolvable import Boolean +from ytdl_sub.script.types.resolvable import ReturnableArgumentA +from ytdl_sub.script.types.resolvable import ReturnableArgumentB + + +class ConditionalFunctions: + @staticmethod + def if_( + condition: Boolean, true: ReturnableArgumentA, false: ReturnableArgumentB + ) -> Union[ReturnableArgumentA, ReturnableArgumentB]: + """ + Conditional ``if`` statement that returns the ``true`` or ``false`` parameter + depending on the ``condition`` value. + """ + if condition.value: + return true + return false + + @staticmethod + def if_passthrough( + maybe_true_arg: ReturnableArgumentA, else_arg: ReturnableArgumentB + ) -> Union[ReturnableArgumentA, ReturnableArgumentB]: + """ + Conditional ``if`` statement that returns the ``maybe_true_arg`` if it evaluates to True, + otherwise returns ``else_arg``. + """ + if bool(maybe_true_arg.value): + return maybe_true_arg + return else_arg diff --git a/src/ytdl_sub/script/functions/date_functions.py b/src/ytdl_sub/script/functions/date_functions.py new file mode 100644 index 00000000..8d68f052 --- /dev/null +++ b/src/ytdl_sub/script/functions/date_functions.py @@ -0,0 +1,13 @@ +from datetime import datetime + +from ytdl_sub.script.types.resolvable import Integer +from ytdl_sub.script.types.resolvable import String + + +class DateFunctions: + @staticmethod + def datetime_strftime(posix_timestamp: Integer, date_format: String) -> String: + """ + Converts a posix timestamp to a date using strftime formatting. + """ + return String(datetime.utcfromtimestamp(posix_timestamp.value).strftime(date_format.value)) diff --git a/src/ytdl_sub/script/functions/error_functions.py b/src/ytdl_sub/script/functions/error_functions.py new file mode 100644 index 00000000..7eb1420f --- /dev/null +++ b/src/ytdl_sub/script/functions/error_functions.py @@ -0,0 +1,23 @@ +from ytdl_sub.script.types.resolvable import AnyArgument +from ytdl_sub.script.types.resolvable import ReturnableArgument +from ytdl_sub.script.types.resolvable import String +from ytdl_sub.script.utils.exceptions import UserThrownRuntimeError + + +class ErrorFunctions: + @staticmethod + def throw(error_message: String) -> AnyArgument: + """ + Explicitly throw an error with the provided error message. + """ + raise UserThrownRuntimeError(error_message) + + @staticmethod + def assert_(value: ReturnableArgument, assert_message: String) -> ReturnableArgument: + """ + Explicitly throw an error with the provided assert message if ``value`` evaluates to False. + If it evaluates to True, it will return ``value``. + """ + if not bool(value.value): + raise UserThrownRuntimeError(assert_message) + return value diff --git a/src/ytdl_sub/script/functions/json_functions.py b/src/ytdl_sub/script/functions/json_functions.py new file mode 100644 index 00000000..915fedf1 --- /dev/null +++ b/src/ytdl_sub/script/functions/json_functions.py @@ -0,0 +1,40 @@ +import json +from typing import Any + +from ytdl_sub.script.types.array import Array +from ytdl_sub.script.types.map import Map +from ytdl_sub.script.types.resolvable import AnyArgument +from ytdl_sub.script.types.resolvable import Boolean +from ytdl_sub.script.types.resolvable import Float +from ytdl_sub.script.types.resolvable import Integer +from ytdl_sub.script.types.resolvable import Resolvable +from ytdl_sub.script.types.resolvable import String +from ytdl_sub.script.utils.exceptions import UNREACHABLE + + +def _from_json(out: Any) -> Resolvable: + # pylint: disable=too-many-return-statements + if out is None: + return String("") + if isinstance(out, int): + return Integer(out) + if isinstance(out, float): + return Float(out) + if isinstance(out, str): + return String(out) + if isinstance(out, bool): + return Boolean(out) + if isinstance(out, list): + return Array(value=[_from_json(arg) for arg in out]) + if isinstance(out, dict): + return Map(value={_from_json(key): _from_json(value) for key, value in out.items()}) + raise UNREACHABLE + + +class JsonFunctions: + @staticmethod + def from_json(argument: String) -> AnyArgument: + """ + Converts a JSON string into an actual type. + """ + return _from_json(json.loads(argument.value)) diff --git a/src/ytdl_sub/script/functions/map_functions.py b/src/ytdl_sub/script/functions/map_functions.py new file mode 100644 index 00000000..45ca13f1 --- /dev/null +++ b/src/ytdl_sub/script/functions/map_functions.py @@ -0,0 +1,102 @@ +from typing import Optional + +from ytdl_sub.script.types.array import Array +from ytdl_sub.script.types.map import Map +from ytdl_sub.script.types.resolvable import AnyArgument +from ytdl_sub.script.types.resolvable import Boolean +from ytdl_sub.script.types.resolvable import Hashable +from ytdl_sub.script.types.resolvable import Integer +from ytdl_sub.script.types.resolvable import LambdaThree +from ytdl_sub.script.types.resolvable import LambdaTwo +from ytdl_sub.script.types.resolvable import String +from ytdl_sub.script.utils.exceptions import FunctionRuntimeException +from ytdl_sub.script.utils.exceptions import KeyDoesNotExistRuntimeException +from ytdl_sub.script.utils.exceptions import KeyNotHashableRuntimeException + + +class MapFunctions: + @staticmethod + def map(maybe_mapping: AnyArgument) -> Map: + """ + Tries to cast an unknown variable type to a Map. + + Raises + ------ + FunctionRuntimeException + If the input type is not actually a Map. + """ + if not isinstance(maybe_mapping, Map): + raise FunctionRuntimeException( + f"Tried and failed to cast {maybe_mapping.type_name()} as a Map" + ) + return maybe_mapping + + @staticmethod + def map_size(mapping: Map) -> Integer: + """ + Returns the size of a Map. + """ + return Integer(len(mapping.value)) + + @staticmethod + def map_contains(mapping: Map, key: AnyArgument) -> Boolean: + """ + Returns True if the key is in the Map. False otherwise. + """ + if not isinstance(key, Hashable): + raise KeyNotHashableRuntimeException( + f"Tried to use {key.type_name()} as a Map key, but it is not hashable." + ) + + return Boolean(key in mapping.value) + + @staticmethod + def map_get( + mapping: Map, key: AnyArgument, default: Optional[AnyArgument] = None + ) -> AnyArgument: + """ + Return ``key``'s value within the Map. If ``key`` does not exist, and ``default`` is + provided, it will return ``default``. Otherwise, will error. + """ + if not MapFunctions.map_contains(mapping=mapping, key=key).value: + if default is not None: + return default + + raise KeyDoesNotExistRuntimeException( + f"Tried to call %map_get with key {key.value}, but it does not exist" + ) + return mapping.value[key] + + @staticmethod + def map_get_non_empty(mapping: Map, key: AnyArgument, default: AnyArgument) -> AnyArgument: + """ + Return ``key``'s value within the Map. If ``key`` does not exist or is an empty string, + return ``default``. Otherwise, will error. + """ + output = MapFunctions.map_get(mapping, key, default) + if isinstance(output, String) and output.value == "": + return default + return output + + # pylint: disable=unused-argument + + @staticmethod + def map_apply(mapping: Map, lambda_function: LambdaTwo) -> Array: + """ + Apply a lambda function on the Map, where each arg + passed to the lambda function is ``key, value`` as two separate args. + """ + return Array([Array([key, value]) for key, value in mapping.value.items()]) + + @staticmethod + def map_enumerate(mapping: Map, lambda_function: LambdaThree) -> Array: + """ + Apply a lambda function on the Map, where each arg + passed to the lambda function is ``idx, key, value`` as three separate args. + """ + return Array( + [ + Array([Integer(idx), key_value[0], key_value[1]]) + for idx, key_value in enumerate(mapping.value.items()) + ] + ) diff --git a/src/ytdl_sub/script/functions/numeric_functions.py b/src/ytdl_sub/script/functions/numeric_functions.py new file mode 100644 index 00000000..5211c690 --- /dev/null +++ b/src/ytdl_sub/script/functions/numeric_functions.py @@ -0,0 +1,88 @@ +import math + +from ytdl_sub.script.types.resolvable import AnyArgument +from ytdl_sub.script.types.resolvable import Float +from ytdl_sub.script.types.resolvable import Integer +from ytdl_sub.script.types.resolvable import Numeric + + +def _to_numeric(value: int | float) -> Numeric: + if int(value) == value: + return Integer(value=value) + return Float(value=value) + + +class NumericFunctions: + @staticmethod + def float(value: AnyArgument) -> Float: + """ + Cast to Float. + """ + return Float(value=float(value.value)) + + @staticmethod + def int(value: AnyArgument) -> Integer: + """ + Cast to Integer. + """ + return Integer(value=int(value.value)) + + @staticmethod + def add(*values: Numeric) -> Numeric: + """ + ``+`` operator. Returns the sum of all values. + """ + return _to_numeric(sum(val.value for val in values)) + + @staticmethod + def sub(*values: Numeric) -> Numeric: + """ + ``-`` operator. Subtracts all values from left to right. + """ + output = values[0].value + for val in values[1:]: + output -= val.value + + return _to_numeric(output) + + @staticmethod + def mul(*values: Numeric) -> Numeric: + """ + ``*`` operator. Returns the product of all values. + """ + return _to_numeric(math.prod([val.value for val in values])) + + @staticmethod + def pow(base: Numeric, exponent: Numeric) -> Numeric: + """ + ``**`` operator. Returns the exponential of the base and exponent value. + """ + return _to_numeric(math.pow(base.value, exponent.value)) + + @staticmethod + def div(left: Numeric, right: Numeric) -> Numeric: + """ + ``/`` operator. Returns ``left / right``. + """ + return _to_numeric(left.value / right.value) + + @staticmethod + def mod(left: Numeric, right: Numeric) -> Numeric: + """ + ``%`` operator. Returns ``left % right``. + """ + return _to_numeric(value=left.value % right.value) + + @staticmethod + def max(*values: Numeric) -> Numeric: + """ + Returns max of all values. + """ + return _to_numeric(max(val.value for val in values)) + + @staticmethod + def min(*values: Numeric) -> Numeric: + """ + Returns min of all values. + """ + return _to_numeric(min(val.value for val in values)) diff --git a/src/ytdl_sub/script/functions/regex_functions.py b/src/ytdl_sub/script/functions/regex_functions.py new file mode 100644 index 00000000..2abca1ec --- /dev/null +++ b/src/ytdl_sub/script/functions/regex_functions.py @@ -0,0 +1,42 @@ +import re +from typing import AnyStr +from typing import Match + +from ytdl_sub.script.types.array import Array +from ytdl_sub.script.types.resolvable import String + + +def _re_output_to_array(re_out: Match[AnyStr] | None) -> Array: + if re_out is None: + return Array([]) + + return Array(list([String(re_out.string)]) + list(String(group) for group in re_out.groups())) + + +class RegexFunctions: + @staticmethod + def regex_match(regex: String, string: String) -> Array: + """ + Checks for a match only at the beginning of the string. If a match exists, returns + the string as the first element of the Array. If there are capture groups, returns each + group as a subsequent element in the Array. + """ + return _re_output_to_array(re.match(regex.value, string.value)) + + @staticmethod + def regex_search(regex: String, string: String) -> Array: + """ + Checks for a match anywhere in the string. If a match exists, returns + the string as the first element of the Array. If there are capture groups, returns each + group as a subsequent element in the Array. + """ + return _re_output_to_array(re.search(regex.value, string.value)) + + @staticmethod + def regex_fullmatch(regex: String, string: String) -> Array: + """ + Checks for entire string to be a match. If a match exists, returns + the string as the first element of the Array. If there are capture groups, returns each + group as a subsequent element in the Array. + """ + return _re_output_to_array(re.fullmatch(regex.value, string.value)) diff --git a/src/ytdl_sub/script/functions/string_functions.py b/src/ytdl_sub/script/functions/string_functions.py new file mode 100644 index 00000000..be3b52c2 --- /dev/null +++ b/src/ytdl_sub/script/functions/string_functions.py @@ -0,0 +1,94 @@ +from typing import Optional + +from ytdl_sub.script.types.resolvable import AnyArgument +from ytdl_sub.script.types.resolvable import Integer +from ytdl_sub.script.types.resolvable import Numeric +from ytdl_sub.script.types.resolvable import String + + +class StringFunctions: + @staticmethod + def string(value: AnyArgument) -> String: + """ + Cast to String. + """ + return String(value=str(value.value)) + + @staticmethod + def slice(string: String, start: Integer, end: Optional[Integer] = None) -> String: + """ + Returns the slice of the Array. + """ + if end is not None: + return String(string.value[start.value : end.value]) + return String(string.value[start.value :]) + + @staticmethod + def lower(string: String) -> String: + """ + Lower-case the entire String. + """ + return String(string.value.lower()) + + @staticmethod + def upper(string: String) -> String: + """ + Upper-case the entire String. + """ + return String(string.value.upper()) + + @staticmethod + def capitalize(string: String) -> String: + """ + Capitalize the first character in the string. + """ + return String(string.value.capitalize()) + + @staticmethod + def titlecase(string: String) -> String: + """ + Capitalize each word in the string. + """ + return String(string.value.title()) + + @staticmethod + def replace( + string: String, old: String, new: String, count: Optional[Integer] = None + ) -> String: + """ + Replace the ``old`` part of the String with the ``new``. Optionally only replace it + ``count`` number of times. + """ + if count: + return String(string.value.replace(old.value, new.value, count.value)) + + return String(string.value.replace(old.value, new.value)) + + @staticmethod + def concat(*values: String) -> String: + """ + Concatenate multiple Strings into a single String. + """ + return String("".join(val.value for val in values)) + + @staticmethod + def pad(string: String, length: Integer, char: String) -> String: + """ + Pads the string to the given length + """ + output = string.value + while len(output) < length.value: + output = f"{char}{output}" + + return String(output) + + @staticmethod + def pad_zero(numeric: Numeric, length: Integer) -> String: + """ + Pads a numeric with zeros to the given length + """ + return StringFunctions.pad( + string=String(str(numeric.value)), + length=length, + char=String("0"), + ) diff --git a/src/ytdl_sub/script/parser.py b/src/ytdl_sub/script/parser.py new file mode 100644 index 00000000..82b66bd9 --- /dev/null +++ b/src/ytdl_sub/script/parser.py @@ -0,0 +1,601 @@ +import json +from enum import Enum +from typing import Dict +from typing import List +from typing import Optional +from typing import Set + +from ytdl_sub.script.functions import Functions +from ytdl_sub.script.types.array import UnresolvedArray +from ytdl_sub.script.types.function import Argument +from ytdl_sub.script.types.function import BuiltInFunction +from ytdl_sub.script.types.function import CustomFunction +from ytdl_sub.script.types.function import Function +from ytdl_sub.script.types.map import UnresolvedMap +from ytdl_sub.script.types.resolvable import Boolean +from ytdl_sub.script.types.resolvable import Float +from ytdl_sub.script.types.resolvable import Integer +from ytdl_sub.script.types.resolvable import Lambda +from ytdl_sub.script.types.resolvable import NonHashable +from ytdl_sub.script.types.resolvable import String +from ytdl_sub.script.types.syntax_tree import SyntaxTree +from ytdl_sub.script.types.variable import FunctionArgument +from ytdl_sub.script.types.variable import Variable +from ytdl_sub.script.utils.exception_formatters import ParserExceptionFormatter +from ytdl_sub.script.utils.exceptions import UNREACHABLE +from ytdl_sub.script.utils.exceptions import CycleDetected +from ytdl_sub.script.utils.exceptions import FunctionDoesNotExist +from ytdl_sub.script.utils.exceptions import IncompatibleFunctionArguments +from ytdl_sub.script.utils.exceptions import InvalidCustomFunctionArgumentName +from ytdl_sub.script.utils.exceptions import InvalidSyntaxException +from ytdl_sub.script.utils.exceptions import InvalidVariableName +from ytdl_sub.script.utils.exceptions import UserException +from ytdl_sub.script.utils.exceptions import VariableDoesNotExist +from ytdl_sub.script.utils.name_validation import validate_variable_name + +# pylint: disable=invalid-name +# pylint: disable=too-many-branches +# pylint: disable=too-many-return-statements +# pylint: disable=consider-using-ternary + + +class ParsedArgType(Enum): + SCRIPT = "script" + FUNCTION = "function" + ARRAY = "array" + MAP_KEY = "map key" + MAP_VALUE = "map value" + + +BRACKET_NOT_CLOSED = InvalidSyntaxException("Bracket not properly closed") + +NUMERICS_ONLY_ARGS = InvalidSyntaxException( + "Numerics can only be used as arguments to functions, maps, or arrays" +) +NUMERICS_INVALID_CHAR = InvalidSyntaxException("Invalid value when parsing a numeric") + + +STRINGS_ONLY_ARGS = InvalidSyntaxException( + "Strings can only be used as arguments to functions, maps, or arrays" +) +STRINGS_NOT_CLOSED = InvalidSyntaxException( + "String was not closed properly. " + "Must open and close with the same type of quote (single/double)" +) + +BOOLEAN_ONLY_ARGS = InvalidSyntaxException( + "Booleans can only be used as arguments to functions, maps, or arrays" +) + +CUSTOM_FUNCTION_ARGUMENTS_ONLY_ARGS = InvalidSyntaxException( + "Custom function arguments can only be used as arguments to functions, maps, or arrays" +) + +FUNCTION_INVALID_CHAR = InvalidSyntaxException("Invalid value when parsing a function") + + +def _UNEXPECTED_CHAR_ARGUMENT(arg_type: ParsedArgType): + return InvalidSyntaxException(f"Unexpected character when parsing {arg_type.value} arguments") + + +def _UNEXPECTED_COMMA_ARGUMENT(arg_type: ParsedArgType): + return InvalidSyntaxException(f"Unexpected comma when parsing {arg_type.value} arguments") + + +MAP_KEY_WITH_NO_VALUE = InvalidSyntaxException("Map has a key with no value") +MAP_KEY_MULTIPLE_VALUES = InvalidSyntaxException( + "Map key has multiple values when there should only be one" +) +MAP_MISSING_KEY = InvalidSyntaxException("Map has a missing key") +MAP_KEY_NOT_HASHABLE = InvalidSyntaxException( + "Map key must be a hashable type (Integer, Float, Boolean, String)" +) + + +def _is_variable_start(char: str) -> bool: + return char.isalpha() and char.islower() + + +def _is_function_name_char(char: str) -> bool: + return (char.isalpha() and char.islower()) or char.isnumeric() or char == "_" + + +def _is_numeric_start(char: str) -> bool: + return char.isnumeric() or char in (".", "-") + + +def _is_string_start_single_char(char: Optional[str]) -> bool: + return char in ["'", '"'] + + +def _is_string_start_multi_char(string: Optional[str]) -> bool: + return string in ["'''", '"""'] + + +def _is_null(string: Optional[str]) -> bool: + return string and string.lower() == "null" + + +def _is_breakable(char: str) -> bool: + return char in ["}", ",", ")", "]", ":"] or char.isspace() + + +def _is_boolean_true(string: Optional[str]) -> bool: + return string and string.lower() == "true" + + +def _is_boolean_false(string: Optional[str]) -> bool: + return string and string.lower() == "false" + + +def _is_custom_function_argument_start(char: str) -> bool: + return char == "$" + + +class _Parser: + def __init__( + self, + text: str, + name: Optional[str], + custom_function_names: Optional[Set[str]], + variable_names: Optional[Set[str]], + ): + self._text = text + self._name = name + self._custom_function_names = custom_function_names + self._variable_names = variable_names + self._pos = 0 + self._error_highlight_pos = 0 + self._ast: List[Argument] = [] + + self._bracket_counter_pos_stack: List[int] = [] + self._bracket_counter = 0 + self._literal_str = "" + + try: + self._syntax_tree = self._parse() + except UserException as exc: + raise ParserExceptionFormatter( + self._text, self._error_highlight_pos, self._pos, exc + ).highlight() from exc + + @property + def ast(self) -> SyntaxTree: + """ + Returns + ------- + Abstract syntax tree of the parsed text + """ + return self._syntax_tree + + def _set_highlight_position(self, pos: Optional[int] = None) -> None: + self._error_highlight_pos = pos if pos is not None else self._pos + + def _read(self, increment_pos: bool = True, length: int = 1) -> Optional[str]: + if isinstance(self._text, int): + pass + if self._pos >= len(self._text): + return None + + try: + ch = self._text[self._pos : (self._pos + length)] + except IndexError: + return None + + if increment_pos: + self._pos += length + return ch + + def _parse_variable(self) -> Variable: + var_name = "" + variable_start_pos = self._pos + while ch := self._read(increment_pos=False): + if ch.isspace() and not var_name: + self._pos += 1 + continue + if _is_breakable(ch): + break + + if not var_name: + variable_start_pos = self._pos + + var_name += ch + self._pos += 1 + + try: + validate_variable_name(var_name) + except InvalidVariableName: + self._set_highlight_position(variable_start_pos) + raise + + if self._variable_names is not None and var_name not in self._variable_names: + self._set_highlight_position(variable_start_pos) + raise VariableDoesNotExist(f"Variable {var_name} does not exist.") + + return Variable(var_name) + + def _parse_custom_function_argument(self) -> FunctionArgument: + """ + Begin parsing function args after the first ``$``, i.e. ``$0`` + """ + var_name = "" + variable_start_pos = self._pos + while ch := self._read(increment_pos=False): + if ch.isspace() and not var_name: + raise InvalidCustomFunctionArgumentName( + "Custom function arguments, denoted by $, cannot have a space proceeding it." + ) + if _is_breakable(ch): + break + + var_name += ch + self._pos += 1 + + if not var_name.isnumeric(): + self._set_highlight_position(variable_start_pos) + raise InvalidCustomFunctionArgumentName( + "Custom function arguments must be numeric and increment starting from zero." + ) + + return FunctionArgument.from_idx(idx=int(var_name), custom_function_name=self._name) + + def _parse_numeric(self) -> Integer | Float: + numeric_string = "" + + if self._read(increment_pos=False) == "-": + numeric_string += "-" + self._pos += 1 + if has_decimal := (self._read(increment_pos=False) == "."): + numeric_string += "." + self._pos += 1 + + while ch := self._read(increment_pos=False): + if ch == "-": + raise NUMERICS_INVALID_CHAR + + if ch == ".": + if has_decimal: + raise NUMERICS_INVALID_CHAR + has_decimal = True + + self._pos += 1 + numeric_string += ch + elif ch.isnumeric(): + self._pos += 1 + numeric_string += ch + elif _is_breakable(ch): + break + else: + self._set_highlight_position() + raise NUMERICS_INVALID_CHAR + + if numeric_string in (".", "-"): + raise NUMERICS_INVALID_CHAR + + try: + numeric_float = float(numeric_string) + except ValueError as exc: + raise UNREACHABLE from exc + + if (numeric_int := int(numeric_float)) == numeric_float: + return Integer(value=numeric_int) + + return Float(value=numeric_float) + + def _parse_string(self, str_open_token: str) -> String: + """ + Begin parsing a string, including the quotation value + """ + self._set_highlight_position() + string_value = "" + + if not _is_string_start_single_char(str_open_token) and not _is_string_start_multi_char( + str_open_token + ): + raise UNREACHABLE + + while ch := self._read(increment_pos=False): + if self._read(increment_pos=False, length=len(str_open_token)) == str_open_token: + self._pos += len(str_open_token) + return String(value=string_value) + + self._pos += 1 + string_value += ch + + raise STRINGS_NOT_CLOSED + + def _parse_function_arg(self, argument_parser: ParsedArgType) -> Argument: + if self._read(increment_pos=False) == "%": + self._pos += 1 + return self._parse_function() + if _is_numeric_start(self._read(increment_pos=False)): + return self._parse_numeric() + if _is_boolean_true(self._read(increment_pos=False, length=4)): + self._pos += 4 + return Boolean(value=True) + if _is_boolean_false(self._read(increment_pos=False, length=5)): + self._pos += 5 + return Boolean(value=False) + if _is_null(self._read(increment_pos=False, length=4)): + self._pos += 4 + return String(value="") + if _is_string_start_multi_char(str_open_token := self._read(increment_pos=False, length=3)): + self._pos += 3 + return self._parse_string(str_open_token) + if _is_string_start_single_char(str_open_token := self._read(increment_pos=False)): + self._pos += 1 + return self._parse_string(str_open_token) + if self._read(increment_pos=False) == "[": + self._pos += 1 + return self._parse_array() + if self._read(increment_pos=False) == "{": + self._pos += 1 + return self._parse_map() + if _is_custom_function_argument_start(self._read(increment_pos=False)): + self._pos += 1 + return self._parse_custom_function_argument() + if _is_variable_start(self._read(increment_pos=False)): + return self._parse_variable() + + self._set_highlight_position() + raise _UNEXPECTED_CHAR_ARGUMENT(arg_type=argument_parser) + + def _parse_args( + self, argument_parser: ParsedArgType, breaking_chars: str = ")" + ) -> List[Argument]: + """ + Begin parsing function args after the first ``(``, i.e. ``function_name(`` + """ + comma_count = 0 + arguments: List[Argument] = [] + while ch := self._read(increment_pos=False): + if ch in breaking_chars: + # i.e. ["arg", ] which is invalid + if arguments and len(arguments) == comma_count: + raise _UNEXPECTED_COMMA_ARGUMENT(argument_parser) + break + + if ch.isspace(): + self._pos += 1 + elif ch == ",": + self._set_highlight_position() + comma_count += 1 + if len(arguments) != comma_count: + raise _UNEXPECTED_COMMA_ARGUMENT(argument_parser) + + self._pos += 1 + else: + arguments.append(self._parse_function_arg(argument_parser=argument_parser)) + + return arguments + + def _parse_function(self) -> Function | Lambda: + """ + Begin parsing a function after reading the first ``%`` + """ + function_name: str = "" + function_args: Optional[List[Argument]] = None + function_start_pos = self._pos + + while ch := self._read(): + if ch == ")": + # Had '(' to indicate there are args + if function_args is not None: + if self._name == function_name: + self._set_highlight_position(function_start_pos) + raise CycleDetected( + f"The custom function %{function_name} cannot call itself." + ) + + if Functions.is_built_in(function_name): + try: + return BuiltInFunction( + name=function_name, args=function_args + ).validate_args() + except IncompatibleFunctionArguments: + self._set_highlight_position(function_start_pos) + raise + + # Is custom function + if ( + self._custom_function_names is not None + and function_name not in self._custom_function_names + ): + self._set_highlight_position(function_start_pos) + raise FunctionDoesNotExist( + f"Function %{function_name} does not exist as a built-in or " + "custom function." + ) + + return CustomFunction( + name=function_name, + args=function_args, + ) + + # Go back one so the parent function can close using the ')' + self._pos -= 1 + return Lambda(value=function_name) + + if _is_function_name_char(ch): + function_name += ch + elif ch == "(": + function_args = self._parse_args(argument_parser=ParsedArgType.FUNCTION) + elif ch.isspace() or ch == ",": + # function with no args, it's a lambda + return Lambda(value=function_name) + else: + break + + self._set_highlight_position(pos=self._pos - 1) + raise FUNCTION_INVALID_CHAR + + def _parse_array(self) -> UnresolvedArray: + """ + Begin parsing an array after reading the first ``[`` + """ + function_args: List[Argument] = [] + + while ch := self._read(increment_pos=False): + if ch == "]": + self._pos += 1 + return UnresolvedArray(value=function_args) + + function_args = self._parse_args( + argument_parser=ParsedArgType.ARRAY, breaking_chars="]" + ) + + raise UNREACHABLE + + def _parse_map(self) -> UnresolvedMap: + """ + Begin parsing a map after reading the first ``{`` + """ + output: Dict[Argument, Argument] = {} + key: Optional[Argument] = None + in_comma = False + + self._set_highlight_position() + while ch := self._read(increment_pos=False): + if ch == "}": + if key is not None: + raise MAP_KEY_WITH_NO_VALUE # key args are parsed immediately + + self._pos += 1 + return UnresolvedMap(value=output) + + if ch == ",": + if in_comma: + raise _UNEXPECTED_COMMA_ARGUMENT(ParsedArgType.MAP_KEY) + if key is not None: + raise MAP_KEY_WITH_NO_VALUE # key args are parsed immediately + if not output: + raise _UNEXPECTED_COMMA_ARGUMENT(ParsedArgType.MAP_KEY) + in_comma = True + self._pos += 1 + elif key is None: + self._set_highlight_position() + in_comma = False + key_args = self._parse_args( + argument_parser=ParsedArgType.MAP_KEY, breaking_chars=":}" + ) + + if len(key_args) == 0 and self._read(increment_pos=False) == "}": + continue # will return the map next iteration + if len(key_args) == 0: + raise MAP_MISSING_KEY + if len(key_args) > 1: + raise MAP_KEY_MULTIPLE_VALUES + key = key_args[0] + elif key is not None and ch == ":": + self._set_highlight_position() + self._pos += 1 + value_args = self._parse_args( + argument_parser=ParsedArgType.MAP_VALUE, breaking_chars=",}" + ) + if len(value_args) == 0: + raise MAP_KEY_WITH_NO_VALUE + if isinstance(key, NonHashable): + raise MAP_KEY_NOT_HASHABLE + if len(value_args) > 1: + raise MAP_KEY_MULTIPLE_VALUES + + output[key] = value_args[0] + key = None + else: + raise UNREACHABLE + + def _parse_main_loop(self, ch: str) -> bool: + if ch == "}": + if self._bracket_counter == 0: + raise BRACKET_NOT_CLOSED + + del self._bracket_counter_pos_stack[-1] + self._bracket_counter -= 1 + return True + if ch == "{": + self._bracket_counter_pos_stack.append(self._pos - 1) # pos incremented when read + self._bracket_counter += 1 + if self._literal_str: + self._ast.append(String(value=self._literal_str)) + self._literal_str = "" + + # Allow whitespace after bracket opening + while ch1 := self._read(increment_pos=False): + if not ch1.isspace(): + break + self._pos += 1 + + if ch1 is None: + return False # will hit closing bracket error + + if ch1 == "%": + self._pos += 1 + self._ast.append(self._parse_function()) + elif ch1 == "[": + self._pos += 1 + self._ast.append(self._parse_array()) + elif ch1 == "{": + self._pos += 1 + self._ast.append(self._parse_map()) + elif _is_variable_start(ch1): + self._ast.append(self._parse_variable()) + elif _is_numeric_start(ch1): + raise NUMERICS_ONLY_ARGS + elif ( + _is_string_start_single_char(ch1) + or _is_string_start_multi_char(self._read(increment_pos=False, length=3)) + or _is_null(self._read(increment_pos=False, length=4)) + ): + raise STRINGS_ONLY_ARGS + elif _is_boolean_true(self._read(increment_pos=False, length=4)) or _is_boolean_false( + self._read(increment_pos=False, length=5) + ): + raise BOOLEAN_ONLY_ARGS + elif _is_custom_function_argument_start(self._read(increment_pos=False)): + raise CUSTOM_FUNCTION_ARGUMENTS_ONLY_ARGS + else: + raise _UNEXPECTED_CHAR_ARGUMENT(arg_type=ParsedArgType.SCRIPT) + elif self._bracket_counter == 0: + # Only accumulate literal str if not in brackets + self._literal_str += ch + else: + # Should only be possible to get here if it's a space + assert ch.isspace() + + return True + + def _parse(self) -> SyntaxTree: + + while ch := self._read(): + continue_parse = self._parse_main_loop(ch) + if not continue_parse: + break + + if self._bracket_counter != 0: + self._error_highlight_pos = self._bracket_counter_pos_stack[-1] + raise BRACKET_NOT_CLOSED + + if self._literal_str: + self._ast.append(String(value=self._literal_str)) + + return SyntaxTree(ast=self._ast) + + +def parse( + text: str, + name: Optional[str] = None, + custom_function_names: Optional[Set[str]] = None, + variable_names: Optional[Set[str]] = None, +) -> SyntaxTree: + """ + Entrypoint for parsing ytdl-sub code into a Syntax Tree + """ + return _Parser( + text=json.dumps(text) if not isinstance(text, str) else text, + name=name, + custom_function_names=custom_function_names, + variable_names=variable_names, + ).ast + + +# pylint: enable=invalid-name diff --git a/src/ytdl_sub/script/script.py b/src/ytdl_sub/script/script.py new file mode 100644 index 00000000..2747f003 --- /dev/null +++ b/src/ytdl_sub/script/script.py @@ -0,0 +1,484 @@ +# pylint: disable=missing-raises-doc +from typing import Dict +from typing import List +from typing import Optional +from typing import Set + +from ytdl_sub.script.functions import Functions +from ytdl_sub.script.parser import parse +from ytdl_sub.script.script_output import ScriptOutput +from ytdl_sub.script.types.resolvable import Lambda +from ytdl_sub.script.types.resolvable import Resolvable +from ytdl_sub.script.types.syntax_tree import SyntaxTree +from ytdl_sub.script.types.variable import Variable +from ytdl_sub.script.utils.exceptions import UNREACHABLE +from ytdl_sub.script.utils.exceptions import CycleDetected +from ytdl_sub.script.utils.exceptions import IncompatibleFunctionArguments +from ytdl_sub.script.utils.exceptions import InvalidCustomFunctionArguments +from ytdl_sub.script.utils.exceptions import RuntimeException +from ytdl_sub.script.utils.name_validation import validate_variable_name +from ytdl_sub.script.utils.type_checking import FunctionSpec + + +def _is_function(override_name: str): + return override_name.startswith("%") + + +def _function_name(function_key: str) -> str: + """ + Drop the % in %custom_function + """ + return function_key[1:] + + +def _to_function_definition_name(function_key: str) -> str: + """ + Add % in %custom_function + """ + return f"%{function_key}" + + +class Script: + """ + Takes a dictionary of both + ``{ variable_names: syntax }`` + and + ``{ %custom_function: syntax }`` + """ + + def _ensure_no_cycle( + self, name: str, dep: str, deps: List[str], definitions: Dict[str, SyntaxTree] + ): + if dep not in definitions: + return # does not exist, will throw downstream in parser + + if name in deps + [dep]: + type_name, pre = ( + ("custom functions", "%") if definitions is self._functions else ("variables", "") + ) + cycle_deps = [name] + deps + [dep] + cycle_deps_str = " -> ".join([f"{pre}{name}" for name in cycle_deps]) + + raise CycleDetected(f"Cycle detected within these {type_name}: {cycle_deps_str}") + + def _traverse_variable_dependencies( + self, + variable_name: str, + variable_dependency: SyntaxTree, + deps: List[str], + ) -> None: + for dep in variable_dependency.variables: + self._ensure_no_cycle( + name=variable_name, dep=dep.name, deps=deps, definitions=self._variables + ) + self._traverse_variable_dependencies( + variable_name=variable_name, + variable_dependency=self._variables[dep.name], + deps=deps + [dep.name], + ) + + def _ensure_no_variable_cycles(self, variables: Dict[str, SyntaxTree]): + for variable_name, variable_definition in variables.items(): + self._traverse_variable_dependencies( + variable_name=variable_name, + variable_dependency=variable_definition, + deps=[], + ) + + def _traverse_custom_function_dependencies( + self, + custom_function_name: str, + custom_function_dependency: SyntaxTree, + deps: List[str], + ) -> None: + for dep in custom_function_dependency.custom_functions: + self._ensure_no_cycle( + name=custom_function_name, dep=dep.name, deps=deps, definitions=self._functions + ) + self._traverse_custom_function_dependencies( + custom_function_name=custom_function_name, + custom_function_dependency=self._functions[dep.name], + deps=deps + [dep.name], + ) + + def _ensure_no_custom_function_cycles(self): + for custom_function_name, custom_function in self._functions.items(): + self._traverse_custom_function_dependencies( + custom_function_name=custom_function_name, + custom_function_dependency=custom_function, + deps=[], + ) + + def _ensure_custom_function_arguments_valid(self): + for custom_function_name, custom_function in self._functions.items(): + indices = sorted([arg.index for arg in custom_function.function_arguments]) + if indices != list(range(len(indices))): + if len(indices) == 1: + raise InvalidCustomFunctionArguments( + f"Custom function %{custom_function_name} has invalid function arguments: " + f"The argument must start with $0, not ${indices[0]}." + ) + raise InvalidCustomFunctionArguments( + f"Custom function %{custom_function_name} has invalid function arguments: " + f"{', '.join(sorted(f'${idx}' for idx in indices))} " + f"do not increment from $0 to ${len(indices) - 1}." + ) + + def _ensure_custom_function_usage_num_input_arguments_valid( + self, prefix: str, name: str, definition: SyntaxTree + ): + for nested_custom_function in definition.custom_functions: + if nested_custom_function.num_input_args != ( + expected_num_args := len( + self._functions[nested_custom_function.name].function_arguments + ) + ): + raise InvalidCustomFunctionArguments( + f"{prefix}{name} has invalid usage of the custom " + f"function %{nested_custom_function.name}: Expects {expected_num_args} " + f"argument{'s' if expected_num_args > 1 else ''} but received " + f"{nested_custom_function.num_input_args}" + ) + + def _ensure_lambda_usage_num_input_arguments_valid( + self, prefix: str, name: str, definition: SyntaxTree + ): + for function in definition.built_in_functions: + spec = FunctionSpec.from_callable(Functions.get(function.name)) + if not (lambda_type := spec.is_lambda_like): + return + + lambda_function_names = set( + lamb.value for lamb in SyntaxTree(function.args).lambdas if isinstance(lamb, Lambda) + ) + + # Only case len(lambda_function_names) > 1 is when used in if-statements + for lambda_function_name in lambda_function_names: + if Functions.is_built_in(lambda_function_name): + lambda_spec = FunctionSpec.from_callable(Functions.get(lambda_function_name)) + if not lambda_spec.is_num_args_compatible(lambda_type.num_input_args()): + expected_args_str = str(lambda_spec.num_required_args) + if lambda_spec.num_required_args != len(lambda_spec.args): + expected_args_str = f"{expected_args_str} - {len(lambda_spec.args)}" + + raise IncompatibleFunctionArguments( + f"{prefix}{name} has invalid usage of the " + f"function %{lambda_function_name} as a lambda: " + f"Expects {expected_args_str} " + f"argument{'s' if expected_args_str != '1' else ''} but will " + f"receive {lambda_type.num_input_args()}." + ) + else: # is custom function + if lambda_function_name not in self._functions: + raise UNREACHABLE # Custom function should have been validated + + expected_num_arguments = len( + self._functions[lambda_function_name].function_arguments + ) + if lambda_type.num_input_args() != expected_num_arguments: + raise IncompatibleFunctionArguments( + f"{prefix}{name} has invalid usage of the custom " + f"function %{lambda_function_name} as a lambda: " + f"Expects {expected_num_arguments} " + f"argument{'s' if expected_num_arguments > 1 else ''} but will " + f"receive {lambda_type.num_input_args()}." + ) + + def _validate(self, added_variables: Optional[Set[str]] = None) -> None: + variables = self._variables + if added_variables is not None: + variables = { + name: ast for name, ast in self._variables.items() if name in added_variables + } + + if added_variables is None: + self._ensure_no_custom_function_cycles() + self._ensure_custom_function_arguments_valid() + + self._ensure_no_variable_cycles(variables) + + to_validate = [("Variable ", variables)] + if added_variables is None: + to_validate.append(("Custom function %", self._functions)) + + for prefix, definitions in to_validate: + for name, definition in definitions.items(): + self._ensure_custom_function_usage_num_input_arguments_valid( + prefix=prefix, name=name, definition=definition + ) + self._ensure_lambda_usage_num_input_arguments_valid( + prefix=prefix, name=name, definition=definition + ) + + def __init__(self, script: Dict[str, str]): + function_names: Set[str] = { + _function_name(name) for name in script.keys() if _is_function(name) + } + variable_names: Set[str] = { + validate_variable_name(name) for name in script.keys() if not _is_function(name) + } + + self._functions: Dict[str, SyntaxTree] = { + # custom_function_name must be passed to properly type custom function + # arguments uniquely if they're nested (i.e. $0 to $custom_func___0) + _function_name(function_key): parse( + text=function_value, + name=_function_name(function_key), + custom_function_names=function_names, + variable_names=variable_names, + ) + for function_key, function_value in script.items() + if _is_function(function_key) + } + + self._variables: Dict[str, SyntaxTree] = { + variable_key: parse( + text=variable_value, + name=variable_key, + custom_function_names=function_names, + variable_names=variable_names, + ) + for variable_key, variable_value in script.items() + if not _is_function(variable_key) + } + self._validate() + + def _update_internally(self, resolved_variables: Dict[str, Resolvable]) -> None: + for variable_name, resolved in resolved_variables.items(): + self._variables[variable_name] = SyntaxTree(ast=[resolved]) + + def _resolve( + self, + pre_resolved: Optional[Dict[str, Resolvable]] = None, + unresolvable: Optional[Set[str]] = None, + update: bool = False, + output_filter: Optional[Set[str]] = None, + ) -> ScriptOutput: + """ + Parameters + ---------- + pre_resolved + Optional. Variables that have been resolved elsewhere and could be used in this script + unresolvable + Optional. Variables that cannot be resolved, forcing any variable that depends on it + to not be resolved. + update + Optional. Whether to update the internal representation of variables with their + resolved value (if they get resolved). + + Returns + ------- + Dict of resolved values + """ + resolved: Dict[Variable, Resolvable] = { + Variable(name): value for name, value in (pre_resolved or {}).items() + } + + unresolvable: Set[Variable] = {Variable(name) for name in (unresolvable or {})} + unresolved_filter = set(resolved.keys()).union(unresolvable) + unresolved: Dict[Variable, SyntaxTree] = { + Variable(name): ast + for name, ast in self._variables.items() + if Variable(name) not in unresolved_filter + } + + while unresolved: + unresolved_count: int = len(unresolved) + + for variable in list(unresolved.keys()): + definition = unresolved[variable] + + # If the definition is already a resolvable, mark it as such + if resolvable := definition.maybe_resolvable: + resolved[variable] = resolvable + del unresolved[variable] + + # If the variable's variable dependencies contain an unresolvable variable, + # declare it as unresolvable and continue + elif definition.contains(unresolvable): + unresolvable.add(variable) + del unresolved[variable] + + # Otherwise, if it has dependencies that are all resolved, then + # resolve the definition + elif not definition.is_subset_of(variables=resolved.keys()): + resolved[variable] = unresolved[variable].resolve( + resolved_variables=resolved, + custom_functions=self._functions, + ) + del unresolved[variable] + + if len(unresolved) == unresolved_count: + # Implies a cycle within the variables. Should never reach + # since cycles are detected in __init__ + raise UNREACHABLE + + resolved_variables = { + variable.name: resolvable for variable, resolvable in resolved.items() + } + if update: + self._update_internally(resolved_variables=resolved_variables) + + if output_filter: + for name in output_filter: + if name not in resolved_variables: + raise ValueError(f"Specified {name} to resolve, but it did not") + + return ScriptOutput( + { + name: resolvable + for name, resolvable in resolved_variables.items() + if name in output_filter + } + ) + + return ScriptOutput(resolved_variables) + + def resolve( + self, + resolved: Optional[Dict[str, Resolvable]] = None, + unresolvable: Optional[Set[str]] = None, + update: bool = False, + ) -> ScriptOutput: + """ + Resolves the script + + Parameters + ---------- + resolved + Optional. Pre-resolved variables that should be used instead of what is in the script. + unresolvable + Optional. Unresolvable variables that will be ignored in resolution, including all + variables with a dependency to them. + update + Whether to update the script's internal values with the resolved variables instead of + their original definition. This helps avoid re-evaluated the same variables repeatedly. + + Returns + ------- + ScriptOutput + Containing all resolved variables. + """ + return self._resolve( + pre_resolved=resolved, unresolvable=unresolvable, update=update, output_filter=None + ) + + def add(self, variables: Dict[str, str], unresolvable: Optional[Set[str]] = None) -> "Script": + """ + Adds parses and adds new variables to the script. + + Parameters + ---------- + variables + Mapping containing variable name to definition. + unresolvable + Optional. Set of unresolved variables that the new variables may contain, but the + script does not (yet). + + Returns + ------- + Script + self + """ + added_variables_to_validate: Set[str] = set() + for variable_name, variable_definition in variables.items(): + self._variables[variable_name] = parse( + text=variable_definition, + name=variable_name, + custom_function_names=set(self._functions.keys()), + variable_names=set(self._variables.keys()) + .union(variables.keys()) + .union(unresolvable or set()), + ) + + if self._variables[variable_name].maybe_resolvable is None: + added_variables_to_validate.add(variable_name) + + if added_variables_to_validate: + self._validate(added_variables=added_variables_to_validate) + + return self + + def resolve_once( + self, + variable_definitions: Dict[str, str], + resolved: Optional[Dict[str, Resolvable]] = None, + unresolvable: Optional[Set[str]] = None, + ) -> Dict[str, Resolvable]: + """ + Given a new set of variable definitions, resolve them using the Script, but do not + add them to the Script itself. + + Parameters + ---------- + variable_definitions + Variables to resolve, but not store in the Script + resolved + Optional. Pre-resolved variables that should be used instead of what is in the script. + unresolvable + Optional. Unresolvable variables that will be ignored in resolution, including all + variables with a dependency to them. + + Returns + ------- + Dict[str, Resolvable] + Dict containing the variable names to their resolved values. + """ + try: + self.add(variable_definitions) + return self._resolve( + pre_resolved=resolved, + unresolvable=unresolvable, + output_filter=set(list(variable_definitions.keys())), + ).output + finally: + for name in variable_definitions.keys(): + if name in self._variables: + del self._variables[name] + + def get(self, variable_name: str) -> Resolvable: + """ + Parameters + ---------- + variable_name + Name of the resolved variable to get. + + Returns + ------- + Resolvable + The resolved variable of the given name. + + Raises + ------ + RuntimeException + If the variable has not been resolved yet in the Script. + """ + if variable_name not in self._variables: + raise RuntimeException( + f"Tried to get resolved variable {variable_name}, but it does not exist" + ) + + if (resolvable := self._variables[variable_name].maybe_resolvable) is not None: + return resolvable + + raise RuntimeException(f"Tried to get unresolved variable {variable_name}") + + @property + def variable_names(self) -> Set[str]: + """ + Returns + ------- + Set[str] + Names of all the variables within the Script. + """ + return set(list(self._variables.keys())) + + @property + def function_names(self) -> Set[str]: + """ + Returns + ------- + Set[str] + Names of all functions within the Script. + """ + return set(_to_function_definition_name(name) for name in self._functions.keys()) diff --git a/src/ytdl_sub/script/script_output.py b/src/ytdl_sub/script/script_output.py new file mode 100644 index 00000000..d8275e6e --- /dev/null +++ b/src/ytdl_sub/script/script_output.py @@ -0,0 +1,52 @@ +from dataclasses import dataclass +from typing import Any +from typing import Dict + +from ytdl_sub.script.types.resolvable import Resolvable +from ytdl_sub.script.utils.exceptions import ScriptVariableNotResolved + + +@dataclass(frozen=True) +class ScriptOutput: + output: Dict[str, Resolvable] + + def as_native(self) -> Dict[str, Any]: + """ + Returns + ------- + The script output as native python types + """ + return {name: out.native for name, out in self.output.items()} + + def get(self, name: str) -> Resolvable: + """ + Returns + ------- + The script output's variable as a resolvable type + + Raises + ------ + ScriptVariableNotResolved + The variable name requested did not resolve + """ + if name not in self.output: + raise ScriptVariableNotResolved( + f"Tried to access resolved variable {name}, but it has not resolved" + ) + return self.output[name] + + def get_native(self, name: str) -> Any: + """ + Returns + ------- + The script output's variable as native python type + """ + return self.get(name).native + + def get_str(self, name: str) -> str: + """ + Returns + ------- + The script output's variable as a string + """ + return str(self.get(name)) diff --git a/src/ytdl_sub/script/types/__init__.py b/src/ytdl_sub/script/types/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/ytdl_sub/script/types/array.py b/src/ytdl_sub/script/types/array.py new file mode 100644 index 00000000..3129ffe7 --- /dev/null +++ b/src/ytdl_sub/script/types/array.py @@ -0,0 +1,58 @@ +from abc import ABC +from dataclasses import dataclass +from typing import Any +from typing import Dict +from typing import List +from typing import Type + +from ytdl_sub.script.types.resolvable import Argument +from ytdl_sub.script.types.resolvable import FutureResolvable +from ytdl_sub.script.types.resolvable import NonHashable +from ytdl_sub.script.types.resolvable import Resolvable +from ytdl_sub.script.types.resolvable import ResolvableToJson +from ytdl_sub.script.types.variable import Variable +from ytdl_sub.script.types.variable_dependency import VariableDependency + + +@dataclass(frozen=True) +class _Array(NonHashable, ABC): + value: List[Resolvable] + + @classmethod + def type_name(cls) -> str: + return "Array" + + +@dataclass(frozen=True) +class UnresolvedArray(_Array, VariableDependency, FutureResolvable): + value: List[Argument] + + @property + def _iterable_arguments(self) -> List[Argument]: + return self.value + + def resolve( + self, + resolved_variables: Dict[Variable, Resolvable], + custom_functions: Dict[str, "VariableDependency"], + ) -> Resolvable: + return Array( + [ + self._resolve_argument_type( + arg=arg, + resolved_variables=resolved_variables, + custom_functions=custom_functions, + ) + for arg in self.value + ] + ) + + def future_resolvable_type(self) -> Type[Resolvable]: + return Array + + +@dataclass(frozen=True) +class Array(_Array, ResolvableToJson): + @property + def native(self) -> Any: + return [val.native for val in self.value] diff --git a/src/ytdl_sub/script/types/function.py b/src/ytdl_sub/script/types/function.py new file mode 100644 index 00000000..94bbad5d --- /dev/null +++ b/src/ytdl_sub/script/types/function.py @@ -0,0 +1,288 @@ +import copy +import functools +from abc import ABC +from dataclasses import dataclass +from typing import Callable +from typing import Dict +from typing import List +from typing import Type +from typing import Union + +from ytdl_sub.script.functions import Functions +from ytdl_sub.script.types.array import Array +from ytdl_sub.script.types.array import UnresolvedArray +from ytdl_sub.script.types.resolvable import Argument +from ytdl_sub.script.types.resolvable import BuiltInFunctionType +from ytdl_sub.script.types.resolvable import FunctionType +from ytdl_sub.script.types.resolvable import FutureResolvable +from ytdl_sub.script.types.resolvable import Lambda +from ytdl_sub.script.types.resolvable import NamedCustomFunction +from ytdl_sub.script.types.resolvable import Resolvable +from ytdl_sub.script.types.resolvable import ReturnableArgument +from ytdl_sub.script.types.resolvable import ReturnableArgumentA +from ytdl_sub.script.types.resolvable import ReturnableArgumentB +from ytdl_sub.script.types.variable import FunctionArgument +from ytdl_sub.script.types.variable import Variable +from ytdl_sub.script.types.variable_dependency import VariableDependency +from ytdl_sub.script.utils.exception_formatters import FunctionArgumentsExceptionFormatter +from ytdl_sub.script.utils.exceptions import UNREACHABLE +from ytdl_sub.script.utils.exceptions import FunctionRuntimeException +from ytdl_sub.script.utils.exceptions import RuntimeException +from ytdl_sub.script.utils.exceptions import UserThrownRuntimeError +from ytdl_sub.script.utils.type_checking import FunctionSpec +from ytdl_sub.script.utils.type_checking import is_union + + +@dataclass(frozen=True) +class Function(FunctionType, VariableDependency, ABC): + @property + def _iterable_arguments(self) -> List[Argument]: + return self.args + + +class CustomFunction(Function, NamedCustomFunction): + def resolve( + self, + resolved_variables: Dict[Variable, Resolvable], + custom_functions: Dict[str, "VariableDependency"], + ) -> Resolvable: + resolved_args: List[Resolvable] = [ + self._resolve_argument_type( + arg=arg, resolved_variables=resolved_variables, custom_functions=custom_functions + ) + for arg in self.args + ] + + if self.name in custom_functions: + if len(self.args) != len(custom_functions[self.name].function_arguments): + # Should be validated in the Script + raise UNREACHABLE + + resolved_variables_with_args = copy.deepcopy(resolved_variables) + for i, arg in enumerate(resolved_args): + function_arg = FunctionArgument.from_idx(idx=i, custom_function_name=self.name) + + if function_arg in resolved_variables_with_args: + # function args should always be unique since they are only defined once + # in the custom function as %custom_function_name___idx + # and returned as a set from each custom function. + raise UNREACHABLE + + resolved_variables_with_args[function_arg] = arg + + return custom_functions[self.name].resolve( + resolved_variables=resolved_variables_with_args, + custom_functions=custom_functions, + ) + + # Implies the custom function does not exist. This should have + # been checked in the parser with + raise UNREACHABLE + + +class BuiltInFunction(Function, BuiltInFunctionType): + def validate_args(self) -> "BuiltInFunction": + """ + Ensures the args are compatible with the BuiltInFunction. + """ + if not self.function_spec.is_compatible(input_args=self.args): + raise FunctionArgumentsExceptionFormatter( + input_spec=self.function_spec, + function_instance=self, + ).highlight() + + return self + + # pylint: disable=missing-raises-doc + + @property + def callable(self) -> Callable[..., Resolvable]: + """ + Returns + ------- + The actual callable of the BuiltInFunction + """ + try: + return Functions.get(self.name) + except Exception as exc: + # Should be validated in the parser + raise UNREACHABLE from exc + + # pylint: enable=missing-raises-doc + + @functools.cached_property + def function_spec(self) -> FunctionSpec: + """ + Returns + ------- + The FunctionSpec of the BuiltInFunction + """ + return FunctionSpec.from_callable(self.callable) + + @classmethod + def _arg_output_type(cls, arg: Argument) -> Type[Argument]: + if isinstance(arg, BuiltInFunction): + return arg.output_type() + if isinstance(arg, FutureResolvable): + return arg.future_resolvable_type() + return type(arg) + + @classmethod + def _instantiate_lambda(cls, lambda_function_name: str, args: List[Argument]) -> Function: + return ( + BuiltInFunction(name=lambda_function_name, args=args) + if Functions.is_built_in(lambda_function_name) + else CustomFunction(name=lambda_function_name, args=args) + ) + + def _output_type(self, union_args: List[Type[Argument]]) -> Type[Resolvable]: + union_types_list = set() + for union_type in union_args: + possible_output_type = union_type + if union_type in (ReturnableArgument, ReturnableArgumentA, ReturnableArgumentB): + generic_arg_index = self.function_spec.args.index(union_type) + possible_output_type = self._arg_output_type(self.args[generic_arg_index]) + + if is_union(possible_output_type): + union_types_list.update(possible_output_type.__args__) + else: + union_types_list.add(possible_output_type) + + return Union[tuple(union_types_list)] + + def output_type(self) -> Type[Resolvable]: + """ + Returns + ------- + The BuiltInFunction's true output type. + """ + if is_union(self.function_spec.return_type): + return self._output_type(self.function_spec.return_type.__args__) + + return self._output_type([self.function_spec.return_type]) + + def _resolve_lambda_function( + self, + resolved_arguments: List[Resolvable | Lambda], + resolved_variables: Dict[Variable, Resolvable], + custom_functions: Dict[str, "VariableDependency"], + ) -> Resolvable: + """ + Resolve the lambda function by + 1. Calling the actual built-in function, which actually forms the input args to the + lambda. NOTE: the lambda argument MUST BE the last argument in the input spec + and output a ResolvedArray, where each element is the input args to the lambda. + 2. Preemptively creating the lambda's unresolved output array using output args from (1) + 3. Resolve it like any other syntax + """ + function_input_lambda_args = [arg for arg in resolved_arguments if isinstance(arg, Lambda)] + if not self.function_spec.is_lambda_function or len(function_input_lambda_args) != 1: + raise UNREACHABLE + + lambda_function_name = function_input_lambda_args[0].value + + try: + lambda_args = self.callable(*resolved_arguments) + except Exception as exc: + raise FunctionRuntimeException( + f"Runtime error occurred when executing the function %{self.name}: {str(exc)}" + ) from exc + + assert isinstance(lambda_args, Array) + + return self._resolve_argument_type( + arg=UnresolvedArray( + [ + self._instantiate_lambda( + lambda_function_name=lambda_function_name, args=lambda_arg.value + ) + for lambda_arg in lambda_args.value + ] + ), + resolved_variables=resolved_variables, + custom_functions=custom_functions, + ) + + def _resolve_lambda_reduce_function( + self, + resolved_arguments: List[Resolvable | Lambda], + resolved_variables: Dict[Variable, Resolvable], + custom_functions: Dict[str, "VariableDependency"], + ) -> Resolvable: + """ + Resolve the lambda reduce function by + 1. Preemptively create the 'reduce-like' call-stack as unresolvable + 2. Resolve it like any other syntax + """ + function_input_lambda_args = [arg for arg in resolved_arguments if isinstance(arg, Lambda)] + if not self.function_spec.is_lambda_reduce_function or len(function_input_lambda_args) != 1: + raise UNREACHABLE + + lambda_function_name = function_input_lambda_args[0].value + + try: + lambda_array = self.callable(*resolved_arguments) + except Exception as exc: + raise FunctionRuntimeException( + f"Runtime error occurred when executing the function %{self.name}: {str(exc)}" + ) from exc + + assert isinstance(lambda_array, Array) + + if len(lambda_array.value) == 1: + return lambda_array.value[0] + + reduced = self._instantiate_lambda( + lambda_function_name=lambda_function_name, + args=[lambda_array.value[0], lambda_array.value[1]], + ) + for idx in range(2, len(lambda_array.value)): + reduced = self._instantiate_lambda( + lambda_function_name=lambda_function_name, args=[reduced, lambda_array.value[idx]] + ) + + return self._resolve_argument_type( + arg=reduced, + resolved_variables=resolved_variables, + custom_functions=custom_functions, + ) + + def resolve( + self, + resolved_variables: Dict[Variable, Resolvable], + custom_functions: Dict[str, "VariableDependency"], + ) -> Resolvable: + # Resolve all non-lambda arguments + resolved_arguments: List[Resolvable | Lambda] = [ + self._resolve_argument_type( + arg=arg, + resolved_variables=resolved_variables, + custom_functions=custom_functions, + ) + for arg in self.args + ] + + # If a lambda is in a function's arg, resolve it differently + if self.function_spec.is_lambda_function: + return self._resolve_lambda_function( + resolved_arguments=resolved_arguments, + resolved_variables=resolved_variables, + custom_functions=custom_functions, + ) + + # If a lambda is in a function's arg, resolve it differently + if self.function_spec.is_lambda_reduce_function: + return self._resolve_lambda_reduce_function( + resolved_arguments=resolved_arguments, + resolved_variables=resolved_variables, + custom_functions=custom_functions, + ) + + try: + return self.callable(*resolved_arguments) + except (UserThrownRuntimeError, RuntimeException): + raise + except Exception as exc: + raise FunctionRuntimeException( + f"Runtime error occurred when executing the function %{self.name}: {str(exc)}" + ) from exc diff --git a/src/ytdl_sub/script/types/map.py b/src/ytdl_sub/script/types/map.py new file mode 100644 index 00000000..0fca1cbb --- /dev/null +++ b/src/ytdl_sub/script/types/map.py @@ -0,0 +1,66 @@ +import itertools +from abc import ABC +from dataclasses import dataclass +from typing import Any +from typing import Dict +from typing import List +from typing import Type + +from ytdl_sub.script.types.resolvable import Argument +from ytdl_sub.script.types.resolvable import FutureResolvable +from ytdl_sub.script.types.resolvable import Hashable +from ytdl_sub.script.types.resolvable import NonHashable +from ytdl_sub.script.types.resolvable import Resolvable +from ytdl_sub.script.types.resolvable import ResolvableToJson +from ytdl_sub.script.types.variable import Variable +from ytdl_sub.script.types.variable_dependency import VariableDependency +from ytdl_sub.script.utils.exceptions import KeyNotHashableRuntimeException + + +@dataclass(frozen=True) +class _Map(NonHashable, ABC): + value: Dict[Hashable, Resolvable] + + @classmethod + def type_name(cls) -> str: + return "Map" + + +@dataclass(frozen=True) +class UnresolvedMap(_Map, VariableDependency, FutureResolvable): + value: Dict[Argument, Argument] + + @property + def _iterable_arguments(self) -> List[Argument]: + return list(itertools.chain(*self.value.items())) + + def resolve( + self, + resolved_variables: Dict[Variable, Resolvable], + custom_functions: Dict[str, VariableDependency], + ) -> Resolvable: + output: Dict[Hashable, Resolvable] = {} + for key, value in self.value.items(): + resolved_key = self._resolve_argument_type( + arg=key, resolved_variables=resolved_variables, custom_functions=custom_functions + ) + if not isinstance(resolved_key, Hashable): + raise KeyNotHashableRuntimeException( + f"Tried to use {resolved_key.type_name()} as a Map key, but it is not hashable." + ) + + output[resolved_key] = self._resolve_argument_type( + arg=value, resolved_variables=resolved_variables, custom_functions=custom_functions + ) + + return Map(output) + + def future_resolvable_type(self) -> Type[Resolvable]: + return Map + + +@dataclass(frozen=True) +class Map(_Map, ResolvableToJson): + @property + def native(self) -> Any: + return {key.native: value.native for key, value in self.value.items()} diff --git a/src/ytdl_sub/script/types/resolvable.py b/src/ytdl_sub/script/types/resolvable.py new file mode 100644 index 00000000..829c0e78 --- /dev/null +++ b/src/ytdl_sub/script/types/resolvable.py @@ -0,0 +1,253 @@ +import json +from abc import ABC +from abc import abstractmethod +from dataclasses import dataclass +from typing import Any +from typing import Generic +from typing import List +from typing import Type +from typing import TypeVar + +T = TypeVar("T") +NumericT = TypeVar("NumericT", bound=int | float) + + +@dataclass(frozen=True) +class NamedType(ABC): + @classmethod + def type_name(cls) -> str: + """ + Returns + ------- + The type name to present to users. Defaults to the class name. + """ + return cls.__name__ + + +@dataclass(frozen=True) +class Argument(NamedType, ABC): + """ + Any possible argument type that has not been resolved yet + """ + + +@dataclass(frozen=True) +class ValueArgument(Argument, ABC): + """ + Argument that has a value + """ + + value: Any + + +@dataclass(frozen=True) +class NamedArgument(Argument, ABC): + """ + Argument that has an explicit name (i.e. custom function or variable) + """ + + name: str + + +@dataclass(frozen=True) +class ReturnableArgument(ValueArgument, NamedType, ABC): + """ + AnyType to express generics in functions that are part of the return type + """ + + +@dataclass(frozen=True) +class ReturnableArgumentA(ValueArgument, NamedType, ABC): + """ + AnyType to express generics in functions when more than one are present (i.e. `if`) + """ + + +@dataclass(frozen=True) +class ReturnableArgumentB(ValueArgument, NamedType, ABC): + """ + AnyType to express generics in functions when more than one are present (i.e. `if`) + """ + + +@dataclass(frozen=True) +class AnyArgument(ReturnableArgument, ReturnableArgumentA, ReturnableArgumentB, ABC): + """ + Human-readable name for Resolvable + """ + + +@dataclass(frozen=True) +class Resolvable(AnyArgument, ABC): + """ + A type that is resolved into a native Python type (and have no dependencies to other types). + """ + + def __str__(self) -> str: + return str(self.value) + + @property + def native(self) -> Any: + """ + Returns + ------- + The resolvable in its native form + """ + return self.value + + +@dataclass(frozen=True) +class FutureResolvable(AnyArgument, ABC): + """ + Used when parsing, it is an unresolved type that will eventually resolve to a known type + (i.e. Maps, Arrays) + """ + + @abstractmethod + def future_resolvable_type(self) -> Type[Resolvable]: + """ + The resolvable type that this type is known to turn into + """ + + +@dataclass(frozen=True) +class Hashable(Resolvable, ABC): + """ + Resolvable type that can be used as hashes (i.e. in Maps) + """ + + +@dataclass(frozen=True) +class NonHashable(NamedType, ABC): + """ + Type that is known to never be hashable. + """ + + +@dataclass(frozen=True) +class ResolvableToJson(Resolvable, ABC): + """ + Types whose string values should be resolved to JSON (i.e. Maps, Arrays) + """ + + def __str__(self): + return json.dumps(self.native) + + +@dataclass(frozen=True) +class ResolvableT(Hashable, ABC, Generic[T]): + """ + Resolvable types that resolve to the generic T + """ + + value: T + + +@dataclass(frozen=True) +class Numeric(ResolvableT[NumericT], ABC, Generic[NumericT]): + """ + Resolvable numeric types (int/float) + """ + + +@dataclass(frozen=True) +class Integer(Numeric[int], Argument): + """ + Resolved Integer type + """ + + +@dataclass(frozen=True) +class Float(Numeric[float], Argument): + """ + Resolved float type + """ + + +@dataclass(frozen=True) +class Boolean(ResolvableT[bool], Argument): + """ + Resolved bool type + """ + + +@dataclass(frozen=True) +class String(ResolvableT[str], Argument): + """ + Resolved String type + """ + + +@dataclass(frozen=True) +class NamedCustomFunction(NamedArgument, ABC): + """ + A custom function with a defined name (but unknown args) + """ + + +@dataclass(frozen=True) +class ParsedCustomFunction(NamedCustomFunction): + num_input_args: int + + +@dataclass(frozen=True) +class FunctionType(NamedArgument, ABC): + args: List[Argument] + + +@dataclass(frozen=True) +class BuiltInFunctionType(FunctionType, ABC): + @abstractmethod + def output_type(self) -> Type[Resolvable]: + """ + Returns + ------- + The known Resolvable type that a BuiltInFunction will output + """ + + +@dataclass(frozen=True) +class Lambda(Resolvable): + value: str + + @property + def native(self) -> Any: + return f"%{self.value}" + + @classmethod + def num_input_args(cls) -> int: + """ + Returns + ------- + The number of input args the Lambda function takes + """ + return 1 + + +@dataclass(frozen=True) +class LambdaTwo(Lambda): + """ + Type-hinting for functions that apply lambdas with two inputs per element + """ + + @classmethod + def num_input_args(cls) -> int: + return 2 + + +@dataclass(frozen=True) +class LambdaThree(Lambda): + """ + Type-hinting for functions that apply lambdas with three inputs per element + """ + + @classmethod + def num_input_args(cls) -> int: + return 3 + + +@dataclass(frozen=True) +class LambdaReduce(LambdaTwo): + """ + Type-hinting for functions that apply a reduce-operation using a lambda (two arguments) + """ diff --git a/src/ytdl_sub/script/types/syntax_tree.py b/src/ytdl_sub/script/types/syntax_tree.py new file mode 100644 index 00000000..d058852c --- /dev/null +++ b/src/ytdl_sub/script/types/syntax_tree.py @@ -0,0 +1,52 @@ +from dataclasses import dataclass +from typing import Dict +from typing import List +from typing import Optional + +from ytdl_sub.script.types.resolvable import Argument +from ytdl_sub.script.types.resolvable import Resolvable +from ytdl_sub.script.types.resolvable import String +from ytdl_sub.script.types.variable import Variable +from ytdl_sub.script.types.variable_dependency import VariableDependency + + +@dataclass(frozen=True) +class SyntaxTree(VariableDependency): + ast: List[Argument] + + @property + def _iterable_arguments(self) -> List[Argument]: + return self.ast + + def resolve( + self, + resolved_variables: Dict[Variable, Resolvable], + custom_functions: Dict[str, VariableDependency], + ) -> Resolvable: + resolved: List[Resolvable] = [] + for token in self.ast: + resolved.append( + self._resolve_argument_type( + arg=token, + resolved_variables=resolved_variables, + custom_functions=custom_functions, + ) + ) + + # If only one resolvable resides in the AST, return as that + if len(resolved) == 1: + return resolved[0] + + # Otherwise, to concat multiple resolved outputs, we must concat as strings + return String("".join([str(res) for res in resolved])) + + @property + def maybe_resolvable(self) -> Optional[Resolvable]: + """ + Returns + ------- + A resolvable if the AST contains a single type that is resolvable. None otherwise. + """ + if len(self.ast) == 1 and isinstance(self.ast[0], Resolvable): + return self.ast[0] + return None diff --git a/src/ytdl_sub/script/types/variable.py b/src/ytdl_sub/script/types/variable.py new file mode 100644 index 00000000..5471ec30 --- /dev/null +++ b/src/ytdl_sub/script/types/variable.py @@ -0,0 +1,28 @@ +from dataclasses import dataclass +from typing import Optional + +from ytdl_sub.script.types.resolvable import NamedArgument + + +@dataclass(frozen=True) +class Variable(NamedArgument): + pass + + +@dataclass(frozen=True) +class FunctionArgument(Variable): + """Arguments for custom functions, i.e. $0, $1, etc""" + + index: int + + @classmethod + def from_idx(cls, idx: int, custom_function_name: Optional[str]) -> "FunctionArgument": + """ + Returns + ------- + FunctionArgument whose variable name is the index, and optionally contains the custom + function name its defined in as a prefix. + """ + if custom_function_name: + return FunctionArgument(name=f"${custom_function_name}___{idx}", index=idx) + return FunctionArgument(name=f"${idx}", index=idx) diff --git a/src/ytdl_sub/script/types/variable_dependency.py b/src/ytdl_sub/script/types/variable_dependency.py new file mode 100644 index 00000000..8cd86134 --- /dev/null +++ b/src/ytdl_sub/script/types/variable_dependency.py @@ -0,0 +1,174 @@ +from abc import ABC +from abc import abstractmethod +from dataclasses import dataclass +from typing import Dict +from typing import Iterable +from typing import List +from typing import Set +from typing import Type +from typing import TypeVar +from typing import final + +from ytdl_sub.script.types.resolvable import Argument +from ytdl_sub.script.types.resolvable import BuiltInFunctionType +from ytdl_sub.script.types.resolvable import FunctionType +from ytdl_sub.script.types.resolvable import Lambda +from ytdl_sub.script.types.resolvable import NamedCustomFunction +from ytdl_sub.script.types.resolvable import ParsedCustomFunction +from ytdl_sub.script.types.resolvable import Resolvable +from ytdl_sub.script.types.variable import FunctionArgument +from ytdl_sub.script.types.variable import Variable +from ytdl_sub.script.utils.exceptions import UNREACHABLE + +TypeT = TypeVar("TypeT") + + +@dataclass(frozen=True) +class VariableDependency(ABC): + @property + @abstractmethod + def _iterable_arguments(self) -> List[Argument]: + """ + Returns + ------- + Any arguments in the VariableDependency that may or may not need to be resolved. + """ + + def _recurse_get(self, ttype: Type[TypeT], subclass: bool = False) -> List[TypeT]: + output: List[TypeT] = [] + for arg in self._iterable_arguments: + if subclass and issubclass(type(arg), ttype): + output.append(arg) + elif isinstance(arg, ttype): + output.append(arg) + + if isinstance(arg, VariableDependency): + # pylint: disable=protected-access + output.extend(arg._recurse_get(ttype)) + # pylint: enable=protected-access + + return output + + @final + @property + def variables(self) -> Set[Variable]: + """ + Returns + ------- + All Variables that this depends on. + """ + return set(self._recurse_get(Variable)) + + @final + @property + def built_in_functions(self) -> List[BuiltInFunctionType]: + """ + Returns + ------- + All BuiltInFunctions that this depends on. + """ + return self._recurse_get(BuiltInFunctionType) + + @final + @property + def function_arguments(self) -> Set[FunctionArgument]: + """ + Returns + ------- + All FunctionArguments that this depends on. + """ + return set(self._recurse_get(FunctionArgument)) + + @final + @property + def lambdas(self) -> Set[Lambda]: + """ + Returns + ------- + All Lambdas that this depends on. + """ + return set(self._recurse_get(Lambda, subclass=True)) + + # pylint: disable=missing-raises-doc + + @final + @property + def custom_functions(self) -> Set[ParsedCustomFunction]: + """ + Returns + ------- + All CustomFunctions that this depends on. + """ + output: Set[ParsedCustomFunction] = set() + for arg in self._iterable_arguments: + if isinstance(arg, NamedCustomFunction): + if not isinstance(arg, FunctionType): + # A NamedCustomFunction should also always be a FunctionType + raise UNREACHABLE + + # Custom funcs aren't hashable, so recreate just the base-class portion + output.add(ParsedCustomFunction(name=arg.name, num_input_args=len(arg.args))) + if isinstance(arg, VariableDependency): + output.update(arg.custom_functions) + + return output + + # pylint: enable=missing-raises-doc + + @abstractmethod + def resolve( + self, + resolved_variables: Dict[Variable, Resolvable], + custom_functions: Dict[str, "VariableDependency"], + ) -> Resolvable: + """ + Parameters + ---------- + resolved_variables + Lookup of variables that have been resolved + custom_functions + Lookup of any custom functions that have been parsed + + Returns + ------- + Resolved value + """ + + @classmethod + def _resolve_argument_type( + cls, + arg: Argument, + resolved_variables: Dict[Variable, Resolvable], + custom_functions: Dict[str, "VariableDependency"], + ) -> Resolvable: + if isinstance(arg, Resolvable): + return arg + if isinstance(arg, Variable): + if arg not in resolved_variables: + # All variables should exist and be resolved at this point + raise UNREACHABLE + return resolved_variables[arg] + if isinstance(arg, VariableDependency): + return arg.resolve( + resolved_variables=resolved_variables, custom_functions=custom_functions + ) + + raise UNREACHABLE + + @final + def is_subset_of(self, variables: Iterable[Variable]) -> bool: + """ + Returns + ------- + True if it contains all input variables as a dependency. False otherwise. + """ + return not self.variables.issubset(variables) + + @final + def contains(self, variables: Iterable[Variable]) -> bool: + """ + Returns + ------- + True if it contains any of the input variables. False otherwise. + """ + return len(self.variables.intersection(variables)) > 0 diff --git a/src/ytdl_sub/script/utils/__init__.py b/src/ytdl_sub/script/utils/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/ytdl_sub/script/utils/exception_formatters.py b/src/ytdl_sub/script/utils/exception_formatters.py new file mode 100644 index 00000000..fb3b5e6e --- /dev/null +++ b/src/ytdl_sub/script/utils/exception_formatters.py @@ -0,0 +1,153 @@ +import sys +from typing import List +from typing import Type +from typing import TypeVar +from typing import Union + +from ytdl_sub.script.types.resolvable import BuiltInFunctionType +from ytdl_sub.script.types.resolvable import NamedType +from ytdl_sub.script.utils.exceptions import IncompatibleFunctionArguments +from ytdl_sub.script.utils.exceptions import UserException +from ytdl_sub.script.utils.type_checking import FunctionSpec +from ytdl_sub.script.utils.type_checking import get_optional_type +from ytdl_sub.script.utils.type_checking import is_optional +from ytdl_sub.script.utils.type_checking import is_union + +TUserException = TypeVar("TUserException", bound=UserException) + + +class ParserExceptionFormatter: + def __init__(self, text: str, start: int, end: int, exception: TUserException): + self._text = text + self._start = start + self._end = end + self._exception = exception + + def _exception_text(self, border: int): + """ + Format for single-line exceptions + """ + text_left = max(0, self._start - border) + text_right = min(len(self._text), self._start + border) + relative_start = self._start - text_left + + exception_text: str = "" + if text_left > 3: + exception_text = "… " + relative_start += len(exception_text) + + exception_text += self._text[text_left:text_right] + + if text_right < len(self._text) - 3: + exception_text += " …" + + exception_text += "\n" + exception_text += f"{' ' * relative_start}^" + + return "\n" + exception_text + + @property + def _is_multi_line(self) -> bool: + return "\n" in self._text + + def _exception_text_lines(self, border_lines: int = 0) -> str: + """ + Format for multi-line exceptions + """ + split_text = self._text.split("\n") + + start_line: int = sys.maxsize + end_line: int = -1 + pos: int = 0 + for idx, line in enumerate(split_text): + if self._start <= pos < self._end: + start_line = min(start_line, idx) + end_line = max(end_line, idx + 1) + pos += len(line) + + true_start_line = start_line + start_line = max(0, start_line - border_lines) + end_line = min(len(split_text), end_line + border_lines) + + # Get min leading spaces between all lines to return + min_leading_spaces = sys.maxsize + for line in split_text[start_line:end_line]: + min_leading_spaces = min(min_leading_spaces, len(line) - len(line.lstrip())) + + to_return: List[str] = [] + for idx in range(start_line, end_line): + if idx == true_start_line: + to_return.append(f">>> {split_text[idx][min_leading_spaces:]}") + else: + to_return.append(f" {split_text[idx][min_leading_spaces:]}") + + return "\n" + "\n".join(to_return) + + def highlight(self) -> TUserException: + """ + Returns + ------- + Exception with human-readable error highlighting for invalid syntax + """ + if self._is_multi_line: + invalid_syntax = self._exception_text_lines(border_lines=3) + else: + invalid_syntax = self._exception_text(border=20) + + return self._exception.__class__(f"{invalid_syntax}\n{str(self._exception)}") + + +class FunctionArgumentsExceptionFormatter: + def __init__( + self, + input_spec: FunctionSpec, + function_instance: BuiltInFunctionType, + ): + self._args = input_spec.args + self._varargs = input_spec.varargs + self._name = function_instance.name + self._input_args = function_instance.args + + @classmethod + def _to_human_readable_name(cls, python_type: Type[NamedType] | Type[Union[NamedType]]) -> str: + if is_optional(python_type): + return f"Optional[{cls._to_human_readable_name(get_optional_type(python_type))}]" + if is_union(python_type): + return ", ".join( + sorted(cls._to_human_readable_name(arg) for arg in python_type.__args__) + ) + return python_type.type_name() + + def _expected_args_str(self) -> str: + if self._args is not None: + return f"({', '.join([self._to_human_readable_name(type_) for type_ in self._args])})" + if self._varargs is not None: + return f"({self._to_human_readable_name(self._varargs)}, ...)" + return "()" + + def _received_args_str(self) -> str: + received_type_names: List[str] = [] + for arg in self._input_args: + if isinstance(arg, BuiltInFunctionType): + if is_union(arg.output_type()): + readable_type_names = ", ".join( + sorted(type_.type_name() for type_ in arg.output_type().__args__) + ) + received_type_names.append(f"%{arg.name}(...)->Union[{readable_type_names}]") + else: + received_type_names.append(f"%{arg.name}(...)->{arg.output_type().type_name()}") + else: + received_type_names.append(arg.type_name()) + + return f"({', '.join(name for name in received_type_names)})" + + def highlight(self) -> IncompatibleFunctionArguments: + """ + Returns + ------- + Exception with human-readable error highlighting for incompatible function arguments + """ + return IncompatibleFunctionArguments( + f"Incompatible arguments passed to function {self._name}.\n" + f"Expected {self._expected_args_str()}\nReceived {self._received_args_str()}" + ) diff --git a/src/ytdl_sub/script/utils/exceptions.py b/src/ytdl_sub/script/utils/exceptions.py new file mode 100644 index 00000000..86f18e1b --- /dev/null +++ b/src/ytdl_sub/script/utils/exceptions.py @@ -0,0 +1,96 @@ +from abc import ABC + +from ytdl_sub.utils.exceptions import ValidationException + +################################################################################################### +# USER EXCEPTIONS + + +class UserException(ValidationException, ABC): + """It's the user's fault!""" + + +class InvalidSyntaxException(UserException): + """Syntax is incorrect""" + + +class InvalidVariableName(UserException): + """Variable name is invalid""" + + +class InvalidFunctionName(UserException): + """Custom function name is invalid""" + + +class InvalidCustomFunctionArguments(UserException): + """Custom function arguments are invalid (i.e. they do not increment)""" + + +class InvalidCustomFunctionArgumentName(UserException): + """Custom function argument name (i.e. $0) is invalid""" + + +class IncompatibleFunctionArguments(UserException): + """Function has invalid arguments""" + + +class FunctionDoesNotExist(UserException): + """Tried to use a function that does not exist""" + + +class VariableDoesNotExist(UserException): + """Tried to use a variable that does not exist""" + + +class ScriptBuilderMissingDefinitions(UserException): + """Tried to build an incomplete ScriptBuilder""" + + +class CycleDetected(UserException): + """A cycle exists within a user's script""" + + +class UserThrownRuntimeError(ValidationException): + """An error explicitly thrown by the user via a function""" + + +class _UnreachableSyntaxException(InvalidSyntaxException): + """For use in places where code _should_ never reach, but might from bugs""" + + +UNREACHABLE = _UnreachableSyntaxException( + "If you see this error, you have discovered a bug in the script parser!\n" + "Please upload your config/subscription file(s) to and make a GitHub issue at " + "https://github.com/jmbannon/ytdl-sub/issues" +) + +################################################################################################### +# RUNTIME EXCEPTIONS + + +class RuntimeException(ValueError, ABC): + """Exception thrown at runtime during resolution""" + + +class ScriptVariableNotResolved(RuntimeException): + """Tried to get a variable's resolved value from a script, but has not resolved yet""" + + +class FunctionRuntimeException(RuntimeException): + """Exception thrown when a ytdl-sub function has an error occur at runtime""" + + +class KeyNotHashableRuntimeException(RuntimeException): + """Map tried to use a non-hashable key at runtime""" + + +class ArrayValueDoesNotExist(RuntimeException): + """Tried to get an index of a value in an Array that does not exist""" + + +class FunctionDoesNotExistRuntimeException(RuntimeException): + """Tried to get a function that does not exist""" + + +class KeyDoesNotExistRuntimeException(RuntimeException): + """Tried to access a key on a map that does not exist, with no default""" diff --git a/src/ytdl_sub/script/utils/name_validation.py b/src/ytdl_sub/script/utils/name_validation.py new file mode 100644 index 00000000..de187ec8 --- /dev/null +++ b/src/ytdl_sub/script/utils/name_validation.py @@ -0,0 +1,60 @@ +import re + +from ytdl_sub.script.functions import Functions +from ytdl_sub.script.utils.exceptions import InvalidFunctionName +from ytdl_sub.script.utils.exceptions import InvalidVariableName + +_NAME_REGEX_VALIDATOR = re.compile(r"^[a-z][a-z0-9_]*$") + + +def is_valid_name(name: str) -> bool: + """ + Returns + ------- + True if the name adheres to the ``snake_case`` format. False otherwise. + """ + return re.match(_NAME_REGEX_VALIDATOR, name) is not None + + +def validate_variable_name(variable_name: str) -> str: + """ + Raises + ------ + InvalidVariableName + if the variable name is invalid + """ + if not is_valid_name(variable_name): + raise InvalidVariableName( + f"Variable name '{variable_name}' is invalid:" + " Names must be lower_snake_cased and begin with a letter." + ) + + if Functions.is_built_in(variable_name): + raise InvalidVariableName( + f"Variable name '{variable_name}' is invalid:" + " The name is used by a built-in function and cannot be overwritten." + ) + + return variable_name + + +def validate_custom_function_name(custom_function_name: str) -> None: + """ + Raises + ------ + InvalidFunctionName + If the function name is invalid + InvalidVariableName + if the variable name is invalid + """ + if not is_valid_name(custom_function_name): + raise InvalidFunctionName( + f"Custom function name '%{custom_function_name}' is invalid:" + " Names must be %lower_snake_cased and begin with a letter." + ) + + if Functions.is_built_in(custom_function_name): + raise InvalidFunctionName( + f"Custom function name '%{custom_function_name}' is invalid:" + " The name is used by a built-in function and cannot be overwritten." + ) diff --git a/src/ytdl_sub/script/utils/type_checking.py b/src/ytdl_sub/script/utils/type_checking.py new file mode 100644 index 00000000..cc7d87ff --- /dev/null +++ b/src/ytdl_sub/script/utils/type_checking.py @@ -0,0 +1,243 @@ +# pylint: disable=missing-raises-doc +import inspect +from dataclasses import dataclass +from inspect import FullArgSpec +from typing import Callable +from typing import List +from typing import Optional +from typing import Type +from typing import TypeVar +from typing import Union +from typing import get_origin + +from ytdl_sub.script.types.resolvable import Argument +from ytdl_sub.script.types.resolvable import BuiltInFunctionType +from ytdl_sub.script.types.resolvable import FutureResolvable +from ytdl_sub.script.types.resolvable import Lambda +from ytdl_sub.script.types.resolvable import LambdaReduce +from ytdl_sub.script.types.resolvable import LambdaThree +from ytdl_sub.script.types.resolvable import LambdaTwo +from ytdl_sub.script.types.resolvable import NamedCustomFunction +from ytdl_sub.script.types.resolvable import NamedType +from ytdl_sub.script.types.resolvable import Resolvable +from ytdl_sub.script.types.variable import Variable +from ytdl_sub.script.utils.exceptions import UNREACHABLE + +TLambda = TypeVar("TLambda", bound=Lambda) + + +def is_union(arg_type: Type) -> bool: + """ + Returns + ------- + True if typing is Union. False otherwise. + """ + return get_origin(arg_type) is Union + + +def is_optional(arg_type: Type) -> bool: + """ + Returns + ------- + True if typing is Optional. False otherwise. + """ + return is_union(arg_type) and type(None) in arg_type.__args__ + + +def get_optional_type(optional_type: Type) -> Type[NamedType]: + """ + Returns + ------- + Type within the Optional[Type] + """ + return [arg for arg in optional_type.__args__ if arg != type(None)][0] + + +def _is_type_compatible( + arg_type: Type[NamedType], + expected_arg_type: Type[Resolvable | Optional[Resolvable]], +) -> bool: + """ + Returns + ------- + True if arg is compatible with expected_arg_type. False otherwise. + """ + if is_union(expected_arg_type): + # See if the arg is a valid against the union + valid_type = False + + # if the input arg is a union, do a direct comparison + if is_union(arg_type): + valid_type = arg_type == expected_arg_type + # otherwise, iterate the union to see if it's compatible + else: + for union_type in expected_arg_type.__args__: + if issubclass(arg_type, union_type): + valid_type = True + break + + if not valid_type: + return False + # If the input is a union and the expected type is not, see if + # each possible union input is compatible with the expected type + elif is_union(arg_type): + for union_type in arg_type.__args__: + if not _is_type_compatible(union_type, expected_arg_type): + return False + elif issubclass(arg_type, (NamedCustomFunction, Variable)): + return True # custom-function/variable can be anything, so pass for now + elif issubclass(arg_type, Lambda) and issubclass(expected_arg_type, arg_type): + # lambda, check if expected_arg_type is a subclass + # Do not return on just that to also allow lambdas to be returned as + # ReturnableArguments (i.e in an %if statement) + return True + + elif not issubclass(arg_type, expected_arg_type): + return False + + return True + + +def is_type_compatible( + arg: NamedType, + expected_arg_type: Type[Resolvable | Optional[Resolvable]], +) -> bool: + """ + Returns + ------- + True if arg is compatible with expected_arg_type. False otherwise. + """ + arg_type: Type[NamedType] = arg.__class__ + if isinstance(arg, BuiltInFunctionType): + arg_type = arg.output_type() # built-in function + elif isinstance(arg, FutureResolvable): + arg_type = arg.future_resolvable_type() + + return _is_type_compatible(arg_type, expected_arg_type) + + +@dataclass(frozen=True) +class FunctionSpec: + return_type: Type[Resolvable] + args: Optional[List[Type[Resolvable | Optional[Resolvable]]]] = None + varargs: Optional[Type[Resolvable]] = None + + def __post_init__(self): + assert (self.args is None) ^ (self.varargs is None) + + def _is_args_compatible(self, input_args: List[Argument]) -> bool: + assert self.args is not None + + if len(input_args) > len(self.args): + return False + + for idx, arg in enumerate(self.args): + input_arg = input_args[idx] if idx < len(input_args) else None + if not is_type_compatible(arg=input_arg, expected_arg_type=arg): + return False + + return True + + def _is_varargs_compatible(self, input_args: List[Argument]) -> bool: + """ + Returns + ------- + True if the input args are compatible with the spec's varargs. False otherwise. + """ + assert self.varargs is not None + + for input_arg in input_args: + if not is_type_compatible(arg=input_arg, expected_arg_type=self.varargs): + return False + + return True + + def is_compatible(self, input_args: List[Argument]) -> bool: + """ + Returns + ------- + True if input_args is compatible. False otherwise. + """ + if self.args is not None: + return self._is_args_compatible(input_args=input_args) + if self.varargs is not None: + return self._is_varargs_compatible(input_args=input_args) + + raise UNREACHABLE # TODO: functions with no args + + def is_num_args_compatible(self, num_input_args: int) -> bool: + """ + Returns + ------- + True if the number of input args is compatible with the function spec. False otherwise. + """ + if self.args is not None: + return self.num_required_args <= num_input_args <= len(self.args) + return True # varargs can take any number + + @property + def num_required_args(self) -> int: + """ + Returns + ------- + The minimum number of args required to call the function. + """ + if self.args is not None: + return sum(1 for arg in self.args if not is_optional(arg)) + return 0 # varargs can take any number + + @property + def is_lambda_reduce_function(self) -> Optional[Type[LambdaReduce]]: + """ + Returns + ------- + True if the function is a Lambda-reduce function. False otherwise. + """ + return LambdaReduce if LambdaReduce in (self.args or []) else None + + @property + def is_lambda_function(self) -> Optional[Type[Lambda | LambdaTwo | LambdaThree]]: + """ + Returns + ------- + True if the function is a Lambda function (excluding reduce). False otherwise. + """ + if LambdaThree in (self.args or []): + return LambdaThree + if LambdaTwo in (self.args or []): + return LambdaTwo + if Lambda in (self.args or []): + return Lambda + return None + + @property + def is_lambda_like(self) -> Optional[Type[TLambda]]: + """ + Returns + ------- + True if the function is a Lambda type (including reduce). + """ + if l_type := self.is_lambda_reduce_function: + return l_type + if l_type := self.is_lambda_function: + return l_type + return None + + @classmethod + def from_callable(cls, callable_ref: Callable[..., Resolvable]) -> "FunctionSpec": + """ + Returns + ------- + FunctionSpec from a built-in function. + """ + arg_spec: FullArgSpec = inspect.getfullargspec(callable_ref) + if arg_spec.varargs: + return FunctionSpec( + return_type=arg_spec.annotations["return"], + varargs=arg_spec.annotations[arg_spec.varargs], + ) + + return FunctionSpec( + return_type=arg_spec.annotations["return"], + args=[arg_spec.annotations[arg_name] for arg_name in arg_spec.args], + ) diff --git a/src/ytdl_sub/subscriptions/base_subscription.py b/src/ytdl_sub/subscriptions/base_subscription.py index 0c61f7e1..c7872e8a 100644 --- a/src/ytdl_sub/subscriptions/base_subscription.py +++ b/src/ytdl_sub/subscriptions/base_subscription.py @@ -3,10 +3,10 @@ from pathlib import Path from typing import Optional from ytdl_sub.config.config_validator import ConfigOptions +from ytdl_sub.config.overrides import Overrides +from ytdl_sub.config.plugin.preset_plugins import PresetPlugins from ytdl_sub.config.preset import Preset -from ytdl_sub.config.preset import PresetPlugins from ytdl_sub.config.preset_options import OutputOptions -from ytdl_sub.config.preset_options import Overrides from ytdl_sub.config.preset_options import YTDLOptions from ytdl_sub.downloaders.url.validators import MultiUrlValidator from ytdl_sub.utils.file_handler import FileHandlerTransactionLog diff --git a/src/ytdl_sub/subscriptions/subscription_download.py b/src/ytdl_sub/subscriptions/subscription_download.py index 312fb46f..7bbd09ca 100644 --- a/src/ytdl_sub/subscriptions/subscription_download.py +++ b/src/ytdl_sub/subscriptions/subscription_download.py @@ -7,8 +7,10 @@ from pathlib import Path from typing import List from typing import Optional -from ytdl_sub.config.plugin import Plugin -from ytdl_sub.config.plugin import SplitPlugin +from ytdl_sub.config.plugin.plugin import Plugin +from ytdl_sub.config.plugin.plugin import SplitPlugin +from ytdl_sub.config.plugin.plugin_mapping import PluginMapping +from ytdl_sub.config.plugin.plugin_operation import PluginOperation from ytdl_sub.downloaders.info_json.info_json_downloader import InfoJsonDownloader from ytdl_sub.downloaders.info_json.info_json_downloader import InfoJsonDownloaderOptions from ytdl_sub.downloaders.source_plugin import SourcePlugin @@ -206,7 +208,9 @@ class SubscriptionDownload(BaseSubscription, ABC): @classmethod def _preprocess_entry(cls, plugins: List[Plugin], entry: Entry) -> Optional[Entry]: maybe_entry: Optional[Entry] = entry - for plugin in sorted(plugins, key=lambda _plugin: _plugin.priority.modify_entry_metadata): + for plugin in PluginMapping.order_plugins_by( + plugins, PluginOperation.MODIFY_ENTRY_METADATA + ): if (maybe_entry := plugin.modify_entry_metadata(maybe_entry)) is None: return None @@ -216,7 +220,7 @@ class SubscriptionDownload(BaseSubscription, ABC): self, plugins: List[Plugin], dry_run: bool, entry: Entry, entry_metadata: FileMetadata ): # Post-process the entry with all plugins - for plugin in sorted(plugins, key=lambda _plugin: _plugin.priority.post_process): + for plugin in PluginMapping.order_plugins_by(plugins, PluginOperation.POST_PROCESS): optional_plugin_entry_metadata = plugin.post_process_entry(entry) if optional_plugin_entry_metadata: entry_metadata.extend(optional_plugin_entry_metadata) @@ -236,7 +240,7 @@ class SubscriptionDownload(BaseSubscription, ABC): entry_: Optional[Entry] = entry # First, modify the entry with all plugins - for plugin in sorted(plugins, key=lambda _plugin: _plugin.priority.modify_entry): + for plugin in PluginMapping.order_plugins_by(plugins, PluginOperation.MODIFY_ENTRY): # Break if it is None, it is indicated to not process any further if (entry_ := plugin.modify_entry(entry_)) is None: break @@ -253,18 +257,10 @@ class SubscriptionDownload(BaseSubscription, ABC): ) -> None: entry_: Optional[Entry] = entry - plugins_pre_split = sorted( - [plugin for plugin in plugins if not plugin.priority.modify_entry_after_split], - key=lambda _plugin: _plugin.priority.modify_entry, - ) - - plugins_post_split = sorted( - [plugin for plugin in plugins if plugin.priority.modify_entry_after_split], - key=lambda _plugin: _plugin.priority.modify_entry, - ) - # First, modify the entry with pre_split plugins - for plugin in plugins_pre_split: + for plugin in PluginMapping.order_plugins_by( + plugins, PluginOperation.MODIFY_ENTRY, before_split=True + ): # Break if it is None, it is indicated to not process any further if (entry_ := plugin.modify_entry(entry_)) is None: break @@ -274,7 +270,9 @@ class SubscriptionDownload(BaseSubscription, ABC): for split_entry, split_entry_metadata in split_plugin.split(entry=entry_): split_entry_: Optional[Entry] = split_entry - for plugin in plugins_post_split: + for plugin in PluginMapping.order_plugins_by( + plugins, PluginOperation.MODIFY_ENTRY, before_split=False + ): # Return if it is None, it is indicated to not process any further. # Break out of the plugin loop if (split_entry_ := plugin.modify_entry(split_entry_)) is None: diff --git a/src/ytdl_sub/subscriptions/subscription_validators.py b/src/ytdl_sub/subscriptions/subscription_validators.py index 678a9758..dbcd5516 100644 --- a/src/ytdl_sub/subscriptions/subscription_validators.py +++ b/src/ytdl_sub/subscriptions/subscription_validators.py @@ -8,7 +8,7 @@ from typing import Optional from typing import final from ytdl_sub.config.config_file import ConfigFile -from ytdl_sub.config.preset_options import Overrides +from ytdl_sub.config.overrides import Overrides from ytdl_sub.entries.variables.override_variables import SUBSCRIPTION_NAME from ytdl_sub.entries.variables.override_variables import SUBSCRIPTION_VALUE from ytdl_sub.entries.variables.override_variables import OverrideVariables diff --git a/src/ytdl_sub/subscriptions/subscription_ytdl_options.py b/src/ytdl_sub/subscriptions/subscription_ytdl_options.py index 6a7a7cea..14916c2e 100644 --- a/src/ytdl_sub/subscriptions/subscription_ytdl_options.py +++ b/src/ytdl_sub/subscriptions/subscription_ytdl_options.py @@ -7,7 +7,7 @@ from typing import TypeVar from yt_dlp import match_filter_func -from ytdl_sub.config.plugin import Plugin +from ytdl_sub.config.plugin.plugin import Plugin from ytdl_sub.config.preset import Preset from ytdl_sub.downloaders.ytdl_options_builder import YTDLOptionsBuilder from ytdl_sub.plugins.audio_extract import AudioExtractPlugin diff --git a/src/ytdl_sub/utils/chapters.py b/src/ytdl_sub/utils/chapters.py index 0875e3bc..9ccfa2c0 100644 --- a/src/ytdl_sub/utils/chapters.py +++ b/src/ytdl_sub/utils/chapters.py @@ -1,14 +1,16 @@ import re from typing import Dict from typing import List -from typing import Optional from typing import Tuple from ytdl_sub.entries.entry import Entry -from ytdl_sub.entries.variables.kwargs import CHAPTERS -from ytdl_sub.entries.variables.kwargs import YTDL_SUB_CUSTOM_CHAPTERS +from ytdl_sub.entries.entry import ytdl_sub_chapters_from_comments +from ytdl_sub.entries.script.variable_definitions import VARIABLES +from ytdl_sub.entries.script.variable_definitions import VariableDefinitions from ytdl_sub.utils.file_handler import FileMetadata +v: VariableDefinitions = VARIABLES + class Timestamp: @@ -157,6 +159,17 @@ class Chapters: """ return self.timestamps[0].timestamp_sec == 0 + def to_yt_dlp_chapter_metadata(self) -> List[Dict[str, str | float]]: + """ + Returns + ------- + Metadata dict + """ + return [ + {"start_time": ts.timestamp_sec, "title": title} + for ts, title in zip(self.timestamps, self.titles) + ] + def to_file_metadata_dict(self) -> Dict: """ Returns @@ -165,7 +178,7 @@ class Chapters: """ return {ts.readable_str: title for ts, title in zip(self.timestamps, self.titles)} - def to_file_metadata(self, title: Optional[str] = None) -> FileMetadata: + def to_file_metadata(self, title: str) -> FileMetadata: """ Parameters ---------- @@ -216,8 +229,23 @@ class Chapters: # If more than 3 timestamps were parsed, return it if len(timestamps) >= 3: return Chapters(timestamps=timestamps, titles=titles) + # Otherwise return empty chapters - return Chapters(timestamps=[], titles=[]) + return cls.from_empty() + + @classmethod + def from_yt_dlp_chapters(cls, chapters: List[Dict[str, str | float]]): + """ + Create a Chapters object from the raw ``chapters`` metadata returned in an info.json + """ + timestamps: List[Timestamp] = [] + titles: List[str] = [] + + for chapter in chapters: + timestamps.append(Timestamp.from_seconds(int(float(chapter["start_time"])))) + titles.append(chapter["title"]) + + return Chapters(timestamps=timestamps, titles=titles) @classmethod def from_entry_chapters(cls, entry: Entry) -> "Chapters": @@ -231,19 +259,12 @@ class Chapters: ------- Chapters object """ - timestamps: List[Timestamp] = [] - titles: List[str] = [] + if chapters := ( + entry.try_get(ytdl_sub_chapters_from_comments, list) or entry.get(v.chapters, list) + ): + return cls.from_yt_dlp_chapters(chapters) - if entry.kwargs_contains(CHAPTERS): - for chapter in entry.kwargs_get(CHAPTERS, []): - timestamps.append(Timestamp.from_seconds(int(float(chapter["start_time"])))) - titles.append(chapter["title"]) - elif entry.kwargs_contains(YTDL_SUB_CUSTOM_CHAPTERS): - for start_time, title in entry.kwargs_get(YTDL_SUB_CUSTOM_CHAPTERS, {}).items(): - timestamps.append(Timestamp.from_str(start_time)) - titles.append(title) - - return Chapters(timestamps=timestamps, titles=titles) + return cls.from_empty() @classmethod def from_empty(cls) -> "Chapters": diff --git a/src/ytdl_sub/utils/datetime.py b/src/ytdl_sub/utils/datetime.py index 07fcaacf..cce045d7 100644 --- a/src/ytdl_sub/utils/datetime.py +++ b/src/ytdl_sub/utils/datetime.py @@ -3,7 +3,7 @@ from typing import Optional from yt_dlp import DateRange from yt_dlp.utils import datetime_from_str -from ytdl_sub.config.preset_options import Overrides +from ytdl_sub.config.overrides import Overrides from ytdl_sub.validators.string_datetime import StringDatetimeValidator diff --git a/src/ytdl_sub/utils/file_path.py b/src/ytdl_sub/utils/file_path.py new file mode 100644 index 00000000..af8246f1 --- /dev/null +++ b/src/ytdl_sub/utils/file_path.py @@ -0,0 +1,61 @@ +import os +from pathlib import Path +from typing import Tuple + +from ytdl_sub.config.defaults import MAX_FILE_NAME_BYTES +from ytdl_sub.utils.file_handler import get_file_extension + + +class FilePathTruncater: + _EXTENSION_BYTES = len("-thumb.jpg".encode("utf-8")) + 8 + _DEFAULT_MAX_BASE_FILE_NAME_BYTES: int = MAX_FILE_NAME_BYTES - _EXTENSION_BYTES + + _MAX_BASE_FILE_NAME_BYTES: int = _DEFAULT_MAX_BASE_FILE_NAME_BYTES + + @classmethod + def set_max_file_name_bytes(cls, max_file_name_bytes: int) -> None: + """Actually sets the max _base_ file name in bytes (excludes extension)""" + max_base_file_name_bytes = max_file_name_bytes - cls._EXTENSION_BYTES + + # bound between (extension_bytes + 20, MAX_FILE_NAME_BYTES) + max_base_file_name_bytes = max(max_base_file_name_bytes, 16) + max_base_file_name_bytes = min( + max_base_file_name_bytes, MAX_FILE_NAME_BYTES - cls._EXTENSION_BYTES + ) + + cls._MAX_BASE_FILE_NAME_BYTES = max_base_file_name_bytes + + @classmethod + def _is_file_name_too_long(cls, file_name: str) -> bool: + return len(file_name.encode("utf-8")) > cls._MAX_BASE_FILE_NAME_BYTES + + @classmethod + def _get_extension_split(cls, file_name: str) -> Tuple[str, str, str]: + if file_name.endswith("-thumb.jpg"): + ext = "-thumb.jpg" + delimiter = "" + else: + ext = get_file_extension(file_name) + delimiter = "." + + return file_name[: -len(ext)], ext, delimiter + + @classmethod + def _truncate_file_name(cls, file_name: str) -> str: + file_sub_name, file_ext, delimiter = cls._get_extension_split(file_name) + + desired_size = cls._MAX_BASE_FILE_NAME_BYTES - len(file_ext.encode("utf-8")) - 1 + while len(file_sub_name.encode("utf-8")) > desired_size: + file_sub_name = file_sub_name[:-1] + + return f"{file_sub_name}{delimiter}{file_ext}" + + @classmethod + def maybe_truncate_file_path(cls, file_path: str) -> str: + """Turn into a Path, then a string, to get correct directory separators""" + file_directory, file_name = os.path.split(Path(file_path)) + + if cls._is_file_name_too_long(file_name): + return str(Path(file_directory) / cls._truncate_file_name(file_name)) + + return str(file_path) diff --git a/src/ytdl_sub/utils/script.py b/src/ytdl_sub/utils/script.py new file mode 100644 index 00000000..d5d36b87 --- /dev/null +++ b/src/ytdl_sub/utils/script.py @@ -0,0 +1,40 @@ +import json +import re +from typing import Any +from typing import Dict + + +class ScriptUtils: + @classmethod + def add_sanitized_variables(cls, variables: Dict[str, str]) -> Dict[str, str]: + """ + Helper to add sanitized variables to a Script + """ + sanitized_variables = { + f"{name}_sanitized": f"{{%sanitize({name})}}" for name in variables.keys() + } + return dict(variables, **sanitized_variables) + + @classmethod + def to_script(cls, value: Any) -> str: + """ + Converts a python value to a script value + """ + if value is None: + out = "" + elif isinstance(value, str): + out = value + elif isinstance(value, int): + out = f"{{%int({value})}}" + elif isinstance(value, float): + out = f"{{%float({value})}}" + elif isinstance(value, bool): + out = f"{{%bool({value})}}" + else: + dumped_json = json.dumps(value, ensure_ascii=False, sort_keys=True) + # Remove triple-single-quotes from JSON to avoid parsing issues + dumped_json = re.sub("'{3,}", "'", dumped_json) + + out = f"{{%from_json('''{dumped_json}''')}}" + + return out diff --git a/src/ytdl_sub/utils/scriptable.py b/src/ytdl_sub/utils/scriptable.py new file mode 100644 index 00000000..8c136023 --- /dev/null +++ b/src/ytdl_sub/utils/scriptable.py @@ -0,0 +1,51 @@ +import copy +from abc import ABC +from typing import Any +from typing import Dict +from typing import Set + +from ytdl_sub.entries.script.function_scripts import CUSTOM_FUNCTION_SCRIPTS +from ytdl_sub.entries.script.variable_definitions import Variable +from ytdl_sub.entries.script.variable_scripts import UNRESOLVED_VARIABLES +from ytdl_sub.entries.script.variable_scripts import VARIABLE_SCRIPTS +from ytdl_sub.script.script import Script +from ytdl_sub.utils.script import ScriptUtils + + +class Scriptable(ABC): + """ + Shared class between Entry and Overrides to manage their underlying Script. + """ + + def __init__(self): + self.script = Script( + ScriptUtils.add_sanitized_variables( + dict(copy.deepcopy(VARIABLE_SCRIPTS), **copy.deepcopy(CUSTOM_FUNCTION_SCRIPTS)) + ) + ) + self.unresolvable: Set[str] = copy.deepcopy(UNRESOLVED_VARIABLES) + + def update_script(self) -> None: + """ + Updates any potential variables to a resolvable. This is done + to avoid re-resolving the same variables over-and-over. + """ + self.script.resolve(unresolvable=self.unresolvable, update=True) + + def add(self, values: Dict[str | Variable, Any]) -> None: + """ + Add new values to the script + """ + values_as_str: Dict[str, str] = { + (key.variable_name if isinstance(key, Variable) else key): val + for key, val in values.items() + } + + self.unresolvable -= set(list(values_as_str.keys())) + self.script.add( + ScriptUtils.add_sanitized_variables( + {name: ScriptUtils.to_script(value) for name, value in values_as_str.items()} + ), + unresolvable=self.unresolvable, + ) + self.update_script() diff --git a/src/ytdl_sub/validators/file_path_validators.py b/src/ytdl_sub/validators/file_path_validators.py index 8d5afe11..c255ed1a 100644 --- a/src/ytdl_sub/validators/file_path_validators.py +++ b/src/ytdl_sub/validators/file_path_validators.py @@ -1,12 +1,8 @@ import os from pathlib import Path from typing import Any -from typing import Dict -from typing import Tuple -from ytdl_sub.config.defaults import MAX_FILE_NAME_BYTES -from ytdl_sub.utils.file_handler import get_file_extension -from ytdl_sub.utils.subtitles import SUBTITLE_EXTENSIONS +from ytdl_sub.script.script import Script from ytdl_sub.validators.string_formatter_validators import OverridesStringFormatterValidator from ytdl_sub.validators.string_formatter_validators import StringFormatterValidator from ytdl_sub.validators.validators import StringValidator @@ -35,57 +31,8 @@ class FFprobeFileValidator(FFmpegFileValidator): _ffmpeg_dependency = "ffprobe" -class FilePathValidatorMixin: - _EXTENSION_BYTES = len("-thumb.jpg".encode("utf-8")) + 8 - _DEFAULT_MAX_BASE_FILE_NAME_BYTES: int = MAX_FILE_NAME_BYTES - _EXTENSION_BYTES - - _MAX_BASE_FILE_NAME_BYTES: int = _DEFAULT_MAX_BASE_FILE_NAME_BYTES - - @classmethod - def set_max_file_name_bytes(cls, max_file_name_bytes: int) -> None: - """Actually sets the max _base_ file name in bytes (excludes extension)""" - max_base_file_name_bytes = max_file_name_bytes - cls._EXTENSION_BYTES - - # bound between (extension_bytes + 20, MAX_FILE_NAME_BYTES) - max_base_file_name_bytes = max(max_base_file_name_bytes, 16) - max_base_file_name_bytes = min( - max_base_file_name_bytes, MAX_FILE_NAME_BYTES - cls._EXTENSION_BYTES - ) - - cls._MAX_BASE_FILE_NAME_BYTES = max_base_file_name_bytes - - @classmethod - def _is_file_name_too_long(cls, file_name: str) -> bool: - return len(file_name.encode("utf-8")) > cls._MAX_BASE_FILE_NAME_BYTES - - @classmethod - def _get_extension_split(cls, file_name: str) -> Tuple[str, str]: - ext = get_file_extension(file_name) - return file_name[: -len(ext)], ext - - @classmethod - def _truncate_file_name(cls, file_name: str) -> str: - file_sub_name, file_ext = cls._get_extension_split(file_name) - - desired_size = cls._MAX_BASE_FILE_NAME_BYTES - len(file_ext.encode("utf-8")) - 1 - while len(file_sub_name.encode("utf-8")) > desired_size: - file_sub_name = file_sub_name[:-1] - - return f"{file_sub_name}.{file_ext}" - - @classmethod - def _maybe_truncate_file_path(cls, file_path: Path) -> str: - """Turn into a Path, then a string, to get correct directory separators""" - file_directory, file_name = os.path.split(Path(file_path)) - - if cls._is_file_name_too_long(file_name): - return str(Path(file_directory) / cls._truncate_file_name(file_name)) - - return str(file_path) - - # pylint: disable=line-too-long -class StringFormatterFileNameValidator(StringFormatterValidator, FilePathValidatorMixin): +class StringFormatterFileNameValidator(StringFormatterValidator): """ Same as a :class:`StringFormatterValidator ` @@ -97,51 +44,30 @@ class StringFormatterFileNameValidator(StringFormatterValidator, FilePathValidat _expected_value_type_name = "filepath" - @classmethod - def _is_file_name_too_long(cls, file_name: str) -> bool: - return len(file_name.encode("utf-8")) > cls._MAX_BASE_FILE_NAME_BYTES - - @classmethod - def _get_extension_split(cls, file_name: str) -> Tuple[str, str]: - """ - Returns - ------- - file_name, ext (including .) - """ - if file_name.endswith(".info.json"): - ext = ".info.json" - elif file_name.endswith("-thumb.jpg"): - ext = "-thumb.jpg" - elif any(file_name.endswith(f".{subtitle_ext}") for subtitle_ext in SUBTITLE_EXTENSIONS): - file_name_split = file_name.split(".") - ext = file_name_split[-1] - - # Try to capture .lang.ext - if len(file_name_split) > 2 and len(file_name_split[-2]) < 6: - ext = f".{file_name_split[-2]}.{file_name_split[-1]}" - else: - ext = f".{file_name.rsplit('.', maxsplit=1)[-1]}" - - return file_name[: -len(ext)], ext - - @classmethod - def _truncate_file_name(cls, file_name: str) -> str: - file_sub_name, file_ext = cls._get_extension_split(file_name) - - while len(file_sub_name.encode("utf-8")) > cls._MAX_BASE_FILE_NAME_BYTES: - file_sub_name = file_sub_name[:-1] - - return f"{file_sub_name}{file_ext}" - - def apply_formatter(self, variable_dict: Dict[str, str]) -> str: - """Turn into a Path, then a string, to get correct directory separators""" - file_path = Path(super().apply_formatter(variable_dict)) - return self._maybe_truncate_file_path(file_path) + def post_process(self, resolved: str) -> str: + return ( + Script( + { + "tmp_var_1": resolved, + "tmp_var_2": "{%to_native_filepath(%truncate_filepath_if_too_long(tmp_var_1))}", + } + ) + .resolve() + .get_str("tmp_var_2") + ) class OverridesStringFormatterFilePathValidator(OverridesStringFormatterValidator): _expected_value_type_name = "static filepath" - def apply_formatter(self, variable_dict: Dict[str, str]) -> str: - """Turn into a Path, then a string, to get correct directory separators""" - return os.path.realpath(super().apply_formatter(variable_dict)) + def post_process(self, resolved: str) -> str: + return ( + Script( + { + "tmp_var_1": resolved, + "tmp_var_2": "{%to_native_filepath(%truncate_filepath_if_too_long(tmp_var_1))}", + } + ) + .resolve() + .get_str("tmp_var_2") + ) diff --git a/src/ytdl_sub/validators/string_datetime.py b/src/ytdl_sub/validators/string_datetime.py index 5818f6f4..d50be211 100644 --- a/src/ytdl_sub/validators/string_datetime.py +++ b/src/ytdl_sub/validators/string_datetime.py @@ -1,5 +1,3 @@ -from typing import Dict - from yt_dlp.utils import datetime_from_str from ytdl_sub.validators.string_formatter_validators import OverridesStringFormatterValidator @@ -21,10 +19,9 @@ class StringDatetimeValidator(OverridesStringFormatterValidator): _expected_value_type_name = "datetime string" - def apply_formatter(self, variable_dict: Dict[str, str]) -> str: - output = super().apply_formatter(variable_dict) + def post_process(self, resolved: str) -> str: try: - _ = datetime_from_str(output) + _ = datetime_from_str(resolved) except Exception as exc: raise self._validation_exception(f"Invalid datetime string: {str(exc)}") - return output + return resolved diff --git a/src/ytdl_sub/validators/string_formatter_validators.py b/src/ytdl_sub/validators/string_formatter_validators.py index 4408680f..59caabe3 100644 --- a/src/ytdl_sub/validators/string_formatter_validators.py +++ b/src/ytdl_sub/validators/string_formatter_validators.py @@ -1,20 +1,23 @@ import re -from collections import OrderedDict -from keyword import iskeyword from typing import Dict -from typing import List +from typing import Set +from typing import Union from typing import final -from yt_dlp.utils import sanitize_filename - +from ytdl_sub.entries.script.variable_definitions import VARIABLES +from ytdl_sub.script.parser import parse +from ytdl_sub.script.script import Script +from ytdl_sub.script.utils.exceptions import UserException +from ytdl_sub.script.utils.exceptions import VariableDoesNotExist from ytdl_sub.utils.exceptions import InvalidVariableNameException -from ytdl_sub.utils.exceptions import StringFormattingException from ytdl_sub.utils.exceptions import StringFormattingVariableNotFoundException +from ytdl_sub.validators.validators import DictValidator from ytdl_sub.validators.validators import ListValidator from ytdl_sub.validators.validators import LiteralDictValidator from ytdl_sub.validators.validators import StringValidator +from ytdl_sub.validators.validators import Validator -_fields_validator = re.compile(r"{([a-z][a-z0-9_]+?)}") +_fields_validator = re.compile(r"{([a-z][a-z0-9_]*?)}") _fields_validator_exception_message: str = ( "{variable_names} must start with a lowercase letter, should only contain lowercase letters, " @@ -74,57 +77,16 @@ class StringFormatterValidator(StringValidator): """ _expected_value_type_name = "format string" - _variable_not_found_error_msg_formatter = ( - "Format variable '{variable_name}' does not exist. Available variables: {available_fields}" - ) - - _max_format_recursion = 8 - - def __validate_and_get_format_variables(self) -> List[str]: - """ - Returns - ------- - list[str] - List of format variables in the format string - - Raises - ------ - ValidationException - If the format string contains invalid variable formatting - """ - open_bracket_count = self.format_string.count("{") - close_bracket_count = self.format_string.count("}") - - if open_bracket_count != close_bracket_count: - raise self._validation_exception( - "Brackets are reserved for {variable_names} and should contain " - "a single open and close bracket.", - exception_class=StringFormattingException, - ) - - format_variables: List[str] = list(re.findall(_fields_validator, self.format_string)) - - if len(format_variables) != open_bracket_count: - raise self._validation_exception( - error_message=_fields_validator_exception_message, - exception_class=StringFormattingException, - ) - - for variable in format_variables: - if iskeyword(variable): - raise self._validation_exception( - f"'{variable}' is a Python keyword and cannot be used as a variable.", - exception_class=StringFormattingException, - ) - - return format_variables def __init__(self, name, value: str): super().__init__(name=name, value=value) - self.format_variables = self.__validate_and_get_format_variables() + try: + _ = parse(str(value)) + except UserException as exc: + raise self._validation_exception(exc) from exc - @final @property + @final def format_string(self) -> str: """ Returns @@ -133,67 +95,17 @@ class StringFormatterValidator(StringValidator): """ return self._value - def _apply_formatter( - self, formatter: "StringFormatterValidator", variable_dict: Dict[str, str] - ) -> "StringFormatterValidator": - # Ensure the variable names exist within the entry and overrides - for variable_name in formatter.format_variables: - # If the variable exists, but is sanitized... - if ( - variable_name.endswith("_sanitized") - and variable_name.removesuffix("_sanitized") in variable_dict - ): - # Resolve just the non-sanitized version, then sanitize it - variable_dict[variable_name] = sanitize_filename( - StringFormatterValidator( - name=self._name, value=f"{{{variable_name.removesuffix('_sanitized')}}}" - ).apply_formatter(variable_dict) - ) - # If the variable doesn't exist, error - elif variable_name not in variable_dict: - available_fields = ", ".join(sorted(variable_dict.keys())) - raise self._validation_exception( - self._variable_not_found_error_msg_formatter.format( - variable_name=variable_name, available_fields=available_fields - ), - exception_class=StringFormattingVariableNotFoundException, - ) + # pylint: disable=no-self-use - return StringFormatterValidator( - name=self._name, - value=formatter.format_string.format(**OrderedDict(variable_dict)), - ) - - def apply_formatter(self, variable_dict: Dict[str, str]) -> str: + def post_process(self, resolved: str) -> str: """ - Calls `format` on the format string using the variable_dict as input kwargs - - Parameters - ---------- - variable_dict - kwargs to pass to the format string - Returns ------- - Format string formatted + Apply any post processing to the resolved value """ - formatter = self - recursion_depth = 0 - max_depth = self._max_format_recursion + return resolved - while formatter.format_variables and recursion_depth < max_depth: - formatter = self._apply_formatter(formatter=formatter, variable_dict=variable_dict) - recursion_depth += 1 - - if formatter.format_variables: - raise self._validation_exception( - f"Attempted to format but failed after reaching max recursion depth of " - f"{max_depth}. Try to keep variables dependent on only one other variable at max. " - f"Unresolved variables: {', '.join(sorted(formatter.format_variables))}", - exception_class=StringFormattingException, - ) - - return formatter.format_string + # pylint: enable=no-self-use # pylint: disable=line-too-long @@ -208,12 +120,6 @@ class OverridesStringFormatterValidator(StringFormatterValidator): :class:`nfo_output_directory ` """ - _variable_not_found_error_msg_formatter = ( - "Override variable '{variable_name}' does not exist. For this field, ensure your override " - "variable does not contain any source variables - it is a requirement that this be a " - "static string. Available override variables: {available_fields}" - ) - # pylint: enable=line-too-long @@ -221,15 +127,15 @@ class OverridesStringFormatterValidator(StringFormatterValidator): class OverridesIntegerFormatterValidator(StringFormatterValidator): _expected_value_type_name = "integer" - def apply_formatter(self, variable_dict: Dict[str, str]) -> str: - output = super().apply_formatter(variable_dict) + def post_process(self, resolved: str) -> str: try: - int(output) + int(resolved) except Exception as exc: raise self._validation_exception( - f"Expected an integer, but received '{output}'" + f"Expected an integer, but received '{resolved}'" ) from exc - return output + + return resolved class ListFormatterValidator(ListValidator[StringFormatterValidator]): @@ -268,3 +174,63 @@ class OverridesDictFormatterValidator(DictFormatterValidator): """ _key_validator = OverridesStringFormatterValidator + + +def _validate_formatter( + mock_script: Script, + unresolved_variables: Set[str], + formatter_validator: Union[StringFormatterValidator, OverridesStringFormatterValidator], +) -> None: + try: + unresolvable = unresolved_variables + if isinstance(formatter_validator, OverridesStringFormatterValidator): + unresolvable = unresolved_variables.union({VARIABLES.entry_metadata.variable_name}) + + mock_script.resolve_once( + {"tmp_var": formatter_validator.format_string}, + unresolvable=unresolvable, + ) + except VariableDoesNotExist as exc: + raise StringFormattingVariableNotFoundException(exc) from exc + + +def validate_formatters( + script: Script, + unresolved_variables: Set[str], + validator: Validator, +) -> None: + """ + Ensure all OverridesStringFormatterValidator's only contain variables from the overrides + and resolve. + """ + if isinstance(validator, DictValidator): + # pylint: disable=protected-access + # Usage of protected variables in other validators is fine. The reason to keep + # them protected is for readability when using them in subscriptions. + for validator_value in validator._validator_dict.values(): + validate_formatters( + script=script, + unresolved_variables=unresolved_variables, + validator=validator_value, + ) + # pylint: enable=protected-access + elif isinstance(validator, ListValidator): + for list_value in validator.list: + validate_formatters( + script=script, + unresolved_variables=unresolved_variables, + validator=list_value, + ) + elif isinstance(validator, (StringFormatterValidator, OverridesStringFormatterValidator)): + _validate_formatter( + mock_script=script, + unresolved_variables=unresolved_variables, + formatter_validator=validator, + ) + elif isinstance(validator, (DictFormatterValidator, OverridesDictFormatterValidator)): + for validator_value in validator.dict.values(): + _validate_formatter( + mock_script=script, + unresolved_variables=unresolved_variables, + formatter_validator=validator_value, + ) diff --git a/src/ytdl_sub/ytdl_additions/enhanced_download_archive.py b/src/ytdl_sub/ytdl_additions/enhanced_download_archive.py index 584f5541..616fe2f9 100644 --- a/src/ytdl_sub/ytdl_additions/enhanced_download_archive.py +++ b/src/ytdl_sub/ytdl_additions/enhanced_download_archive.py @@ -14,7 +14,9 @@ from yt_dlp import DateRange from yt_dlp.utils import make_archive_id from ytdl_sub.entries.entry import Entry -from ytdl_sub.entries.variables.kwargs import SPLIT_BY_CHAPTERS_PARENT_ENTRY +from ytdl_sub.entries.entry import ytdl_sub_split_by_chapters_parent_uid +from ytdl_sub.entries.script.variable_definitions import VARIABLES +from ytdl_sub.entries.script.variable_definitions import VariableDefinitions from ytdl_sub.utils.file_handler import FileHandler from ytdl_sub.utils.file_handler import FileHandlerTransactionLog from ytdl_sub.utils.file_handler import FileMetadata @@ -22,6 +24,8 @@ from ytdl_sub.utils.logger import Logger logger = Logger.get("archive") +v: VariableDefinitions = VARIABLES + @dataclass class DownloadMapping: @@ -71,8 +75,8 @@ class DownloadMapping: DownloadMapping for the entry """ return DownloadMapping( - upload_date=entry.upload_date_standardized, - extractor=entry.extractor, + upload_date=entry.get(v.upload_date_standardized, str), + extractor=entry.download_archive_extractor, file_names=set(), ) @@ -219,10 +223,14 @@ class DownloadMappings: ------- self """ - if entry.uid not in self.entry_ids: - self._entry_mappings[entry.uid] = DownloadMapping.from_entry(entry=entry) + uid = entry.uid + if parent_uid := entry.try_get(ytdl_sub_split_by_chapters_parent_uid, str): + uid = parent_uid - self._entry_mappings[entry.uid].file_names.add(entry_file_path) + if uid not in self.entry_ids: + self._entry_mappings[uid] = DownloadMapping.from_entry(entry=entry) + + self._entry_mappings[uid].file_names.add(entry_file_path) return self def remove_entry(self, entry_id: str) -> "DownloadMappings": @@ -646,15 +654,7 @@ class EnhancedDownloadArchive: if output_file_name is None: output_file_name = file_name - # If the entry is created from splitting via chapters, store it to the mapping - # using its parent entry - if entry and entry.kwargs_contains(SPLIT_BY_CHAPTERS_PARENT_ENTRY): - parent_entry = Entry( - entry_dict=entry.kwargs(SPLIT_BY_CHAPTERS_PARENT_ENTRY), - working_directory=entry.working_directory(), - ) - self.mapping.add_entry(parent_entry, entry_file_path=output_file_name) - elif entry: + if entry: self.mapping.add_entry(entry=entry, entry_file_path=output_file_name) is_modified = self._file_handler.move_file_to_output_directory( diff --git a/tests/conftest.py b/tests/conftest.py index aa35b6ed..b7ec75ff 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -19,6 +19,7 @@ from resources import file_fixture_path from yt_dlp.utils import sanitize_filename from ytdl_sub.config.config_file import ConfigFile +from ytdl_sub.entries.script.custom_functions import CustomFunctions from ytdl_sub.subscriptions.subscription_download import SubscriptionDownload from ytdl_sub.utils.file_handler import FileHandler from ytdl_sub.utils.logger import Logger @@ -26,6 +27,14 @@ from ytdl_sub.utils.logger import LoggerLevels from ytdl_sub.utils.yaml import load_yaml +@pytest.fixture(autouse=True) +def register_custom_functions(): + """ + Clean logs after every test + """ + CustomFunctions.register() + + @pytest.fixture(autouse=True) def cleanup_debug_file(): """ diff --git a/tests/e2e/plugins/test_regex.py b/tests/e2e/plugins/test_regex.py index 504f74bb..58abb325 100644 --- a/tests/e2e/plugins/test_regex.py +++ b/tests/e2e/plugins/test_regex.py @@ -302,32 +302,40 @@ class TestRegex: preset_dict=regex_subscription_dict, ) - def test_regex_fails_capture_group_is_source_variable( + def test_regex_fails_capture_group_is_entry_variable( self, regex_subscription_dict, default_config ): - regex_subscription_dict["regex"]["from"]["title"]["capture_group_names"][0] = "uid" + regex_subscription_dict["regex"]["from"]["playlist_uid"] = { + "match": [".*http:\\/\\/(.+).com.*"], + "capture_group_names": ["uid"], + } + with pytest.raises( ValidationException, match=re.escape( - "'uid' cannot be used as a capture group name because it is a source variable" + "Cannot use the variable name uid because it exists as a built-in " + "ytdl-sub variable name." ), ): _ = Subscription.from_dict( config=default_config, - preset_name="test_regex_fails_capture_group_is_source_variable", + preset_name="test_regex_fails_capture_group_is_entry_variable", preset_dict=regex_subscription_dict, ) def test_regex_fails_capture_group_is_override_variable( self, regex_subscription_dict, default_config ): - regex_subscription_dict["regex"]["from"]["title"]["capture_group_names"][ - 0 - ] = "in_regex_default" + regex_subscription_dict["regex"]["from"]["playlist_uid"] = { + "match": [".*http:\\/\\/(.+).com.*"], + "capture_group_names": ["contains_regex_default"], + } + with pytest.raises( ValidationException, match=re.escape( - "'in_regex_default' cannot be used as a capture group name because it is an override variable" + "Override variable with name contains_regex_default cannot be used since it is " + "added by a plugin." ), ): _ = Subscription.from_dict( @@ -344,9 +352,7 @@ class TestRegex: ) with pytest.raises( ValidationException, - match=re.escape( - "cannot regex capture 'dne' because it is not a source or override variable" - ), + match=re.escape("cannot regex capture 'dne' because it is not a defined variable"), ): _ = Subscription.from_dict( config=default_config, diff --git a/tests/e2e/plugins/test_split_by_chapters.py b/tests/e2e/plugins/test_split_by_chapters.py index 597a1fe4..cb02715e 100644 --- a/tests/e2e/plugins/test_split_by_chapters.py +++ b/tests/e2e/plugins/test_split_by_chapters.py @@ -34,7 +34,7 @@ def yt_album_as_chapters_with_regex_preset_dict(yt_album_as_chapters_preset_dict "from": { # Ensure regex can handle override variables that come from the # post-metadata stage - "override_chapter_title": { + "chapter_title": { "match": r"\d+\. (.+)", "capture_group_names": "captured_track_title", "capture_group_defaults": "{chapter_title}", @@ -56,7 +56,6 @@ def yt_album_as_chapters_with_regex_preset_dict(yt_album_as_chapters_preset_dict } }, "overrides": { - "override_chapter_title": "{chapter_title}", "track_title": "{captured_track_title}", "track_album": "{captured_track_album}", "track_artist": "{captured_track_artist}", diff --git a/tests/unit/config/test_config_file.py b/tests/unit/config/test_config_file.py index 265cf840..56b97498 100644 --- a/tests/unit/config/test_config_file.py +++ b/tests/unit/config/test_config_file.py @@ -5,7 +5,7 @@ from typing import Optional import pytest from ytdl_sub.config.config_file import ConfigFile -from ytdl_sub.config.plugin_mapping import PluginMapping +from ytdl_sub.config.plugin.plugin_mapping import PluginMapping from ytdl_sub.config.preset import PRESET_KEYS from ytdl_sub.utils.exceptions import ValidationException diff --git a/tests/unit/config/test_preset.py b/tests/unit/config/test_preset.py index 158b2434..239a46c6 100644 --- a/tests/unit/config/test_preset.py +++ b/tests/unit/config/test_preset.py @@ -129,7 +129,7 @@ class TestPreset: ): with pytest.raises( StringFormattingVariableNotFoundException, - match="Format variable 'dne_var' does not exist", + match="Variable dne_var does not exist.", ): _ = Preset( config=config_file, @@ -145,7 +145,7 @@ class TestPreset: ): with pytest.raises( StringFormattingVariableNotFoundException, - match="Override variable 'dne_var' does not exist", + match="Variable dne_var does not exist", ): _ = Preset( config=config_file, @@ -161,7 +161,7 @@ class TestPreset: ): with pytest.raises( StringFormattingVariableNotFoundException, - match="Format variable 'dne_var' does not exist", + match="Variable dne_var does not exist", ): _ = Preset( config=config_file, @@ -182,7 +182,7 @@ class TestPreset: ): with pytest.raises( StringFormattingVariableNotFoundException, - match="Format variable 'dne_var' does not exist", + match="Variable dne_var does not exist", ): _ = Preset( config=config_file, @@ -208,22 +208,151 @@ class TestPreset: }, ) - def test_preset_with_multi_url__contains_all_empty_urls_errors( - self, config_file, output_options + @pytest.mark.parametrize( + "entry_variable_name", + [ + "title", + "playlist_uid", + "source_title", + "playlist_max_upload_year", + ], + ) + def test_preset_error_override_variable_collides_with_entry_variable( + self, config_file, output_options, youtube_video, entry_variable_name: str ): with pytest.raises( ValidationException, match=re.escape( - "Validation error in test.download: Must contain at least one " - "url that is non-empty" + f"Override variable with name {entry_variable_name} cannot be used since" + " it is a built-in ytdl-sub entry variable name." ), ): _ = Preset( config=config_file, name="test", value={ - "download": [{"url": "{url}"}, {"url": "{url2}"}], - "output_options": output_options, - "overrides": {"url": "", "url2": ""}, + "download": youtube_video, + "output_options": {"output_directory": "dir", "file_name": "{dne_var}"}, + "overrides": {entry_variable_name: "fail"}, + }, + ) + + def test_preset_error_override_variable_collides_added_variable( + self, config_file, output_options, youtube_video + ): + with pytest.raises( + ValidationException, + match=re.escape( + f"Override variable with name subtitles_ext cannot be used since" + " it is added by a plugin." + ), + ): + _ = Preset( + config=config_file, + name="test", + value={ + "download": youtube_video, + "output_options": {"output_directory": "dir", "file_name": "ack"}, + "subtitles": { + "embed_subtitles": True, + }, + "overrides": {"subtitles_ext": "collide"}, + }, + ) + + @pytest.mark.parametrize( + "function_name", + [ + "%extract_field_from_siblings", + "%extract_field_from_metadata_array", + "%sanitize", + "%array", + ], + ) + def test_preset_error_override_variable_collides_with_custom_function( + self, config_file, output_options, youtube_video, function_name: str + ): + with pytest.raises( + ValidationException, + match=re.escape( + f"Override function definition with name {function_name} cannot be used since" + " it is a built-in ytdl-sub function name." + ), + ): + _ = Preset( + config=config_file, + name="test", + value={ + "download": youtube_video, + "output_options": {"output_directory": "dir", "file_name": "{dne_var}"}, + "overrides": {function_name: "fail"}, + }, + ) + + def test_preset_error_override_added_variable_collides_with_built_in( + self, config_file, output_options + ): + with pytest.raises( + ValidationException, + match=re.escape( + "Cannot use the variable name title because it exists as a " + "built-in ytdl-sub variable name." + ), + ): + _ = Preset( + config=config_file, + name="test", + value={ + "download": { + "url": "youtube.com/watch?v=123abc", + "variables": {"title": "nope"}, + }, + "output_options": {"output_directory": "dir", "file_name": "acjk"}, + }, + ) + + def test_preset_error_override_added_variable_collides_with_override( + self, config_file, output_options + ): + with pytest.raises( + ValidationException, + match=re.escape( + "Override variable with name the_bad_one cannot be used since " + "it is added by a plugin." + ), + ): + _ = Preset( + config=config_file, + name="test", + value={ + "download": { + "url": "youtube.com/watch?v=123abc", + "variables": {"the_bad_one": "should error"}, + }, + "output_options": {"output_directory": "dir", "file_name": "acjk"}, + "overrides": {"the_bad_one": "ack"}, + }, + ) + + def test_preset_error_added_url_variable_cannot_resolve(self, config_file, output_options): + with pytest.raises( + ValidationException, + match=re.escape( + "variable the_bad_one cannot use the variables subtitles_ext because it " + "depends on other variables that are computed later in execution" + ), + ): + _ = Preset( + config=config_file, + name="test", + value={ + "download": { + "url": "youtube.com/watch?v=123abc", + "variables": {"the_bad_one": "{subtitles_ext}"}, + }, + "subtitles": { + "embed_subtitles": True, + }, + "output_options": {"output_directory": "dir", "file_name": "acjk"}, }, ) diff --git a/tests/unit/config/test_subscription.py b/tests/unit/config/test_subscription.py index 291079d8..ba687b6d 100644 --- a/tests/unit/config/test_subscription.py +++ b/tests/unit/config/test_subscription.py @@ -257,7 +257,6 @@ def test_subscription_file_value_applies_sub_file_takes_precedence( assert value_sub.get("test_file_subscription_value") == "is_overwritten" assert value_sub.get("test_config_subscription_value") == "original" assert value_sub.get("subscription_name") == "test_value" - assert value_sub.get("subscription_name_sanitized") == "test_value" assert value_sub.get("subscription_value") == "is_overwritten" assert value_sub.get("current_override") == "__preset__" # ensure __preset__ takes precedence @@ -276,7 +275,6 @@ def test_subscription_file_value_applies_from_config( assert value_sub.get("test_file_subscription_value") == "original" assert value_sub.get("test_config_subscription_value") == "is_overwritten" assert value_sub.get("subscription_name") == "test_value" - assert value_sub.get("subscription_name_sanitized") == "test_value" assert value_sub.get("subscription_value") == "is_overwritten" assert value_sub.get("current_override") == "__preset__" # ensure __preset__ takes precedence @@ -297,13 +295,11 @@ def test_subscription_file_value_applies_from_config_and_nested( assert sub_1.get("test_config_subscription_value") == "is_1_overwritten" assert sub_1.get("subscription_name") == "test_1" - assert sub_1.get("subscription_name_sanitized") == "test_1" assert sub_1.get("subscription_value") == "is_1_overwritten" assert sub_1.get("current_override") == "__preset__" # ensure __preset__ takes precedence assert sub_2_1.get("test_config_subscription_value") == "is_2_1_overwritten" assert sub_2_1.get("subscription_name") == "test_2_1" - assert sub_2_1.get("subscription_name_sanitized") == "test_2_1" assert sub_2_1.get("subscription_value") == "is_2_1_overwritten" assert sub_2_1.get("current_override") == "__preset__" # ensure __preset__ takes precedence @@ -322,7 +318,6 @@ def test_subscription_list( sub_2_1 = [sub for sub in subs if sub.name == "test_2_1"][0].overrides.dict_with_format_strings assert sub_2_1.get("subscription_name") == "test_2_1" - assert sub_2_1.get("subscription_name_sanitized") == "test_2_1" assert sub_2_1.get("subscription_value") == "is_2_1_overwritten" assert sub_2_1.get("subscription_value_1") == "is_2_1_overwritten" assert sub_2_1.get("subscription_value_2") == "is_2_1_list_2" @@ -343,7 +338,6 @@ def test_subscription_overrides_tilda( sub_2_1 = [sub for sub in subs if sub.name == "test_2_1"][0].overrides.dict_with_format_strings assert sub_2_1.get("subscription_name") == "test_2_1" - assert sub_2_1.get("subscription_name_sanitized") == "test_2_1" assert sub_2_1.get("current_override") == "test_2_1" # tilda sub takes precedence @@ -371,7 +365,6 @@ def test_subscription_file_value_applies_from_config_and_nested_and_indent_varia assert sub_1.get("test_config_subscription_value") == "is_1_overwritten" assert sub_1.get("subscription_name") == "test_1" - assert sub_1.get("subscription_name_sanitized") == "test_1" assert sub_1.get("subscription_value") == "is_1_overwritten" assert sub_1.get("subscription_indent_1") == "INDENT_1" assert sub_1.get("subscription_indent_2") == "INDENT_2" @@ -379,7 +372,6 @@ def test_subscription_file_value_applies_from_config_and_nested_and_indent_varia assert sub_2_1.get("test_config_subscription_value") == "is_2_1_overwritten" assert sub_2_1.get("subscription_name") == "test_2_1" - assert sub_2_1.get("subscription_name_sanitized") == "test_2_1" assert sub_2_1.get("subscription_value") == "is_2_1_overwritten" assert sub_2_1.get("subscription_indent_1") == "INDENT_1" assert sub_2_1.get("subscription_indent_2") == "original_2" @@ -417,7 +409,6 @@ def test_subscription_file_value_applies_from_config_and_nested_and_indent_varia assert sub_1.get("test_config_subscription_value") == "is_1_overwritten" assert sub_1.get("subscription_name") == "test_1" - assert sub_1.get("subscription_name_sanitized") == "test_1" assert sub_1.get("subscription_value") == "is_1_overwritten" assert sub_1.get("subscription_indent_1") == "INDENT_1" assert sub_1.get("subscription_indent_2") == "INDENT_2" @@ -426,7 +417,6 @@ def test_subscription_file_value_applies_from_config_and_nested_and_indent_varia assert sub_2_1.get("test_config_subscription_value") == "is_2_1_overwritten" assert sub_2_1.get("subscription_name") == "test_2_1" - assert sub_2_1.get("subscription_name_sanitized") == "test_2_1" assert sub_2_1.get("subscription_value") == "is_2_1_overwritten" assert sub_2_1.get("subscription_indent_1") == "INDENT_1" assert sub_2_1.get("subscription_indent_2") == "original_2" @@ -499,7 +489,6 @@ def test_tv_show_subscriptions(config_file: ConfigFile, tv_show_subscriptions_pa jake_train_overrides = subs[3].overrides.dict_with_format_strings assert jake_train_overrides["subscription_name"] == "Jake Trains" - assert jake_train_overrides["subscription_name_sanitized"] == "Jake Trains" assert jake_train_overrides["subscription_value"] == "https://www.youtube.com/@JakeTrains" assert jake_train_overrides["subscription_indent_1"] == "Kids" assert jake_train_overrides["subscription_indent_2"] == "TV-Y" @@ -517,7 +506,6 @@ def test_advanced_tv_show_subscriptions( jake_train_overrides = subs[3].overrides.dict_with_format_strings assert jake_train_overrides["subscription_name"] == "Jake Trains" - assert jake_train_overrides["subscription_name_sanitized"] == "Jake Trains" assert jake_train_overrides["subscription_value"] == "https://www.youtube.com/@JakeTrains" assert jake_train_overrides["subscription_indent_1"] == "Kids" assert jake_train_overrides["subscription_indent_2"] == "TV-Y" @@ -526,10 +514,6 @@ def test_advanced_tv_show_subscriptions( overrides = subs[5].overrides assert overrides.apply_formatter(overrides.dict["subscription_name"]) == "Gardening with Ciscoe" - assert ( - overrides.apply_formatter(overrides.dict["subscription_name_sanitized"]) - == "Gardening with Ciscoe" - ) assert ( overrides.apply_formatter(overrides.dict["url"]) == "https://www.youtube.com/@gardeningwithciscoe4430" @@ -550,7 +534,6 @@ def test_music_subscriptions(default_config: ConfigFile, music_subscriptions_pat monk = subs[2].overrides.dict_with_format_strings assert monk["subscription_name"] == "Stan Getz" - assert monk["subscription_name_sanitized"] == "Stan Getz" assert monk["subscription_value"] == "https://www.youtube.com/@stangetzofficial/releases" assert monk["subscription_indent_1"] == "Jazz" @@ -565,7 +548,6 @@ def test_music_video_subscriptions(default_config: ConfigFile, music_video_subsc monk = subs[1].overrides.dict_with_format_strings assert monk["subscription_name"] == "Michael Jackson" - assert monk["subscription_name_sanitized"] == "Michael Jackson" assert ( monk["subscription_value"] == "https://www.youtube.com/playlist?list=OLAK5uy_mnY03zP6abNWH929q2XhGzWD_2uKJ_n8E" diff --git a/tests/unit/conftest.py b/tests/unit/conftest.py index 672361dd..b20b302c 100644 --- a/tests/unit/conftest.py +++ b/tests/unit/conftest.py @@ -12,18 +12,10 @@ from resources import copy_file_fixture from ytdl_sub.config.config_file import ConfigFile from ytdl_sub.downloaders.url.downloader import MultiUrlDownloader from ytdl_sub.downloaders.ytdlp import YTDLP -from ytdl_sub.entries.variables.kwargs import DESCRIPTION -from ytdl_sub.entries.variables.kwargs import EPOCH -from ytdl_sub.entries.variables.kwargs import EXT -from ytdl_sub.entries.variables.kwargs import EXTRACTOR -from ytdl_sub.entries.variables.kwargs import PLAYLIST_COUNT -from ytdl_sub.entries.variables.kwargs import PLAYLIST_ENTRY -from ytdl_sub.entries.variables.kwargs import PLAYLIST_INDEX -from ytdl_sub.entries.variables.kwargs import PLAYLIST_TITLE -from ytdl_sub.entries.variables.kwargs import TITLE -from ytdl_sub.entries.variables.kwargs import UID -from ytdl_sub.entries.variables.kwargs import UPLOAD_DATE -from ytdl_sub.entries.variables.kwargs import WEBPAGE_URL +from ytdl_sub.entries.script.variable_definitions import VARIABLES +from ytdl_sub.entries.script.variable_definitions import VariableDefinitions + +v: VariableDefinitions = VARIABLES @pytest.fixture @@ -62,23 +54,23 @@ def mock_entry_dict_factory(mock_downloaded_file_path) -> Callable: is_extracted_audio: bool = False, ) -> Dict: entry_dict = { - UID: uid, - EPOCH: 1596878400, - PLAYLIST_TITLE: playlist_title, - PLAYLIST_INDEX: playlist_index, - PLAYLIST_COUNT: playlist_count, - EXTRACTOR: "mock-entry-dict", - "extractor_key": "mock-entry-dict", - TITLE: f"Mock Entry {uid}", - EXT: "mp4", - UPLOAD_DATE: upload_date, - WEBPAGE_URL: f"https://{uid}.com", - PLAYLIST_ENTRY: {"thumbnails": []}, - DESCRIPTION: "The Description", + v.uid.metadata_key: uid, + v.epoch.metadata_key: 1596878400, + v.playlist_title.metadata_key: playlist_title, + v.playlist_index.metadata_key: playlist_index, + v.playlist_count.metadata_key: playlist_count, + v.extractor.metadata_key: "mock-entry-extractor", + v.extractor_key.metadata_key: "mock-entry-dict", + v.title.metadata_key: f"Mock Entry {uid}", + v.ext.metadata_key: "mp4", + v.upload_date.metadata_key: upload_date, + v.webpage_url.metadata_key: f"https://{uid}.com", + v.playlist_metadata.metadata_key: {"thumbnails": []}, + v.description.metadata_key: "The Description", } if is_youtube_channel: - entry_dict[PLAYLIST_ENTRY]["thumbnails"] = [ + entry_dict[v.playlist_metadata.metadata_key]["thumbnails"] = [ { "id": "avatar_uncropped", "url": "https://avatar_uncropped.com", diff --git a/tests/unit/entries/conftest.py b/tests/unit/entries/conftest.py index f4895fac..9421d35e 100644 --- a/tests/unit/entries/conftest.py +++ b/tests/unit/entries/conftest.py @@ -163,7 +163,7 @@ def mock_entry_kwargs( "id": uid, "epoch": 1596878400, "extractor": extractor, - "extractor_key": extractor, + "extractor_key": "test_extractor_key", "title": title, "ext": ext, "upload_date": upload_date, @@ -174,20 +174,4 @@ def mock_entry_kwargs( @pytest.fixture def mock_entry(mock_entry_kwargs): - return Entry(entry_dict=mock_entry_kwargs, working_directory=".") - - -@pytest.fixture -def validate_entry_dict_contains_valid_formatters(): - def _validate_entry_dict_contains_valid_formatters(entry: Entry): - for key, value in entry.to_dict().items(): - expected_string = f"test {value} formatting works" - formatter = StringFormatterValidator( - name="test", value=f"test {{{key}}} formatting works" - ) - - assert formatter.apply_formatter(entry.to_dict()) == expected_string - - return True - - return _validate_entry_dict_contains_valid_formatters + return Entry(entry_dict=mock_entry_kwargs, working_directory=".").initialize_script() diff --git a/tests/unit/entries/test_entry.py b/tests/unit/entries/test_entry.py index 4885420d..4797092d 100644 --- a/tests/unit/entries/test_entry.py +++ b/tests/unit/entries/test_entry.py @@ -1,22 +1,19 @@ import pytest +from ytdl_sub.entries.entry import Entry +from ytdl_sub.entries.script.variable_definitions import VARIABLES +from ytdl_sub.entries.script.variable_definitions import VariableDefinitions + +v: VariableDefinitions = VARIABLES + class TestEntry(object): def test_entry_to_dict(self, mock_entry, mock_entry_to_dict): - assert mock_entry.to_dict() == mock_entry_to_dict + out = mock_entry.to_dict() - def test_entry_dict_contains_valid_formatters( - self, mock_entry, validate_entry_dict_contains_valid_formatters - ): - assert validate_entry_dict_contains_valid_formatters(mock_entry) - - def test_entry_missing_kwarg(self, mock_entry): - key = "dne" - expected_error_msg = f"Expected '{key}' in Entry but does not exist." - - assert mock_entry.kwargs_contains(key) is False - with pytest.raises(KeyError, match=expected_error_msg): - mock_entry.kwargs(key) + # HACK: Ensure legacy variables are in new output and equal + for key, expected_value in mock_entry_to_dict.items(): + assert out[key] == expected_value, f"{key} does not equal" @pytest.mark.parametrize( "upload_date, year_rev, month_rev, day_rev, month_rev_pad, day_rev_pad", @@ -26,16 +23,23 @@ class TestEntry(object): ], ) def test_entry_reverse_variables( - self, mock_entry, upload_date, year_rev, month_rev, day_rev, month_rev_pad, day_rev_pad + self, + mock_entry_kwargs, + upload_date, + year_rev, + month_rev, + day_rev, + month_rev_pad, + day_rev_pad, ): - mock_entry._kwargs["upload_date"] = upload_date - assert mock_entry.upload_year_truncated_reversed == year_rev - assert mock_entry.upload_month_reversed == month_rev - assert mock_entry.upload_day_reversed == day_rev - - assert mock_entry.upload_month_reversed_padded == month_rev_pad - assert mock_entry.upload_day_reversed_padded == day_rev_pad + mock_entry_kwargs["upload_date"] = upload_date + entry = Entry(entry_dict=mock_entry_kwargs, working_directory=".").initialize_script() + assert entry.get(v.upload_year_truncated_reversed, int) == year_rev + assert entry.get(v.upload_month_reversed, int) == month_rev + assert entry.get(v.upload_day_reversed, int) == day_rev + assert entry.get(v.upload_month_reversed_padded, str) == month_rev_pad + assert entry.get(v.upload_day_reversed_padded, str) == day_rev_pad @pytest.mark.parametrize( "upload_date, day_year, day_year_rev, day_year_pad, day_year_rev_pad", @@ -45,11 +49,12 @@ class TestEntry(object): ], ) def test_entry_upload_day_of_year_variables( - self, mock_entry, upload_date, day_year, day_year_rev, day_year_pad, day_year_rev_pad + self, mock_entry_kwargs, upload_date, day_year, day_year_rev, day_year_pad, day_year_rev_pad ): - mock_entry._kwargs["upload_date"] = upload_date + mock_entry_kwargs["upload_date"] = upload_date + entry = Entry(entry_dict=mock_entry_kwargs, working_directory=".").initialize_script() - assert mock_entry.upload_day_of_year == day_year - assert mock_entry.upload_day_of_year_reversed == day_year_rev - assert mock_entry.upload_day_of_year_padded == day_year_pad - assert mock_entry.upload_day_of_year_reversed_padded == day_year_rev_pad + assert entry.get(v.upload_day_of_year, int) == day_year + assert entry.get(v.upload_day_of_year_reversed, int) == day_year_rev + assert entry.get(v.upload_day_of_year_padded, str) == day_year_pad + assert entry.get(v.upload_day_of_year_reversed_padded, str) == day_year_rev_pad diff --git a/tests/unit/prebuilt_presets/test_prebuilt_presets.py b/tests/unit/prebuilt_presets/test_prebuilt_presets.py index 90b0b993..0f7cf5a2 100644 --- a/tests/unit/prebuilt_presets/test_prebuilt_presets.py +++ b/tests/unit/prebuilt_presets/test_prebuilt_presets.py @@ -149,6 +149,7 @@ class TestPrebuiltTVShowPresets: is_youtube_channel: bool, is_many_urls: bool, ): + expected_summary_name = "unit/{}/{}/is_yt_{}{}".format( media_player_preset, tv_show_structure_preset, @@ -201,20 +202,22 @@ class TestPrebuiltTVShowPresets: "_many_urls" if is_many_urls else "", ) - reformatted_subscription = Subscription.from_dict( - config=config, - preset_name=subscription_name, - preset_dict={ - "preset": parent_presets + [reformatted_tv_show_structure_preset], - "output_options": { - "migrated_download_archive_name": ".ytdl-sub-{tv_show_name_sanitized}-download-archive.json" - }, - "overrides": { - "url": "https://your.name.here", - "tv_show_name": "Best Prebuilt TV Show by Date", - "tv_show_directory": output_directory, - }, + reformatted_preset_dict = { + "preset": parent_presets + [reformatted_tv_show_structure_preset], + "output_options": { + "migrated_download_archive_name": ".ytdl-sub-{tv_show_name_sanitized}-download-archive.json" }, + "overrides": { + "url": "https://your.name.here", + "tv_show_name": "Best Prebuilt TV Show by Date", + "tv_show_directory": output_directory, + }, + } + if is_many_urls: + reformatted_preset_dict["overrides"]["url2"] = "https://url.number.2.here" + + reformatted_subscription = Subscription.from_dict( + config=config, preset_name=subscription_name, preset_dict=reformatted_preset_dict ) reformatted_transaction_log = reformatted_subscription.update_with_info_json(dry_run=False) diff --git a/tests/unit/script/__init__.py b/tests/unit/script/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/unit/script/conftest.py b/tests/unit/script/conftest.py new file mode 100644 index 00000000..951f5aff --- /dev/null +++ b/tests/unit/script/conftest.py @@ -0,0 +1,14 @@ +from ytdl_sub.script.script import Script + + +def single_variable_output(script: str): + output = ( + Script( + { + "output": script, + } + ) + .resolve(update=True) + .get_native("output") + ) + return output diff --git a/tests/unit/script/functions/__init__.py b/tests/unit/script/functions/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/unit/script/functions/test_array_functions.py b/tests/unit/script/functions/test_array_functions.py new file mode 100644 index 00000000..7ad81b5d --- /dev/null +++ b/tests/unit/script/functions/test_array_functions.py @@ -0,0 +1,98 @@ +import pytest +from unit.script.conftest import single_variable_output + +from ytdl_sub.script.script import Script +from ytdl_sub.script.utils.exceptions import FunctionRuntimeException + + +class TestArrayFunctions: + def test_array_extend(self): + output = ( + Script( + { + "array1": "{['a']}", + "array2": "{['b']}", + "array3": "{['c']}", + "output": "{%array_extend(array1, array2, array3)}", + } + ) + .resolve(update=True) + .get("output") + .native + ) + assert output == ["a", "b", "c"] + + def test_array_at(self): + output = single_variable_output("{%array_at(['a', 'b', 'c'], 1)}") + assert output == "b" + + def test_array_flatten(self): + output = single_variable_output("{%array_flatten(['a', ['b'], [['c']]])}") + assert output == ["a", "b", "c"] + + def test_array_contains(self): + output = single_variable_output("{%array_contains(['a', ['b'], [['c']]], [['c']])}") + assert output is True + + def test_array_index(self): + output = single_variable_output("{%array_index(['a', ['b'], [['c']]], [['c']])}") + assert output == 2 + + def test_array_slice(self): + output = single_variable_output("{%array_slice(['a', ['b'], [['c']]], 1, -1)}") + assert output == [["b"]] + + def test_array_reverse(self): + output = single_variable_output("{%array_reverse(['a', 'b', 'c'])}") + assert output == ["c", "b", "a"] + + def test_array_product(self): + output = single_variable_output("{%array_product(['a', 'b', 'c'], ['arg'])}") + assert output == [["a", "arg"], ["b", "arg"], ["c", "arg"]] + + def test_array_apply(self): + output = single_variable_output("{%array_apply(['a', 'b', 'c'], %capitalize)}") + assert output == ["A", "B", "C"] + + def test_array_reduce(self): + output = single_variable_output("{%array_reduce([1, 2, 3, 4], %add)}") + assert output == 10 + + def test_array_enumerate(self): + output = ( + Script( + { + "%enumerate_output": "{[$0, $1]}", + "array1": "{['a', 'b', 'c']}", + "output": "{%array_enumerate(array1, %enumerate_output)}", + } + ) + .resolve(update=True) + .get("output") + .native + ) + assert output == [[0, "a"], [1, "b"], [2, "c"]] + + def test_cast_array(self): + output = ( + Script( + { + "map_test": "{ {'key': [1, 2, 3]} }", + "output": "{ %array( %map_get(map_test, 'key') )}", + } + ) + .resolve(update=True) + .get("output") + .native + ) + assert output == [1, 2, 3] + + def test_array_size(self): + output = single_variable_output("{%array_size([1, 2, 3])}") + assert output == 3 + + def test_cast_array_errors_cannot_cast(self): + with pytest.raises( + FunctionRuntimeException, match="Tried and failed to cast Integer as an Array" + ): + single_variable_output("{%array(1)}") diff --git a/tests/unit/script/functions/test_boolean_functions.py b/tests/unit/script/functions/test_boolean_functions.py new file mode 100644 index 00000000..aab6afdf --- /dev/null +++ b/tests/unit/script/functions/test_boolean_functions.py @@ -0,0 +1,128 @@ +import pytest +from unit.script.conftest import single_variable_output + +from ytdl_sub.script.script import Script + + +class TestBooleanFunctions: + @pytest.mark.parametrize( + "lhs, rhs, expected_output", + [ + ("'abc'", "'abc'", True), + ("123", "123", True), + ("3.14", "3.14", True), + ("True", "True", True), + ("False", "False", True), + ("[1, 2, 3]", "[1, 2, 3]", True), + ("{'key': 'value'}", "{'key': 'value'}", True), + ("{'key': 'value'}", 5, False), + ("{'key': 'value'}", "[1, 2, 3]", False), + ], + ) + @pytest.mark.parametrize("is_ne", [True, False]) + def test_eq_ne(self, lhs: str, rhs: str, expected_output: bool, is_ne: bool): + op = "ne" if is_ne else "eq" + output = single_variable_output(f"{{%{op}({lhs}, {rhs})}}") + + if is_ne: + assert output != expected_output + else: + assert output == expected_output + + @pytest.mark.parametrize( + "lhs, rhs, expected_output", + [ + ("'abc'", "'abc'", True), + ("123", "123", True), + ("3.14", "3.14", True), + ("3.14", "4.0", True), + ], + ) + @pytest.mark.parametrize("is_gt", [True, False]) + def test_lte_gt(self, lhs: str, rhs: str, expected_output: bool, is_gt: bool): + op = "gt" if is_gt else "lte" + output = single_variable_output(f"{{%{op}({lhs}, {rhs})}}") + + if is_gt: + assert output != expected_output + else: + assert output == expected_output + + @pytest.mark.parametrize( + "lhs, rhs, expected_output", + [ + ("'abc'", "'abc'", True), + ("123", "123", True), + ("3.14", "3.14", True), + ("5.32", "4", True), + ("3.14", "4.0", False), + ], + ) + @pytest.mark.parametrize("is_lt", [True, False]) + def test_gte_lt(self, lhs: str, rhs: str, expected_output: bool, is_lt: bool): + op = "lt" if is_lt else "gte" + output = single_variable_output(f"{{%{op}({lhs}, {rhs})}}") + + if is_lt: + assert output != expected_output + else: + assert output == expected_output + + @pytest.mark.parametrize( + "values, expected_output", + [ + ("True", True), + ("True, True", True), + ("True, True, True", True), + ("False", False), + ("False, True", False), + ("True, False, True", False), + ], + ) + def test_and(self, values: str, expected_output: bool): + output = single_variable_output(f"{{%and({values})}}") + assert output == expected_output + + @pytest.mark.parametrize( + "values, expected_output", + [ + ("True", True), + ("True, True", True), + ("True, True, True", True), + ("False", False), + ("False, True", True), + ("True, False, True", True), + ("False, False, False", False), + ], + ) + def test_or(self, values: str, expected_output: bool): + output = single_variable_output(f"{{%or({values})}}") + assert output == expected_output + + @pytest.mark.parametrize( + "values, expected_output", + [ + ("True", True), + ("True, True", False), + ("True, True, True", False), + ("False", False), + ("False, True", True), + ("True, False, True", False), + ("False, False, False", False), + ("False, True, False", True), + ], + ) + def test_xor(self, values: str, expected_output: bool): + output = single_variable_output(f"{{%xor({values})}}") + assert output == expected_output + + @pytest.mark.parametrize( + "value, expected_output", + [ + ("True", False), + ("False", True), + ], + ) + def test_not(self, value: str, expected_output: bool): + output = single_variable_output(f"{{%not({value})}}") + assert output == expected_output diff --git a/tests/unit/script/functions/test_conditional_functions.py b/tests/unit/script/functions/test_conditional_functions.py new file mode 100644 index 00000000..259c0a33 --- /dev/null +++ b/tests/unit/script/functions/test_conditional_functions.py @@ -0,0 +1,35 @@ +import pytest + +from ytdl_sub.script.script import Script + + +class TestConditionalFunction: + @pytest.mark.parametrize( + "function_str, expected_output", + [ + ("{%if(True, True, False)}", True), + ("{%if(False, True, False)}", False), + ], + ) + def test_if_function(self, function_str: str, expected_output: bool): + output = Script({"output": function_str}).resolve(update=True).get("output").native + assert output == expected_output + + def test_nested_if_function(self): + function_str = """{ + %if( + True, + %if( + True, + %if( + True, + "winner", + True + ), + True + ), + True + ) + }""" + output = Script({"output": function_str}).resolve(update=True).get("output").native + assert output == "winner" diff --git a/tests/unit/script/functions/test_date_functions.py b/tests/unit/script/functions/test_date_functions.py new file mode 100644 index 00000000..1c826ea4 --- /dev/null +++ b/tests/unit/script/functions/test_date_functions.py @@ -0,0 +1,17 @@ +import pytest +from unit.script.conftest import single_variable_output + +from ytdl_sub.script.script import Script + + +class TestNumericFunctions: + @pytest.mark.parametrize( + "timestamp, date_format, expected_output", + [ + (1596877200, "'%Y%m%d'", "20200808"), + (1596877200, "'%m'", "08"), + ], + ) + def test_datetime_strftime(self, timestamp: int, date_format: str, expected_output: str): + output = single_variable_output(f"{{%datetime_strftime({timestamp}, {date_format})}}") + assert output == expected_output diff --git a/tests/unit/script/functions/test_error_functions.py b/tests/unit/script/functions/test_error_functions.py new file mode 100644 index 00000000..9f1092b4 --- /dev/null +++ b/tests/unit/script/functions/test_error_functions.py @@ -0,0 +1,25 @@ +import re + +import pytest +from unit.script.conftest import single_variable_output + +from ytdl_sub.script.script import Script +from ytdl_sub.script.utils.exceptions import UserThrownRuntimeError + + +class TestErrorFunctions: + def test_user_throw(self): + with pytest.raises(UserThrownRuntimeError, match=re.escape("test this error message")): + Script({"throw_error": "{%throw('test this error message')}"}).resolve() + + def test_user_assert_raises(self): + with pytest.raises(UserThrownRuntimeError, match=re.escape("test this error message")): + Script({"throw_error": "{%assert(False, 'test this error message')}"}).resolve() + + def test_user_assert_passthrough(self): + output = single_variable_output("{%assert(['a'], 'test this error message')}") + assert output == ["a"] + + def test_user_assert_passthrough_as_arg(self): + output = single_variable_output("{%int(%assert('123', 'test this error message'))}") + assert output == 123 diff --git a/tests/unit/script/functions/test_json_functions.py b/tests/unit/script/functions/test_json_functions.py new file mode 100644 index 00000000..b2477ac5 --- /dev/null +++ b/tests/unit/script/functions/test_json_functions.py @@ -0,0 +1,31 @@ +import json + +import pytest +from unit.script.conftest import single_variable_output + + +class TestJsonFunctions: + @pytest.mark.parametrize("str_token", ["'''", '"""']) + def test_from_json(self, str_token: str): + json_dict = { + "string": "value", + "quotes": "has '' and \"\"", + "int": 1, + "bool": True, + "list": [1, 2, 3], + "dict": {"a": 1, "b": 2}, + "float": 3.14, + "nested_dict": { + "string": "value", + "int": 1, + "bool": True, + "list": [1, 2, 3], + "dict": {"a": 1, "b": 2}, + "float": 3.14, + }, + } + + output = single_variable_output( + f"{{ %from_json({str_token}{json.dumps(json_dict)}{str_token}) }}" + ) + assert output == json_dict diff --git a/tests/unit/script/functions/test_map_functions.py b/tests/unit/script/functions/test_map_functions.py new file mode 100644 index 00000000..709445e6 --- /dev/null +++ b/tests/unit/script/functions/test_map_functions.py @@ -0,0 +1,140 @@ +import re + +import pytest +from unit.script.conftest import single_variable_output + +from ytdl_sub.script.script import Script +from ytdl_sub.script.utils.exceptions import FunctionRuntimeException +from ytdl_sub.script.utils.exceptions import KeyDoesNotExistRuntimeException +from ytdl_sub.script.utils.exceptions import KeyNotHashableRuntimeException + + +class TestMapFunctions: + def test_map_get(self): + output = ( + Script( + { + "input_map": "{{'key': 'value'}}", + "output": "{%map_get(input_map, 'key')}", + } + ) + .resolve(update=True) + .get("output") + .native + ) + assert output == "value" + + def test_map_get_optional(self): + output = ( + Script( + { + "input_map": "{{'key': 'value'}}", + "output": "{%map_get(input_map, 'dne', 'optional_value')}", + } + ) + .resolve(update=True) + .get("output") + .native + ) + assert output == "optional_value" + + def test_map_get_errors_missing_key(self): + with pytest.raises( + KeyDoesNotExistRuntimeException, + match=re.escape("Tried to call %map_get with key dne, but it does not exist"), + ): + Script( + { + "input_map": "{{'key': 'value'}}", + "output": "{%map_get(input_map, 'dne')}", + } + ).resolve() + + def test_map_get_errors_key_not_hashable(self): + with pytest.raises( + KeyNotHashableRuntimeException, + match=re.escape("Tried to use Array as a Map key, but it is not hashable."), + ): + Script( + { + "non_hashable_key": "{%array([1, 2, ['nest']])}", + "input_map": "{{'key': 'value'}}", + "output": "{ %map_get( input_map, %array_at(non_hashable_key, 2)) }", + } + ).resolve() + + @pytest.mark.parametrize( + "contains_value, expected_value", + [ + ("'key'", True), + ("'dne'", False), + ("%string(%array_at(['dne', 'key'], 1))", True), + ], + ) + def test_map_contains(self, contains_value: str, expected_value: bool): + output = ( + Script( + { + "input_map": "{{'key': 'value'}}", + "output": f"{{%map_contains(input_map, {contains_value})}}", + } + ) + .resolve(update=True) + .get("output") + .native + ) + assert output == expected_value + + def test_map_apply(self): + output = ( + Script( + { + "%custom_func": "{[%upper($0), %lower($1)]}", + "map1": "{{'Key1': 'Value1', 'Key2': 'Value2'}}", + "output": "{%map_apply(map1, %custom_func)}", + } + ) + .resolve(update=True) + .get("output") + .native + ) + assert output == [["KEY1", "value1"], ["KEY2", "value2"]] + + def test_map_enumerate(self): + output = ( + Script( + { + "%custom_func": "{[$0, %upper($1), %lower($2)]}", + "map1": "{{'Key1': 'Value1', 'Key2': 'Value2'}}", + "output": "{%map_enumerate(map1, %custom_func)}", + } + ) + .resolve(update=True) + .get("output") + .native + ) + assert output == [[0, "KEY1", "value1"], [1, "KEY2", "value2"]] + + def test_map_size(self): + output = single_variable_output("{%map_size({'key': 'value', 1: 3})}") + assert output == 2 + + def test_cast_map(self): + output = ( + Script( + { + "array_test": "{ [1, 2, {'key': 'value'}] }", + "output": "{ %map( %array_at(array_test, 2) )}", + } + ) + .resolve(update=True) + .get("output") + .native + ) + assert output == {"key": "value"} + + def test_cast_map_errors_cannot_cast(self): + with pytest.raises( + FunctionRuntimeException, match="Tried and failed to cast Integer as a Map" + ): + single_variable_output("{%map(1)}") diff --git a/tests/unit/script/functions/test_numeric_functions.py b/tests/unit/script/functions/test_numeric_functions.py new file mode 100644 index 00000000..c41f4c59 --- /dev/null +++ b/tests/unit/script/functions/test_numeric_functions.py @@ -0,0 +1,78 @@ +import pytest +from unit.script.conftest import single_variable_output + + +class TestNumericFunctions: + @pytest.mark.parametrize( + "values, expected_output", [("1, 2, 3", 6), ("1", 1), ("-1, -2, -3", -6), ("1.1, 1.2", 2.3)] + ) + def test_add(self, values: str, expected_output: int): + output = single_variable_output(f"{{%add({values})}}") + assert output == expected_output + + @pytest.mark.parametrize( + "values, expected_output", [("1, 2, 3", -4), ("1", 1), ("-1, -2, -3", 4), ("1.5, 2.5", -1)] + ) + def test_sub(self, values: str, expected_output: int): + output = single_variable_output(f"{{%sub({values})}}") + assert output == expected_output + + @pytest.mark.parametrize( + "values, expected_output", + [("1, 2, 3", 6), ("1", 1), ("-1, -2, -3", -6), ("1.5, 2.5", 3.75)], + ) + def test_mul(self, values: str, expected_output: int): + output = single_variable_output(f"{{%mul({values})}}") + assert output == expected_output + + @pytest.mark.parametrize( + "values, expected_output", [("2, 2", 1), ("10, 5", 2), ("4.5, 0.5", 9), ("-3.5, -2", 1.75)] + ) + def test_div(self, values: str, expected_output: int): + output = single_variable_output(f"{{%div({values})}}") + assert output == expected_output + + @pytest.mark.parametrize( + "values, expected_output", + [ + ("8, 3", 2), + ("1, 1", 0), + ], + ) + def test_mod(self, values: str, expected_output: int): + output = single_variable_output(f"{{%mod({values})}}") + assert output == expected_output + + @pytest.mark.parametrize( + "values, expected_output", + [ + ("8, 3, 0.3, 0.2, 1.4, 99.9", 99.9), + ("1, 1, 0, 1", 1), + ], + ) + def test_max(self, values: str, expected_output: int): + output = single_variable_output(f"{{%max({values})}}") + assert output == expected_output + + @pytest.mark.parametrize( + "values, expected_output", + [ + ("8, 3, 0.3, 0.2, 1.4, 99.9", 0.2), + ("1, 1, 0, 1", 0), + ], + ) + def test_min(self, values: str, expected_output: int): + output = single_variable_output(f"{{%min({values})}}") + assert output == expected_output + + @pytest.mark.parametrize( + "values, expected_output", + [ + ("2, 2", 2**2), + ("9, 0.5", 9**0.5), + ("4.4, 2.2", 4.4**2.2), + ], + ) + def test_pow(self, values: str, expected_output: float): + output = single_variable_output(f"{{ %pow({values}) }}") + assert output == expected_output diff --git a/tests/unit/script/functions/test_regex_functions.py b/tests/unit/script/functions/test_regex_functions.py new file mode 100644 index 00000000..b0ca2e16 --- /dev/null +++ b/tests/unit/script/functions/test_regex_functions.py @@ -0,0 +1,45 @@ +import pytest +from unit.script.conftest import single_variable_output + +from ytdl_sub.script.script import Script + + +class TestNumericFunctions: + @pytest.mark.parametrize( + "values, expected_output", + [ + ("'ow', 'lower'", []), + ("'.*ow.*', 'lower'", ["lower"]), + ("'.*(ow).*', 'lower'", ["lower", "ow"]), + ("'(.*)(ow)(.*)', 'lower'", ["lower", "l", "ow", "er"]), + ], + ) + def test_regex_match(self, values: str, expected_output: str): + output = single_variable_output(f"{{%regex_match({values})}}") + assert output == expected_output + + @pytest.mark.parametrize( + "values, expected_output", + [ + ("'ow', 'lower'", ["lower"]), + ("'.*ow.*', 'lower'", ["lower"]), + ("'.*(ow).*', 'lower'", ["lower", "ow"]), + ("'(.*)(ow)(.*)', 'lower'", ["lower", "l", "ow", "er"]), + ], + ) + def test_regex_search(self, values: str, expected_output: str): + output = single_variable_output(f"{{%regex_search({values})}}") + assert output == expected_output + + @pytest.mark.parametrize( + "values, expected_output", + [ + ("'ow', 'lower'", []), + ("'.*ow.*', 'lower'", ["lower"]), + ("'.*(ow).*', 'lower'", ["lower", "ow"]), + ("'(.*)(ow)(.*)', 'lower'", ["lower", "l", "ow", "er"]), + ], + ) + def test_regex_fullmatch(self, values: str, expected_output: str): + output = single_variable_output(f"{{%regex_fullmatch({values})}}") + assert output == expected_output diff --git a/tests/unit/script/functions/test_string_functions.py b/tests/unit/script/functions/test_string_functions.py new file mode 100644 index 00000000..4533d5bf --- /dev/null +++ b/tests/unit/script/functions/test_string_functions.py @@ -0,0 +1,109 @@ +import pytest +from unit.script.conftest import single_variable_output + + +class TestNumericFunctions: + @pytest.mark.parametrize( + "values, expected_output", + [ + ("'lower'", "LOWER"), + ("'UPPER'", "UPPER"), + ], + ) + def test_upper(self, values: str, expected_output: str): + output = single_variable_output(f"{{%upper({values})}}") + assert output == expected_output + + @pytest.mark.parametrize( + "values, expected_output", + [ + ("'lower'", "lower"), + ("'UPPER'", "upper"), + ], + ) + def test_lower(self, values: str, expected_output: str): + output = single_variable_output(f"{{%lower({values})}}") + assert output == expected_output + + @pytest.mark.parametrize( + "values, expected_output", + [ + ("'lower First word'", "Lower first word"), + ("'UPPER First word'", "Upper first word"), + ], + ) + def test_capitalize(self, values: str, expected_output: str): + output = single_variable_output(f"{{%capitalize({values})}}") + assert output == expected_output + + @pytest.mark.parametrize( + "values, expected_output", + [ + ("'lower First word'", "Lower First Word"), + ("'UPPER First word'", "Upper First Word"), + ], + ) + def test_titlecase(self, values: str, expected_output: str): + output = single_variable_output(f"{{%titlecase({values})}}") + assert output == expected_output + + @pytest.mark.parametrize( + "values, expected_output", + [ + ( + "'lower First word second word', 'word', 'string'", + "lower First string second string", + ), + ( + "'lower First word second word', 'word', 'string', 1", + "lower First string second word", + ), + ], + ) + def test_replace(self, values: str, expected_output: str): + output = single_variable_output(f"{{%replace({values})}}") + assert output == expected_output + + @pytest.mark.parametrize( + "values, expected_output", + [ + ("'lower First word'", "lower First word"), + ("'lower First word', ' second', ' string'", "lower First word second string"), + ], + ) + def test_concat(self, values: str, expected_output: str): + output = single_variable_output(f"{{%concat({values})}}") + assert output == expected_output + + @pytest.mark.parametrize( + "values, expected_output", + [ + ("'HI', 6, '.'", "....HI"), + ("'HI', 2, '.'", "HI"), + ], + ) + def test_pad(self, values: str, expected_output: str): + output = single_variable_output(f"{{%pad({values})}}") + assert output == expected_output + + @pytest.mark.parametrize( + "values, expected_output", + [ + ("2012, 6", "002012"), + ("2012, 2", "2012"), + ], + ) + def test_pad_zero(self, values: str, expected_output: str): + output = single_variable_output(f"{{%pad_zero({values})}}") + assert output == expected_output + + @pytest.mark.parametrize( + "values, expected_output", + [ + ("'2012', 2", "12"), + ("'2012', 1, 3", "01"), + ], + ) + def test_slice(self, values, expected_output): + output = single_variable_output(f"{{%slice({values})}}") + assert output == expected_output diff --git a/tests/unit/script/test_parser.py b/tests/unit/script/test_parser.py new file mode 100644 index 00000000..196ae8f2 --- /dev/null +++ b/tests/unit/script/test_parser.py @@ -0,0 +1,210 @@ +import re +from typing import Optional +from typing import Union + +import pytest + +from ytdl_sub.script.parser import _UNEXPECTED_CHAR_ARGUMENT +from ytdl_sub.script.parser import BRACKET_NOT_CLOSED +from ytdl_sub.script.parser import ParsedArgType +from ytdl_sub.script.parser import parse +from ytdl_sub.script.types.array import UnresolvedArray +from ytdl_sub.script.types.function import BuiltInFunction +from ytdl_sub.script.types.resolvable import Boolean +from ytdl_sub.script.types.resolvable import Float +from ytdl_sub.script.types.resolvable import Integer +from ytdl_sub.script.types.resolvable import Lambda +from ytdl_sub.script.types.resolvable import String +from ytdl_sub.script.types.syntax_tree import SyntaxTree +from ytdl_sub.script.types.variable import Variable +from ytdl_sub.script.utils.exceptions import InvalidSyntaxException + + +class TestParser: + def test_simple(self): + parsed = parse("hello world") + assert parsed == SyntaxTree([String(value="hello world")]) + assert parsed.variables == set() + + def test_single_function_one_arg(self): + parsed = parse("hello {%capitalize('hi mom')}") + assert parsed == SyntaxTree( + [ + String("hello "), + BuiltInFunction(name="capitalize", args=[String(value="hi mom")]), + ] + ) + + def test_conditional(self): + parsed = parse("hello {%if(True, 'hi', 3.4)}") + assert parsed == SyntaxTree( + [ + String("hello "), + BuiltInFunction( + name="if", args=[Boolean(value=True), String(value="hi"), Float(value=3.4)] + ), + ] + ) + assert parsed.ast[1].output_type() == Union[String, Float] + + def test_conditional_as_input_same_outputs(self): + parsed = parse("hello {%concat(%if(True, 'hi', 'mom'), 'and dad')}") + assert parsed == SyntaxTree( + [ + String("hello "), + BuiltInFunction( + name="concat", + args=[ + BuiltInFunction( + name="if", args=[Boolean(value=True), String("hi"), String("mom")] + ), + String(value="and dad"), + ], + ), + ] + ) + + def test_conditional_as_input_different_outputs(self): + parsed = parse("hello {%string(%if(True, 'hi', 4))}") + assert parsed == SyntaxTree( + [ + String("hello "), + BuiltInFunction( + name="string", + args=[ + BuiltInFunction(name="if", args=[Boolean(True), String("hi"), Integer(4)]), + ], + ), + ] + ) + + def test_nested_if_output_type(self): + parsed = parse( + """{ + %if( + True, + %if( + True, + %if( + True, + "winner", + True + ), + True + ), + True + ) + }""" + ) + assert len(parsed.ast) == 1 + token = parsed.ast[0] + assert isinstance(token, BuiltInFunction) + assert token.output_type() == Union[String, Boolean] + + def test_single_function_one_vararg(self): + parsed = parse("hello {%concat('hi mom')}") + assert parsed == SyntaxTree( + [ + String("hello "), + BuiltInFunction(name="concat", args=[String(value="hi mom")]), + ] + ) + + def test_single_function_many_vararg(self): + parsed = parse("hello {%concat('hi', 'mom')}") + assert parsed == SyntaxTree( + [ + String("hello "), + BuiltInFunction(name="concat", args=[String(value="hi"), String(value="mom")]), + ] + ) + + def test_single_function_many_args_with_optional_none(self): + parsed = parse("hello {%replace('hi mom', 'hi', '')}") + assert parsed == SyntaxTree( + [ + String("hello "), + BuiltInFunction( + name="replace", + args=[String(value="hi mom"), String(value="hi"), String(value="")], + ), + ] + ) + + def test_single_function_many_args_with_optional_provided(self): + parsed = parse("hello {%replace('hi mom', 'hi', '', 1)}") + assert parsed == SyntaxTree( + [ + String("hello "), + BuiltInFunction( + name="replace", + args=[ + String(value="hi mom"), + String(value="hi"), + String(value=""), + Integer(value=1), + ], + ), + ] + ) + + @pytest.mark.parametrize("whitespace", [None, " ", " ", "\n", " \n "]) + def test_single_function_multiple_args(self, whitespace: Optional[str]): + s = whitespace + if s is None: + s = "" + + input_str = ( + f"hello{s}{{{s}%concat({s}'string'{s},{s}%string(1){s},{s}%string(2.4){s}," + f"{s}%string(True){s},{s}%string(variable_name){s},{s}%capitalize({s}'hi'{s}){s})}}" + f"{s}" + ) + parsed = parse(input_str) + assert parsed == SyntaxTree( + [ + String(value=f"hello{s}"), + BuiltInFunction( + name="concat", + args=[ + String(value="string"), + BuiltInFunction(name="string", args=[Integer(value=1)]), + BuiltInFunction(name="string", args=[Float(value=2.4)]), + BuiltInFunction(name="string", args=[Boolean(value=True)]), + BuiltInFunction(name="string", args=[Variable(name="variable_name")]), + BuiltInFunction(name="capitalize", args=[String(value="hi")]), + ], + ), + ] + + ([String(value=s)] if s else []) + ) + assert parsed.variables == {Variable(name="variable_name")} + + def test_lambda_function(self): + assert parse("{%array_apply([1], %times_two)}") == SyntaxTree( + [ + BuiltInFunction( + name="array_apply", + args=[ + UnresolvedArray(value=[Integer(1)]), + Lambda(value="times_two"), + ], + ) + ] + ) + + +class TestParserBracketFailures: + def test_bracket_open(self): + with pytest.raises(InvalidSyntaxException, match=re.escape(str(BRACKET_NOT_CLOSED))): + parse("{") + + def test_bracket_close(self): + with pytest.raises(InvalidSyntaxException, match=re.escape(str(BRACKET_NOT_CLOSED))): + parse("}") + + def test_bracket_in_function(self): + with pytest.raises( + InvalidSyntaxException, + match=re.escape(str(_UNEXPECTED_CHAR_ARGUMENT(ParsedArgType.MAP_KEY))), + ): + parse("hello {%capitalize({as_arg)}") diff --git a/tests/unit/script/test_script.py b/tests/unit/script/test_script.py new file mode 100644 index 00000000..0b86e2be --- /dev/null +++ b/tests/unit/script/test_script.py @@ -0,0 +1,63 @@ +from ytdl_sub.script.script import Script +from ytdl_sub.script.script_output import ScriptOutput +from ytdl_sub.script.types.map import Map +from ytdl_sub.script.types.resolvable import String + + +class TestScript: + def test_pre_resolved(self): + assert Script( + { + "%custom_func": "return {[$0, $1]}", + "aa": "a", + "bb": "b", + "cc": "{%custom_func(aa, bb)}", + } + ).resolve(resolved={"bb": String("bb_override")}) == ScriptOutput( + { + "aa": String("a"), + "bb": String("bb_override"), + "cc": String('return ["a", "bb_override"]'), + } + ) + + def test_partial_resolve(self): + assert Script( + { + "%custom_func": "return {[$0, $1]}", + "aa": "a", + "bb": "b", + "cc": "{%custom_func(aa, bb)}", + } + ).resolve(unresolvable={"bb"}) == ScriptOutput({"aa": String("a")}) + + def test_partial_update_script(self): + # to be resolved later + entry_map = Map({String("title"): String("the title")}) + + script = Script( + { + "entry": "{%throw('entry has not been populated yet')}", + "title": "{%map_get(entry, 'title')}", + "override": "hi", + "resolved_override": "{override} mom", + } + ) + + script.resolve(unresolvable={"entry"}, update=True) + assert script.get("override") == String("hi") + assert script.get("resolved_override") == String("hi mom") + + script.add( + { + "new_variable_titlecase": "{%titlecase(new_variable_upper)}", + "new_variable": "{resolved_override} {title}", + "new_variable_upper": "{%upper(new_variable)}", + } + ).resolve(resolved={"entry": entry_map}, update=True) + + assert script.get("title") == String("the title") + assert script.get("new_variable") == String("hi mom the title") + assert script.get("new_variable_upper") == String("HI MOM THE TITLE") + assert script.get("new_variable_titlecase") == String("Hi Mom The Title") + assert script.get("entry") == entry_map diff --git a/tests/unit/script/types/__init__.py b/tests/unit/script/types/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/unit/script/types/test_array.py b/tests/unit/script/types/test_array.py new file mode 100644 index 00000000..f816b1bd --- /dev/null +++ b/tests/unit/script/types/test_array.py @@ -0,0 +1,119 @@ +import re + +import pytest + +from ytdl_sub.script.parser import _UNEXPECTED_CHAR_ARGUMENT +from ytdl_sub.script.parser import _UNEXPECTED_COMMA_ARGUMENT +from ytdl_sub.script.parser import ParsedArgType +from ytdl_sub.script.script import Script +from ytdl_sub.script.script_output import ScriptOutput +from ytdl_sub.script.types.array import Array +from ytdl_sub.script.types.resolvable import Float +from ytdl_sub.script.types.resolvable import String +from ytdl_sub.script.utils.exceptions import InvalidSyntaxException + + +class TestArray: + def test_return(self): + assert Script({"arr": "{['a', 3.14]}"}).resolve() == ScriptOutput( + {"arr": Array([String("a"), Float(3.14)])} + ) + + def test_return_as_str(self): + assert Script({"arr": "str: {['a', 3.14]}"}).resolve() == ScriptOutput( + {"arr": String('str: ["a", 3.14]')} + ) + + def test_nested_array(self): + assert Script( + {"arr": "{['level1', ['level2', ['level3', 'level3'], 'level2'], 'level1']}"} + ).resolve() == ScriptOutput( + { + "arr": Array( + [ + String("level1"), + Array( + [ + String("level2"), + Array([String("level3"), String("level3")]), + String("level2"), + ], + ), + String("level1"), + ] + ) + } + ) + + @pytest.mark.parametrize( + "array", + [ + "{[]}", + "{ [] }", + "{ [ ] }", + "{[\n]}", + ], + ) + def test_empty(self, array: str): + assert Script({"arr": array}).resolve() == ScriptOutput({"arr": Array([])}) + + @pytest.mark.parametrize( + "array", + [ + "{[,]}", + "{[ ,]}", + "{ [ , ]}", + "{ ['test',] }", + "{ [ 'test', ] }", + "{[\n,\n]}", + ], + ) + def test_unexpected_comma(self, array: str): + with pytest.raises( + InvalidSyntaxException, + match=re.escape(str(_UNEXPECTED_COMMA_ARGUMENT(ParsedArgType.ARRAY))), + ): + Script({"arr": array}).resolve() + + @pytest.mark.parametrize( + "array", + [ + "{[}", + "{[ }", + "{[\n}", + "{['key'}", + "{[ 'key' }", + ], + ) + def test_array_not_closed(self, array: str): + with pytest.raises( + InvalidSyntaxException, + match=re.escape(str(_UNEXPECTED_CHAR_ARGUMENT(ParsedArgType.ARRAY))), + ): + assert Script({"arr": array}).resolve() + + @pytest.mark.parametrize( + "array", + [ + "{]}" "{ ]}", + "{\n]}", + ], + ) + def test_array_not_opened(self, array: str): + with pytest.raises( + InvalidSyntaxException, + match=re.escape(str(_UNEXPECTED_CHAR_ARGUMENT(ParsedArgType.SCRIPT))), + ): + assert Script({"arr": array}).resolve() + + def test_custom_function(self): + assert Script( + { + "%custom_func": "return {[$0, $1]}", + "aa": "a", + "bb": "b", + "cc": "{%custom_func(aa, bb)}", + } + ).resolve() == ScriptOutput( + {"aa": String("a"), "bb": String("b"), "cc": String('return ["a", "b"]')} + ) diff --git a/tests/unit/script/types/test_bool.py b/tests/unit/script/types/test_bool.py new file mode 100644 index 00000000..5de7ddb1 --- /dev/null +++ b/tests/unit/script/types/test_bool.py @@ -0,0 +1,66 @@ +import re + +import pytest + +from ytdl_sub.script.parser import BOOLEAN_ONLY_ARGS +from ytdl_sub.script.script import Script +from ytdl_sub.script.script_output import ScriptOutput +from ytdl_sub.script.types.resolvable import Boolean +from ytdl_sub.script.types.resolvable import String +from ytdl_sub.script.utils.exceptions import InvalidSyntaxException + + +class TestBool: + @pytest.mark.parametrize( + "boolean", + [ + "{True}", + "{ True }", + "{False}", + "{ False }", + ], + ) + def test_boolean_not_arg(self, boolean: str): + with pytest.raises(InvalidSyntaxException, match=re.escape(str(BOOLEAN_ONLY_ARGS))): + Script({"boolean": boolean}).resolve() + + @pytest.mark.parametrize( + "boolean, expected_boolean", + [ + ("{%bool(True)}", True), + ("{%bool(False)}", False), + ("{%bool( True )}", True), + ("{%bool( False )}", False), + ], + ) + def test_boolean(self, boolean: bool, expected_boolean: bool): + assert Script( + {"boolean": boolean, "as_string": "{%string(boolean)}"} + ).resolve() == ScriptOutput( + { + "boolean": Boolean(expected_boolean), + "as_string": String(str(expected_boolean)), + } + ) + + @pytest.mark.parametrize( + "to_cast, expected_bool", + [ + ("{%bool(False)}", False), + ("{%bool(True)}", True), + ("{%bool(0)}", False), + ("{%bool(1)}", True), + ("{%bool(0.0)}", False), + ("{%bool(0.1)}", True), + ("{%bool('')}", False), + ("{%bool('false')}", True), + ("{%bool([])}", False), + ("{%bool([False])}", True), + ("{%bool({})}", False), + ("{%bool({'key': 'value'})}", True), + ], + ) + def test_cast_as_bool(self, to_cast: str, expected_bool: bool): + assert Script({"as_bool": to_cast}).resolve() == ScriptOutput( + {"as_bool": Boolean(expected_bool)} + ) diff --git a/tests/unit/script/types/test_custom_function.py b/tests/unit/script/types/test_custom_function.py new file mode 100644 index 00000000..e7b76e80 --- /dev/null +++ b/tests/unit/script/types/test_custom_function.py @@ -0,0 +1,235 @@ +import re + +import pytest + +from ytdl_sub.script.parser import CUSTOM_FUNCTION_ARGUMENTS_ONLY_ARGS +from ytdl_sub.script.script import Script +from ytdl_sub.script.script_output import ScriptOutput +from ytdl_sub.script.types.resolvable import Integer +from ytdl_sub.script.utils.exceptions import CycleDetected +from ytdl_sub.script.utils.exceptions import FunctionDoesNotExist +from ytdl_sub.script.utils.exceptions import InvalidCustomFunctionArgumentName +from ytdl_sub.script.utils.exceptions import InvalidCustomFunctionArguments +from ytdl_sub.script.utils.exceptions import InvalidSyntaxException + + +class TestCustomFunction: + def test_custom_function_use_input_param_multiple_times(self): + assert Script( + { + "%custom_square": "{%mul($0, $0)}", + "output": "{%custom_square(3)}", + } + ).resolve() == ScriptOutput({"output": Integer(9)}) + + def test_custom_function_cycle(self): + with pytest.raises( + CycleDetected, match=re.escape("The custom function %cycle_func cannot call itself.") + ): + Script( + {"%cycle_func": "{%mul(%cycle_func(1), $0)}", "output": "{%cycle_func(1)}"} + ).resolve() + + def test_custom_function_chained_cycle(self): + with pytest.raises( + CycleDetected, + match=re.escape( + "Cycle detected within these custom functions: " + "%cycle_func1 -> %cycle_func0 -> %cycle_func1" + ), + ): + Script( + { + "%cycle_func1": "{%mul(%cycle_func0(1), $0)}", + "%cycle_func0": "{%mul(%cycle_func1(1), $0)}", + "output": "{%cycle_func0(1)}", + } + ).resolve() + + def test_custom_function_deep_chained_cycle(self): + with pytest.raises( + CycleDetected, + match=re.escape( + "Cycle detected within these custom functions: " + "%cycle_func4 -> " + "%cycle_func0 -> " + "%cycle_func1 -> " + "%cycle_func2 -> " + "%cycle_func3 -> " + "%cycle_func4" + ), + ): + Script( + { + "%nested_safe_func": "{%mul($0, 1)}", + "%safe_func": "{%nested_safe_func($0, 1)}", + "%cycle_func4": "{%mul(%cycle_func0(1), %safe_func($0))}", + "%cycle_func3": "{%mul(%cycle_func4(1), %safe_func($0))}", + "%cycle_func2": "{%mul(%cycle_func3(1), %safe_func($0))}", + "%cycle_func1": "{%mul(%cycle_func2(1), %safe_func($0))}", + "%cycle_func0": "{%mul(%cycle_func1(1), %safe_func($0))}", + "output": "{%cycle_func0(1)}", + } + ).resolve() + + def test_custom_function_uses_non_existent_function(self): + with pytest.raises( + FunctionDoesNotExist, + match=re.escape("Function %lolnope does not exist as a built-in or custom function."), + ): + Script( + { + "%func1": "{%mul(%lolnope(1), $0)}", + "%func0": "{%mul(%func1(1), $0)}", + "output": "{%func(1)}", + } + ).resolve() + + @pytest.mark.parametrize( + "name", + [ + "$00invalid", + "$abc", + "$3.14", + ], + ) + def test_custom_function_invalid_function_argument_names(self, name: str): + with pytest.raises( + InvalidCustomFunctionArgumentName, + match=re.escape( + "Custom function arguments must be numeric and increment starting from zero." + ), + ): + Script( + { + "%func1": f"{{%mul(1, {name})}}", + "%func0": "{%mul(%func1(1), $0)}", + "output": "{%func0(1)}", + } + ).resolve() + + def test_custom_function_function_argument_usage_in_brackets(self): + with pytest.raises( + InvalidSyntaxException, + match=re.escape(str(CUSTOM_FUNCTION_ARGUMENTS_ONLY_ARGS)), + ): + Script({"%func1": "{$0}"}).resolve() + + @pytest.mark.parametrize( + "argument", + [ + "$1", + "$2", + "$3", + ], + ) + def test_custom_function_invalid_function_argument_single(self, argument: str): + with pytest.raises( + InvalidCustomFunctionArguments, + match=re.escape( + f"Custom function %func1 has invalid function arguments: " + f"The argument must start with $0, not {argument}." + ), + ): + Script( + { + "%func1": f"{{[{argument}]}}", + } + ).resolve() + + @pytest.mark.parametrize( + "arguments", + [ + "$0, $2", + "$1, $2, $3", + ], + ) + def test_custom_function_invalid_function_argument_out_of_order(self, arguments: str): + with pytest.raises( + InvalidCustomFunctionArguments, + match=re.escape( + f"Custom function %func1 has invalid function arguments: " + f"{arguments} do not increment from $0 to ${len(arguments.split(',')) - 1}." + ), + ): + Script( + { + "%func1": f"{{[{arguments}]}}", + } + ).resolve() + + def test_custom_function_uses_custom_function_wrong_number_of_arguments(self): + with pytest.raises( + InvalidCustomFunctionArguments, + match=re.escape( + "Custom function %func0 has invalid usage of the custom function %func1: " + "Expects 1 argument but received 2" + ), + ): + Script( + { + "%func1": "{%mul(1, $0)}", + "%func0": "{%mul(%func1(1, 2), $0)}", + "output": "{%func0(1)}", + } + ).resolve() + + def test_custom_function_uses_custom_function_wrong_number_of_arguments_plural(self): + with pytest.raises( + InvalidCustomFunctionArguments, + match=re.escape( + "Custom function %func0 has invalid usage of the custom function %func1: " + "Expects 2 arguments but received 1" + ), + ): + Script( + { + "%func1": "{%mul($1, $0)}", + "%func0": "{%mul(%func1(1), $0)}", + "output": "{%func0(1)}", + } + ).resolve() + + def test_variable_uses_custom_function_wrong_number_of_arguments(self): + with pytest.raises( + InvalidCustomFunctionArguments, + match=re.escape( + "Variable output has invalid usage of the custom function %func1: " + "Expects 1 argument but received 2" + ), + ): + Script( + { + "%func1": "{%mul(1, $0)}", + "output": "{%mul(%func1(1, 2), 1)}", + } + ).resolve() + + def test_variable_uses_custom_function_wrong_number_of_arguments_plural(self): + with pytest.raises( + InvalidCustomFunctionArguments, + match=re.escape( + "Variable output has invalid usage of the custom function %func1: " + "Expects 2 arguments but received 1" + ), + ): + Script( + { + "%func1": "{%mul($1, $0)}", + "output": "{%mul(%func1(1), 1)}", + } + ).resolve() + + def test_function_argument_errors_has_spaces(self): + with pytest.raises( + InvalidCustomFunctionArgumentName, + match=re.escape( + "Custom function arguments, denoted by $, cannot have a space proceeding it." + ), + ): + Script( + { + "%func1": "{%mul($ 1, $0)}", + "output": "{%mul(%func1(1), 1)}", + } + ) diff --git a/tests/unit/script/types/test_float.py b/tests/unit/script/types/test_float.py new file mode 100644 index 00000000..e285ce17 --- /dev/null +++ b/tests/unit/script/types/test_float.py @@ -0,0 +1,81 @@ +import re + +import pytest + +from ytdl_sub.script.parser import NUMERICS_INVALID_CHAR +from ytdl_sub.script.parser import NUMERICS_ONLY_ARGS +from ytdl_sub.script.script import Script +from ytdl_sub.script.script_output import ScriptOutput +from ytdl_sub.script.types.resolvable import Float +from ytdl_sub.script.types.resolvable import String +from ytdl_sub.script.utils.exceptions import InvalidSyntaxException + + +class TestFloat: + @pytest.mark.parametrize( + "integer", + [ + "{1.0}", + "{ 1.2 }", + "{-1.4}", + "{ -1.5 }", + "{0001.2}", + "{ 0001.5 }", + ], + ) + def test_float_not_arg(self, integer: str): + with pytest.raises(InvalidSyntaxException, match=re.escape(str(NUMERICS_ONLY_ARGS))): + Script({"out": integer}).resolve() + + @pytest.mark.parametrize( + "float_, expected_float", + [ + ("{%float(1.1)}", 1.1), + ("{%float( 1.2345 )}", 1.2345), + ("{%float(-1.34)}", -1.34), + ("{%float( -1.535 )}", -1.535), + ("{%float(0001.)}", 1.0), + ("{%float( 0001. )}", 1.0), + ("{%float(.2)}", 0.2), + ("{%float(-.1)}", -0.1), + ], + ) + def test_float(self, float_: str, expected_float: int): + assert Script({"out": float_, "as_string": "{%string(out)}"}).resolve() == ScriptOutput( + { + "out": Float(expected_float), + "as_string": String(str(expected_float)), + } + ) + + @pytest.mark.parametrize( + "float_", + [ + "{%add(0, --1.0)}", + "{%add(0, 1-.0 )}", + "{%add(0,-1.0.)}", + "{%add(0, -1.0. )}", + "{%add(0,0001.a)}", + "{%add(0, 0001.- )}", + "{%add(0, ..3)}", + ], + ) + def test_invalid_float(self, float_: str): + with pytest.raises(InvalidSyntaxException, match=re.escape(str(NUMERICS_INVALID_CHAR))): + Script({"out": float_}).resolve() + + @pytest.mark.parametrize( + "to_cast, expected_float", + [ + ("{%float(5)}", 5.0), + ("{%float(0.9)}", 0.9), + ("{%float(-3.00)}", -3.0), + ("{%float(True)}", 1.0), + ("{%float(False)}", 0.0), + ("{%float('142.43')}", 142.43), + ], + ) + def test_cast_as_float(self, to_cast: str, expected_float: float): + assert Script({"as_float": to_cast}).resolve() == ScriptOutput( + {"as_float": Float(expected_float)} + ) diff --git a/tests/unit/script/types/test_function.py b/tests/unit/script/types/test_function.py new file mode 100644 index 00000000..1331d485 --- /dev/null +++ b/tests/unit/script/types/test_function.py @@ -0,0 +1,97 @@ +import re + +import pytest +from unit.script.conftest import single_variable_output + +from ytdl_sub.script.functions import Functions +from ytdl_sub.script.parser import FUNCTION_INVALID_CHAR +from ytdl_sub.script.script import Script +from ytdl_sub.script.types.resolvable import Integer +from ytdl_sub.script.utils.exceptions import FunctionDoesNotExist +from ytdl_sub.script.utils.exceptions import FunctionRuntimeException +from ytdl_sub.script.utils.exceptions import IncompatibleFunctionArguments +from ytdl_sub.script.utils.exceptions import InvalidSyntaxException + + +def _incompatible_arguments_match(expected: str, recieved: str) -> str: + return re.escape(f"Expected ({expected})\nReceived ({recieved})") + + +def mock_register_function(integer: Integer) -> Integer: + return Integer(integer.value + 100) + + +class TestFunction: + def test_nested_if_function_incompatible(self): + function_str = """{ + %map_get( + %if( + True, + %if( + True, + {}, + [] + ), + {} + ), + "key" + ) + }""" + with pytest.raises( + IncompatibleFunctionArguments, + match=_incompatible_arguments_match( + expected="Map, AnyArgument, Optional[AnyArgument]", + recieved="%if(...)->Union[Array, Map], String", + ), + ): + Script({"func": function_str}).resolve() + + @pytest.mark.parametrize( + "function_str, expected_types, received_types", + [ + ("{%array_at({'a': 'dict?'}, 1)}", "Array, Integer", "Map, Integer"), + ("{%array_extend('not', 'array')}", "Array, ...", "String, String"), + ( + "{%replace('hi mom', 'mom', 'dad', 1, 0)}", + "String, String, String, Optional[Integer]", + "String, String, String, Integer, Integer", + ), + ], + ) + def test_incompatible_types(self, function_str: str, expected_types: str, received_types: str): + with pytest.raises( + IncompatibleFunctionArguments, + match=_incompatible_arguments_match(expected=expected_types, recieved=received_types), + ): + Script({"func": function_str}).resolve() + + def test_runtime_error(self): + with pytest.raises( + FunctionRuntimeException, + match=re.escape( + "Runtime error occurred when executing the function %div: division by zero" + ), + ): + Script({"divide_by_zero": "{%div(8820, 0)}"}).resolve() + + def test_function_does_not_exist(self): + with pytest.raises( + FunctionDoesNotExist, + match=re.escape("Function %lolnope does not exist as a built-in or custom function."), + ): + Script({"dne": "{%lolnope(False, 'test this error message')}"}).resolve() + + def test_function_does_not_close(self): + with pytest.raises( + InvalidSyntaxException, + match=re.escape(str(FUNCTION_INVALID_CHAR)), + ): + Script({"dne": "{%throw}"}).resolve() + + def test_register_function(self): + try: + Functions.register_function(function=mock_register_function) + output = single_variable_output(f"{{%mock_register_function(10)}}") + assert output == 110 + finally: + del Functions._custom_functions[mock_register_function.__name__] diff --git a/tests/unit/script/types/test_integer.py b/tests/unit/script/types/test_integer.py new file mode 100644 index 00000000..18747e11 --- /dev/null +++ b/tests/unit/script/types/test_integer.py @@ -0,0 +1,81 @@ +import re + +import pytest + +from ytdl_sub.script.parser import NUMERICS_INVALID_CHAR +from ytdl_sub.script.parser import NUMERICS_ONLY_ARGS +from ytdl_sub.script.script import Script +from ytdl_sub.script.script_output import ScriptOutput +from ytdl_sub.script.types.resolvable import Boolean +from ytdl_sub.script.types.resolvable import Integer +from ytdl_sub.script.types.resolvable import String +from ytdl_sub.script.utils.exceptions import InvalidSyntaxException + + +class TestInteger: + @pytest.mark.parametrize( + "integer", + [ + "{1}", + "{ 1 }", + "{-1}", + "{ -1 }", + "{0001}", + "{ 0001 }", + ], + ) + def test_integer_not_arg(self, integer: str): + with pytest.raises(InvalidSyntaxException, match=re.escape(str(NUMERICS_ONLY_ARGS))): + Script({"integer": integer}).resolve() + + @pytest.mark.parametrize( + "integer, expected_integer", + [ + ("{%int(1)}", 1), + ("{%int( 1 )}", 1), + ("{%int(-1)}", -1), + ("{%int( -1 )}", -1), + ("{%int(0001)}", 1), + ("{%int( 0001 )}", 1), + ], + ) + def test_integer(self, integer: str, expected_integer: int): + assert Script( + {"integer": integer, "as_string": "{%string(integer)}"} + ).resolve() == ScriptOutput( + { + "integer": Integer(expected_integer), + "as_string": String(str(expected_integer)), + } + ) + + @pytest.mark.parametrize( + "integer", + [ + "{%add(0, --1)}", + "{%add(0, 1- )}", + "{%add(0,-1-)}", + "{%add(0, -1 - )}", + "{%add(0,0001a)}", + "{%add(0, 0001b )}", + ], + ) + def test_invalid_integer(self, integer: str): + with pytest.raises(InvalidSyntaxException, match=re.escape(str(NUMERICS_INVALID_CHAR))): + Script({"integer": integer}).resolve() + + @pytest.mark.parametrize( + "to_cast, expected_int", + [ + ("{%int(5)}", 5), + ("{%int(0.9)}", 0), + ("{%int(-3.00)}", -3), + ("{%int(True)}", 1), + ("{%int(False)}", 0), + ("{%int('142')}", 142), + ], + ) + def test_cast_as_integer(self, to_cast: str, expected_int: int): + assert Script({"as_int": to_cast}).resolve() == ScriptOutput( + {"as_int": Integer(expected_int)} + ) diff --git a/tests/unit/script/types/test_lambda_function.py b/tests/unit/script/types/test_lambda_function.py new file mode 100644 index 00000000..379c2207 --- /dev/null +++ b/tests/unit/script/types/test_lambda_function.py @@ -0,0 +1,127 @@ +import re + +import pytest + +from ytdl_sub.script.script import Script +from ytdl_sub.script.script_output import ScriptOutput +from ytdl_sub.script.types.array import Array +from ytdl_sub.script.types.resolvable import Integer +from ytdl_sub.script.utils.exceptions import IncompatibleFunctionArguments + + +class TestLambdaFunction: + def test_lambda_with_custom_function(self): + assert Script( + {"%times_two": "{%mul($0, 2)}", "wip": "{%array_apply([1, 2, 3], %times_two)}"} + ).resolve() == ScriptOutput({"wip": Array([Integer(2), Integer(4), Integer(6)])}) + + def test_conditional_lambda_with_custom_functions(self): + assert Script( + { + "%times_three": "{%mul($0, 3)}", + "%times_two": "{%mul($0, 2)}", + "wip": "{%array_apply([1, 2, 3], %if(False, %times_two, %times_three))}", + } + ).resolve() == ScriptOutput({"wip": Array([Integer(3), Integer(6), Integer(9)])}) + + def test_nested_custom_functions(self): + assert Script( + { + "%times_three": "{%mul($0, 3)}", + "%times_two": "{%mul($0, 2)}", + "identity": "{%times_three(%times_two(1))}", + } + ).resolve() == ScriptOutput({"identity": Integer(6)}) + + def test_nested_custom_functions_within_custom_functions(self): + assert Script( + { + "%power_2": "{%mul($0, 2)}", + "%power_3": "{%mul(%power_2($0), 2)}", + "%power_4": "{%mul(%power_3($0), 2)}", + "power_of_4": "{%power_4(2)}", + } + ).resolve() == ScriptOutput({"power_of_4": Integer(16)}) + + def test_nested_lambda_custom_functions_within_custom_functions(self): + assert Script( + { + "%nest4": "{%mul($0, 2)}", + "%nest3": "{%array_at(%array_apply([$0], %nest4), 0)}", + "%nest2": "{%array_at(%array_apply([$0], %nest3), 0)}", + "%nest1": "{%array_at(%array_apply([$0], %nest2), 0)}", + "output": "{%array_at(%array_apply([2], %nest1), 0)}", + } + ).resolve() == ScriptOutput({"output": Integer(4)}) + + +class TestLambdaFunctionIncompatibleNumArguments: + @pytest.mark.parametrize( + "lambda_value", ["%enumerate_output", "%if(False, %capitalize, %enumerate_output)"] + ) + def test_custom_function_lambda_in_variable(self, lambda_value: str): + with pytest.raises( + IncompatibleFunctionArguments, + match=re.escape( + "Variable output has invalid usage of the custom function " + "%enumerate_output as a lambda: Expects 2 arguments but will receive 1." + ), + ): + Script( + { + "%enumerate_output": "{[$0, $1]}", + "array1": "{['a', 'b', 'c']}", + "output": f"{{%array_apply(array1, {lambda_value})}}", + } + ) + + @pytest.mark.parametrize("lambda_value", ["%replace", "%if(False, %capitalize, %replace)"]) + def test_function_lambda_in_variable(self, lambda_value: str): + with pytest.raises( + IncompatibleFunctionArguments, + match=re.escape( + "Variable output has invalid usage of the function %replace as a lambda: " + "Expects 3 - 4 arguments but will receive 1." + ), + ): + Script( + { + "array1": "{['a', 'b', 'c']}", + "output": f"{{%array_apply(array1, {lambda_value})}}", + } + ) + + @pytest.mark.parametrize( + "lambda_value", ["%enumerate_output", "%if(False, %concat, %enumerate_output)"] + ) + def test_custom_function_lambda_in_custom_function(self, lambda_value: str): + with pytest.raises( + IncompatibleFunctionArguments, + match=re.escape( + "Custom function %output has invalid usage of the custom function " + "%enumerate_output as a lambda: Expects 3 arguments but will receive 2." + ), + ): + Script( + { + "%enumerate_output": "{[$0, $1, $2]}", + "array1": "{['a', 'b', 'c']}", + "%output": f"{{%array_enumerate(array1, {lambda_value})}}", + } + ) + + @pytest.mark.parametrize("lambda_value", ["%replace", "%if(False, %concat, %replace)"]) + def test_function_lambda_in_custom_function(self, lambda_value: str): + with pytest.raises( + IncompatibleFunctionArguments, + match=re.escape( + "Custom function %output has invalid usage of the function " + "%replace as a lambda: Expects 3 - 4 arguments but will receive 2." + ), + ): + Script( + { + "array1": "{['a', 'b', 'c']}", + "%output": f"{{%array_enumerate(array1, {lambda_value})}}", + } + ) diff --git a/tests/unit/script/types/test_map.py b/tests/unit/script/types/test_map.py new file mode 100644 index 00000000..3a6608cf --- /dev/null +++ b/tests/unit/script/types/test_map.py @@ -0,0 +1,200 @@ +import re + +import pytest + +from ytdl_sub.script.parser import _UNEXPECTED_COMMA_ARGUMENT +from ytdl_sub.script.parser import BRACKET_NOT_CLOSED +from ytdl_sub.script.parser import MAP_KEY_MULTIPLE_VALUES +from ytdl_sub.script.parser import MAP_KEY_NOT_HASHABLE +from ytdl_sub.script.parser import MAP_KEY_WITH_NO_VALUE +from ytdl_sub.script.parser import MAP_MISSING_KEY +from ytdl_sub.script.parser import ParsedArgType +from ytdl_sub.script.script import Script +from ytdl_sub.script.script_output import ScriptOutput +from ytdl_sub.script.types.map import Map +from ytdl_sub.script.types.resolvable import Float +from ytdl_sub.script.types.resolvable import String +from ytdl_sub.script.utils.exceptions import InvalidSyntaxException +from ytdl_sub.script.utils.exceptions import KeyNotHashableRuntimeException + + +class TestMap: + def test_return(self): + assert Script({"dict": "{{'a': 3.14}}"}).resolve() == ScriptOutput( + {"dict": Map({String("a"): Float(3.14)})} + ) + + def test_return_as_str(self): + assert Script({"dict": "json: {{'a': 3.14}}"}).resolve() == ScriptOutput( + {"dict": String('json: {"a": 3.14}')} + ) + + def test_nested_map(self): + map_str = """{ + { + 'level1': { + 'level2': { + 'level3': { + 'level4_key': 'level4_value' + }, + 'level3_key': 'level3_value' + }, + 'level2_key': 'level2_value' + }, + 'level1_key': 'level1_value' + } + }""" + + assert Script({"dict": map_str}).resolve() == ScriptOutput( + { + "dict": Map( + { + String("level1"): Map( + { + String("level2"): Map( + { + String("level3"): Map( + {String("level4_key"): String("level4_value")} + ), + String("level3_key"): String("level3_value"), + } + ), + String("level2_key"): String("level2_value"), + } + ), + String("level1_key"): String("level1_value"), + } + ) + } + ) + + @pytest.mark.parametrize( + "empty_map", + [ + "{{}}", + "{{ }}", + "{{ }}", + "{{\n}}", + ], + ) + def test_empty_map(self, empty_map: str): + assert Script({"dict": empty_map}).resolve() == ScriptOutput({"dict": Map({})}) + + @pytest.mark.parametrize( + "map", + [ + "{{}", + "{{ }", + "{{\n}", + "{{'key': 'value'}", + "{{ 'key' : 'value' }", + "{{ }", + ], + ) + def test_map_not_closed(self, map: str): + with pytest.raises(InvalidSyntaxException, match=re.escape(str(BRACKET_NOT_CLOSED))): + Script({"dict": map}).resolve() + + @pytest.mark.parametrize( + "value", + [ + "{{'key': }}", + "{{'key':}}", + "{{'key': 'value', 'key2':}}", + "{{'key1': 'value1','value2'}}", + "{{'key': 'value', 'key2': }}", + "{{ 'key': 'value', 'key2':\n}}", + "{{ 'key': ,\n}}", + ], + ) + def test_key_has_no_value(self, value: str): + with pytest.raises(InvalidSyntaxException, match=re.escape(str(MAP_KEY_WITH_NO_VALUE))): + Script({"dict": value}).resolve() + + @pytest.mark.parametrize( + "value", + [ + "{{,}}", + "{{ , }}", + "{{'key':'value',,}}", + "{{'key': 'value', ,}}", + ], + ) + def test_map_unexpected_comma(self, value: str): + with pytest.raises( + InvalidSyntaxException, + match=re.escape(str(_UNEXPECTED_COMMA_ARGUMENT(ParsedArgType.MAP_KEY))), + ): + Script({"dict": value}).resolve() + + @pytest.mark.parametrize( + "value", + [ + "{{'key1','key2'}}", + "{{'key1' , 'key2'}}", + "{{ 'key1', 'key2': 'value' }}", + ], + ) + def test_map_multiple_keys(self, value: str): + with pytest.raises(InvalidSyntaxException, match=re.escape(str(MAP_KEY_MULTIPLE_VALUES))): + Script({"dict": value}).resolve() + + @pytest.mark.parametrize( + "value", + [ + "{{'key1': 'value1',}}", + "{{'key1': 'value1' , }}", + ], + ) + def test_map_trailing_comma_okay(self, value: str): + assert Script({"dict": value}).resolve().get_native("dict") == {"key1": "value1"} + + @pytest.mark.parametrize( + "value", + [ + "{{:}}", + "{{ : }}", + "{{ : 'value' }}", + ], + ) + def test_map_missing_key(self, value: str): + with pytest.raises(InvalidSyntaxException, match=re.escape(str(MAP_MISSING_KEY))): + Script({"dict": value}).resolve() + + @pytest.mark.parametrize( + "value", + [ + "{{{}:'value'}}", + "{{ {} : 'value' }}", + "{{[]:'value'}}", + "{{ [] : 'value' }}", + ], + ) + def test_map_key_not_hashable(self, value: str): + with pytest.raises(InvalidSyntaxException, match=re.escape(str(MAP_KEY_NOT_HASHABLE))): + Script({"dict": value}).resolve() + + def test_map_key_is_hashable_variable(self): + assert Script( + { + "dict": "{{key_variable : 'value' }}", + "key_variable": "hashable", + } + ).resolve() == ScriptOutput( + { + "key_variable": String("hashable"), + "dict": Map({String("hashable"): String("value")}), + } + ) + + def test_map_key_is_non_hashable_variable(self): + with pytest.raises( + KeyNotHashableRuntimeException, + match=re.escape("Tried to use Array as a Map key, but it is not hashable."), + ): + Script( + { + "dict": "{{key_variable : 'value' }}", + "key_variable": "{['non-hashable']}", + } + ).resolve() diff --git a/tests/unit/script/types/test_string.py b/tests/unit/script/types/test_string.py new file mode 100644 index 00000000..c3749845 --- /dev/null +++ b/tests/unit/script/types/test_string.py @@ -0,0 +1,67 @@ +import re + +import pytest + +from ytdl_sub.script.parser import STRINGS_NOT_CLOSED +from ytdl_sub.script.parser import STRINGS_ONLY_ARGS +from ytdl_sub.script.script import Script +from ytdl_sub.script.script_output import ScriptOutput +from ytdl_sub.script.types.resolvable import String +from ytdl_sub.script.utils.exceptions import InvalidSyntaxException + + +class TestString: + @pytest.mark.parametrize( + "string", + [ + "{'323'}", + '{ "4253" }', + '{"hi"}', + '{ "asfsd" }', + '{"sdfasf"}', + "{ '3fsdf' }", + ], + ) + def test_string_not_arg(self, string: str): + with pytest.raises(InvalidSyntaxException, match=re.escape(str(STRINGS_ONLY_ARGS))): + Script({"out": string}).resolve() + + @pytest.mark.parametrize( + "string, expected_string", + [ + ("{%string('')}", ""), + ('{%string("")}', ""), + ('{%string("\\")}', "\\"), + ("{%string('323')}", "323"), + ('{%string( "4253" )}', "4253"), + ('{%string("hi")}', "hi"), + ('{%string( "asfsd" )}', "asfsd"), + ('{%string("sdfasf")}', "sdfasf"), + ("{%string( '3fsdf' )}", "3fsdf"), + ("{%string('newlines \n newlines')}", "newlines \n newlines"), + ("{%string('in function')} out of function", "in function out of function"), + ("{%string('supports \" in string')}", 'supports " in string'), + ('{%string("supports \' in string")}', "supports ' in string"), + ("{%string('backslash \\\\')}", "backslash \\\\"), + ("{%string('''triple quote with \" ' \\''')}", "triple quote with \" ' \\"), + ('{%string("""triple quote with " \' \\""")}', "triple quote with \" ' \\"), + ], + ) + def test_string(self, string: str, expected_string: str): + assert Script({"out": string}).resolve() == ScriptOutput({"out": String(expected_string)}) + + def test_null_is_empty_string(self): + assert Script({"out": "{%string(null)}"}).resolve() == ScriptOutput({"out": String("")}) + + @pytest.mark.parametrize( + "string", + [ + "{%string('open only single)}", + '{%string( "open only double )}', + "{%string(\"open double close single ')}", + "{%string( 'open single close double\" )}", + ], + ) + def test_string_not_closed_properly(self, string: str): + with pytest.raises(InvalidSyntaxException, match=re.escape(str(STRINGS_NOT_CLOSED))): + Script({"out": string}).resolve() diff --git a/tests/unit/script/types/test_variable.py b/tests/unit/script/types/test_variable.py new file mode 100644 index 00000000..526e1476 --- /dev/null +++ b/tests/unit/script/types/test_variable.py @@ -0,0 +1,130 @@ +import re + +import pytest + +from ytdl_sub.script.script import Script +from ytdl_sub.script.script_output import ScriptOutput +from ytdl_sub.script.types.resolvable import String +from ytdl_sub.script.utils.exceptions import CycleDetected +from ytdl_sub.script.utils.exceptions import InvalidVariableName +from ytdl_sub.script.utils.exceptions import VariableDoesNotExist + + +class TestVariable: + def test_simple(self): + assert Script({"a": "a", "b": "{b_}", "b_": "b"}).resolve() == ScriptOutput( + { + "a": String("a"), + "b": String("b"), + "b_": String("b"), + } + ) + + def test_multiple_variables(self): + assert Script({"a": "a", "b": "b", "b_": " {a} {b} "}).resolve() == ScriptOutput( + { + "a": String("a"), + "b": String("b"), + "b_": String(" a b "), + } + ) + + def test_simple_with_function(self): + assert Script({"a": "a", "b": "{%capitalize(b_)}", "b_": "b"}).resolve() == ScriptOutput( + { + "a": String("a"), + "b": String("B"), + "b_": String("b"), + } + ) + + def test_simple_cycle(self): + with pytest.raises( + CycleDetected, + match=re.escape("Cycle detected within these variables: a -> b -> a"), + ): + Script({"a": "{b}", "b": "{a}"}).resolve() + + def test_simple_cycle_with_function(self): + with pytest.raises( + CycleDetected, + match=re.escape("Cycle detected within these variables: b -> b_ -> b"), + ): + Script({"b": "{%capitalize(b_)}", "b_": "{b}"}).resolve() + + def test_undefined_variable(self): + with pytest.raises( + VariableDoesNotExist, + match=re.escape("Variable c does not exist."), + ): + Script({"a": "a", "b": "{c}"}).resolve() + + @pytest.mark.parametrize( + "name", + [ + "vali_LOL_INVALID", + "name!!", + "na(", + "na[", + ], + ) + def test_invalid_variable_name_inline(self, name: str): + with pytest.raises( + InvalidVariableName, + match=re.escape( + f"Variable name '{name}' is invalid:" + " Names must be lower_snake_cased and begin with a letter." + ), + ): + Script({"a": f"{{{name}}}"}).resolve() + + @pytest.mark.parametrize( + "name", + ["float", "bool", "mul"], + ) + def test_invalid_variable_name_inline_is_built_in(self, name: str): + with pytest.raises( + InvalidVariableName, + match=re.escape( + f"Variable name '{name}' is invalid:" + " The name is used by a built-in function and cannot be overwritten." + ), + ): + Script({"a": f"{{{name}}}"}).resolve() + + @pytest.mark.parametrize( + "name", + [ + "vali_LOL_INVALID", + "name!!", + "na(", + "na[", + "$2232", + "1245", + "CAN_CATCH_MORE", + "{brackets_in_definition}", + ], + ) + def test_invalid_variable_name_definition(self, name: str): + with pytest.raises( + InvalidVariableName, + match=re.escape( + f"Variable name '{name}' is invalid:" + " Names must be lower_snake_cased and begin with a letter." + ), + ): + Script({f"{name}": "value"}).resolve() + + @pytest.mark.parametrize( + "name", + ["float", "bool", "mul"], + ) + def test_invalid_variable_name_definition_is_built_in(self, name: str): + with pytest.raises( + InvalidVariableName, + match=re.escape( + f"Variable name '{name}' is invalid:" + " The name is used by a built-in function and cannot be overwritten." + ), + ): + Script({f"{name}": "value"}).resolve() diff --git a/tests/unit/utils/test_script_utils.py b/tests/unit/utils/test_script_utils.py new file mode 100644 index 00000000..8b2b0e90 --- /dev/null +++ b/tests/unit/utils/test_script_utils.py @@ -0,0 +1,33 @@ +import copy + +from unit.script.conftest import single_variable_output + +from ytdl_sub.utils.script import ScriptUtils + + +class TestScriptUtils: + def test_dict_to_script(self): + json_dict = { + "string": "value", + "quotes": "has '' and \"\"", + "triple-single-quote": "right here! '''''''''''''''''''''''''''''' ack '''''''", + "int": 1, + "bool": True, + "list": [1, 2, 3], + "dict": {"a": 1, "b": 2}, + "float": 3.14, + "nested_dict": { + "string": "value", + "int": 1, + "bool": True, + "list": [1, 2, 3], + "dict": {"a": 1, "b": 2}, + "float": 3.14, + }, + } + + expected_output = copy.deepcopy(json_dict) + expected_output["triple-single-quote"] = "right here! ' ack '" + + output = single_variable_output(ScriptUtils.to_script(json_dict)) + assert output == expected_output diff --git a/tests/unit/validators/test_file_path_validators.py b/tests/unit/validators/test_file_path_validators.py index 7cf7b9f2..7ff62143 100644 --- a/tests/unit/validators/test_file_path_validators.py +++ b/tests/unit/validators/test_file_path_validators.py @@ -5,11 +5,13 @@ import pytest from ytdl_sub.config.config_file import ConfigFile from ytdl_sub.config.defaults import MAX_FILE_NAME_BYTES +from ytdl_sub.script.script import Script +from ytdl_sub.utils.file_path import FilePathTruncater from ytdl_sub.utils.subtitles import SUBTITLE_EXTENSIONS -from ytdl_sub.validators.file_path_validators import FilePathValidatorMixin from ytdl_sub.validators.file_path_validators import StringFormatterFileNameValidator +@pytest.mark.usefixtures("register_custom_functions") class TestStringFormatterFilePathValidator: @pytest.mark.parametrize( "ext", @@ -28,7 +30,9 @@ class TestStringFormatterFilePathValidator: file_path = str(Path(temp_dir) / file_name) formatter = StringFormatterFileNameValidator(name="test", value=str(file_path)) - truncated_file_path = formatter.apply_formatter({}) + truncated_file_path = formatter.post_process( + Script({"file_name": formatter.format_string}).resolve().get_str("file_name") + ) assert truncated_file_path.count(".") == ext.count(".") assert str(Path(temp_dir)) in truncated_file_path @@ -56,7 +60,9 @@ class TestStringFormatterFilePathValidator: file_path = str(Path(temp_dir) / f"{base_file_name}{ext}") formatter = StringFormatterFileNameValidator(name="test", value=str(file_path)) - truncated_file_path = formatter.apply_formatter({}) + truncated_file_path = formatter.post_process( + Script({"file_name": formatter.format_string}).resolve().get_str("file_name") + ) assert truncated_file_path == str( Path(temp_dir) @@ -74,16 +80,16 @@ class TestStringFormatterFilePathValidator: @pytest.mark.parametrize( "file_name_max_bytes, expected_max", [ - (50, 50 - FilePathValidatorMixin._EXTENSION_BYTES), + (50, 50 - FilePathTruncater._EXTENSION_BYTES), (0, 16), - (10000, MAX_FILE_NAME_BYTES - FilePathValidatorMixin._EXTENSION_BYTES), + (10000, MAX_FILE_NAME_BYTES - FilePathTruncater._EXTENSION_BYTES), ], ) def test_config_changes_max_file_name_bytes(self, file_name_max_bytes: int, expected_max: int): # Ensure the default is set assert ( - FilePathValidatorMixin._MAX_BASE_FILE_NAME_BYTES - == FilePathValidatorMixin._DEFAULT_MAX_BASE_FILE_NAME_BYTES + FilePathTruncater._MAX_BASE_FILE_NAME_BYTES + == FilePathTruncater._DEFAULT_MAX_BASE_FILE_NAME_BYTES ) try: @@ -98,8 +104,8 @@ class TestStringFormatterFilePathValidator: } ) - assert FilePathValidatorMixin._MAX_BASE_FILE_NAME_BYTES == expected_max + assert FilePathTruncater._MAX_BASE_FILE_NAME_BYTES == expected_max finally: - FilePathValidatorMixin._MAX_BASE_FILE_NAME_BYTES = ( - FilePathValidatorMixin._DEFAULT_MAX_BASE_FILE_NAME_BYTES + FilePathTruncater._MAX_BASE_FILE_NAME_BYTES = ( + FilePathTruncater._DEFAULT_MAX_BASE_FILE_NAME_BYTES ) diff --git a/tests/unit/validators/test_string_formatter_validator.py b/tests/unit/validators/test_string_formatter_validator.py index fbafda4d..0928fc24 100644 --- a/tests/unit/validators/test_string_formatter_validator.py +++ b/tests/unit/validators/test_string_formatter_validator.py @@ -9,22 +9,6 @@ from ytdl_sub.validators.string_formatter_validators import OverridesStringForma from ytdl_sub.validators.string_formatter_validators import StringFormatterValidator -@pytest.fixture -def error_message_unequal_brackets_str(): - return ( - "Brackets are reserved for {variable_names} and " - "should contain a single open and close bracket." - ) - - -@pytest.fixture -def error_message_unequal_regex_matches_str(): - return ( - "{variable_names} must start with a lowercase letter, should only contain lowercase " - "letters, numbers, underscores, and have a single open and close bracket." - ) - - @pytest.mark.parametrize( "string_formatter_class", [StringFormatterValidator, OverridesStringFormatterValidator] ) @@ -34,16 +18,6 @@ class TestStringFormatterValidator(object): validator = string_formatter_class(name="test_format_variables", value=format_string) assert validator.format_string == format_string - assert validator.format_variables == ["var_one1", "var_two"] - - def test_format_variables(self, string_formatter_class): - format_string = "No vars 💩" - assert ( - string_formatter_class( - name="test_format_variables_empty", value=format_string - ).format_variables - == [] - ) @pytest.mark.parametrize( "format_string", @@ -54,12 +28,8 @@ class TestStringFormatterValidator(object): "Try }var_one} and {var_one}", ], ) - def test_validate_fail_uneven_brackets( - self, string_formatter_class, format_string, error_message_unequal_brackets_str - ): - expected_error_msg = f"Validation error in fail: {error_message_unequal_brackets_str}" - - with pytest.raises(ValidationException, match=expected_error_msg): + def test_validate_fail_uneven_brackets(self, string_formatter_class, format_string): + with pytest.raises(ValidationException, match="Validation error in fail:"): _ = string_formatter_class(name="fail", value=format_string) @pytest.mark.parametrize( @@ -74,124 +44,10 @@ class TestStringFormatterValidator(object): "Try {} empty", ], ) - def test_validate_fail_bad_variable( - self, string_formatter_class, format_string, error_message_unequal_regex_matches_str - ): - expected_error_msg = f"Validation error in fail: {error_message_unequal_regex_matches_str}" - - with pytest.raises(ValidationException, match=expected_error_msg): + def test_validate_fail_bad_variable(self, string_formatter_class, format_string): + with pytest.raises(ValidationException, match="Validation error in fail:"): _ = string_formatter_class(name="fail", value=format_string) - @pytest.mark.parametrize( - "format_string, bad_variable", - [ - ("keyword {while}", "while"), - ("{try} {valid_var}", "try"), - ], - ) - def test_validate_fail_variable_keyword_or_not_identifier( - self, string_formatter_class, format_string, bad_variable - ): - expected_error_msg = ( - f"Validation error in fail: " - f"'{bad_variable}' is a Python keyword and cannot be used as a variable." - ) - - with pytest.raises(ValidationException, match=expected_error_msg): - _ = string_formatter_class(name="fail", value=format_string) - - def test_entry_formatter_fails_missing_field(self, string_formatter_class): - format_string = string_formatter_class(name="test", value=f"prefix {{bah_humbug}} suffix") - variable_dict = {"varb": "a", "vara": "b"} - expected_error_msg = ( - f"Validation error in test: Format variable 'bah_humbug' does not exist. " - f"Available variables: {', '.join(sorted(variable_dict.keys()))}" - ) - if string_formatter_class == OverridesStringFormatterValidator: - expected_error_msg = ( - f"Validation error in test: Override variable 'bah_humbug' does not exist. " - f"For this field, ensure your override variable does not contain any source " - f"variables - it is a requirement that this be a static string. " - f"Available override variables: {', '.join(sorted(variable_dict.keys()))}" - ) - - with pytest.raises(StringFormattingException, match=expected_error_msg): - assert format_string.apply_formatter(variable_dict=variable_dict) - - def test_string_formatter_single_field(self, string_formatter_class): - uid = "this uid" - format_string = string_formatter_class(name="test", value=f"prefix {{uid}} suffix") - expected_string = f"prefix {uid} suffix" - - assert format_string.apply_formatter(variable_dict={"uid": uid}) == expected_string - - def test_entry_formatter_duplicate_fields(self, string_formatter_class): - upload_year = "2022" - format_string = string_formatter_class( - name="test", value=f"prefix {{upload_year}} {{upload_year}} suffix" - ) - expected_string = f"prefix {upload_year} {upload_year} suffix" - - assert ( - format_string.apply_formatter(variable_dict={"upload_year": upload_year}) - == expected_string - ) - - def test_entry_formatter_override_recursive(self, string_formatter_class): - variable_dict = { - "level_a": "level a", - "level_b": "level b and {level_a}", - "level_c": "level c and {level_b}", - } - - format_string = string_formatter_class(name="test", value="level d and {level_c}") - expected_string = "level d and level c and level b and level a" - - assert format_string.apply_formatter(variable_dict=variable_dict) == expected_string - - def test_entry_formatter_override_sanitized_recursive(self, string_formatter_class): - variable_dict = { - "level_a": "level a", - "level_b": "level b ? {level_a}", - "level_c": "level c and {level_b}", - } - - format_string = string_formatter_class(name="test", value="level d and {level_c_sanitized}") - expected_string = "level d and " + sanitize_filename("level c and level b ? level a") - - assert format_string.apply_formatter(variable_dict=variable_dict) == expected_string - - def test_entry_formatter_override_sanitized_recursive_inner(self, string_formatter_class): - variable_dict = { - "level_a": "level a ?", - "level_b": "level b ? {level_a_sanitized}", - "level_c": "level c and {level_b_sanitized}", - } - - format_string = string_formatter_class(name="test", value="level d and {level_c}") - expected_string = "level d and level c and " + sanitize_filename("level b ? level a ?") - - assert format_string.apply_formatter(variable_dict=variable_dict) == expected_string - - def test_entry_formatter_override_recursive_fail_cycle(self, string_formatter_class): - variable_dict = { - "level_a": "{level_b}", - "level_b": "{level_a}", - } - - # Max depth is 3 so should go level_a -(0)-> level_b -(1)-> level_a -(2)-> level_b - expected_error_msg = ( - "Validation error in test: Attempted to format but failed after reaching max recursion " - "depth of 3. Try to keep variables dependent on only one other variable at max. " - "Unresolved variables: level_b" - ) - - format_string = string_formatter_class(name="test", value="{level_a}") - format_string._max_format_recursion = 3 - - with pytest.raises(StringFormattingException, match=expected_error_msg): - _ = format_string.apply_formatter(variable_dict=variable_dict) - class TestDictFormatterValidator(object): @pytest.mark.parametrize( @@ -215,9 +71,6 @@ class TestDictFormatterValidator(object): assert validator.dict["key1"].format_string == key1_format_string assert validator.dict["key2"].format_string == key2_format_string - assert validator.dict["key1"].format_variables == ["variable"] - assert validator.dict["key2"].format_variables == [] - assert validator.dict_with_format_strings == { "key1": key1_format_string, "key2": key2_format_string, diff --git a/tools/linter b/tools/linter index 28394065..b3a4fb7f 100755 --- a/tools/linter +++ b/tools/linter @@ -3,11 +3,9 @@ if [[ $1 = "check" ]]; then isort . --check-only --diff \ && black . --check \ - && pylint src/ \ - && pydocstyle src/* + && pylint src/ else isort . black . pylint src/ - pydocstyle src/* fi \ No newline at end of file