diff --git a/docs/config.rst b/docs/config.rst index 08db9eb0..521628e6 100644 --- a/docs/config.rst +++ b/docs/config.rst @@ -192,6 +192,17 @@ nfo_output_directory ------------------------------------------------------------------------------- +regex +''''' +.. autoclass:: ytdl_sub.plugins.regex.RegexOptions() + :members: skip_if_match_fails + +.. autoclass:: ytdl_sub.plugins.regex.SourceVariableRegex() + :members: match, capture_group_names, capture_group_defaults + :member-order: bysource + +------------------------------------------------------------------------------- + .. _subscription_yaml: subscription.yaml diff --git a/src/ytdl_sub/config/preset.py b/src/ytdl_sub/config/preset.py index 3e4e879e..b95b589a 100644 --- a/src/ytdl_sub/config/preset.py +++ b/src/ytdl_sub/config/preset.py @@ -83,6 +83,10 @@ class Preset(StrictDictValidator): # and ensure required keys are present. _optional_keys = PRESET_KEYS + @property + def _source_variables(self) -> List[str]: + return self.downloader.downloader_entry_type.source_variables() + def __validate_and_get_downloader(self, downloader_source: str) -> Type[Downloader]: return self._validate_key(key=downloader_source, validator=DownloadStrategyValidator).get( downloader_source=downloader_source @@ -141,6 +145,9 @@ class Preset(StrictDictValidator): plugin = PluginMapping.get(plugin=key) plugin_options = self._validate_key(key=key, validator=plugin.plugin_options_type) + plugin_options.validate_with_variables( + source_variables=self._source_variables, override_variables=self.overrides.keys + ) plugins.append((plugin, plugin_options)) @@ -156,12 +163,20 @@ class Preset(StrictDictValidator): # If the formatter supports source variables, set the formatter variables to include # both source and override variables if not isinstance(formatter_validator, OverridesStringFormatterValidator): - source_variables = { - source_var: "dummy_string" - for source_var in self.downloader.downloader_entry_type.source_variables() - } + source_variables = {source_var: "dummy_string" for source_var in self._source_variables} variable_dict = dict(source_variables, **variable_dict) + # For all plugins, add in any extra added source variables + for _, plugin_options in self.plugins: + added_plugin_variables = { + source_var: "dummy_string" for source_var in plugin_options.added_source_variables() + } + # sanity check plugin variables do not override source variables + expected_len = len(variable_dict) + len(added_plugin_variables) + variable_dict = dict(variable_dict, **added_plugin_variables) + + assert len(variable_dict) == expected_len, "plugin variables overwrote source variables" + _ = formatter_validator.apply_formatter(variable_dict=variable_dict) def __recursive_preset_validate( diff --git a/src/ytdl_sub/config/preset_class_mappings.py b/src/ytdl_sub/config/preset_class_mappings.py index c1eec9fa..e6d7bf2d 100644 --- a/src/ytdl_sub/config/preset_class_mappings.py +++ b/src/ytdl_sub/config/preset_class_mappings.py @@ -13,6 +13,7 @@ from ytdl_sub.plugins.music_tags import MusicTagsPlugin from ytdl_sub.plugins.nfo_tags import NfoTagsPlugin from ytdl_sub.plugins.output_directory_nfo_tags import OutputDirectoryNfoTagsPlugin from ytdl_sub.plugins.plugin import Plugin +from ytdl_sub.plugins.regex import RegexPlugin class DownloadStrategyMapping: @@ -107,6 +108,7 @@ class PluginMapping: "music_tags": MusicTagsPlugin, "nfo_tags": NfoTagsPlugin, "output_directory_nfo_tags": OutputDirectoryNfoTagsPlugin, + "regex": RegexPlugin, } @classmethod diff --git a/src/ytdl_sub/entries/base_entry.py b/src/ytdl_sub/entries/base_entry.py index 88d4e19c..aa8906b1 100644 --- a/src/ytdl_sub/entries/base_entry.py +++ b/src/ytdl_sub/entries/base_entry.py @@ -22,6 +22,8 @@ class BaseEntry(ABC): self._working_directory = working_directory self._kwargs = entry_dict + self._additional_variables: Dict[str, str] = {} + def kwargs_contains(self, key: str) -> bool: """Returns whether internal kwargs contains the specified key""" return key in self._kwargs @@ -39,3 +41,28 @@ class BaseEntry(ABC): The working directory """ return self._working_directory + + def add_variables(self, variables_to_add: Dict[str, str]) -> "BaseEntry": + """ + Parameters + ---------- + variables_to_add + Variables to add to this entry + + Returns + ------- + self + + Raises + ------ + ValueError + If a variable trying to be added already exists as a source variable + """ + for variable_name in variables_to_add.keys(): + if self.kwargs_contains(variable_name): + raise ValueError( + f"Cannot add variable '{variable_name}': already exists in the kwargs" + ) + + self._additional_variables = dict(self._additional_variables, **variables_to_add) + return self diff --git a/src/ytdl_sub/entries/variables/entry_variables.py b/src/ytdl_sub/entries/variables/entry_variables.py index 46fa3766..4e75d443 100644 --- a/src/ytdl_sub/entries/variables/entry_variables.py +++ b/src/ytdl_sub/entries/variables/entry_variables.py @@ -37,6 +37,14 @@ class SourceVariables: """ return self.kwargs("extractor") + def _added_variables(self: BaseEntry) -> Dict[str, str]: + """ + Returns + ------- + Dict of variables added to this entry + """ + return self._additional_variables + @classmethod def source_variables(cls) -> List[str]: """ @@ -54,7 +62,10 @@ class SourceVariables: ------- Dictionary containing all variables """ - return {source_var: getattr(self, source_var) for source_var in self.source_variables()} + source_variable_dict = { + source_var: getattr(self, source_var) for source_var in self.source_variables() + } + return dict(source_variable_dict, **self._added_variables()) class EntryVariables(SourceVariables): diff --git a/src/ytdl_sub/plugins/plugin.py b/src/ytdl_sub/plugins/plugin.py index 16767b00..6283f758 100644 --- a/src/ytdl_sub/plugins/plugin.py +++ b/src/ytdl_sub/plugins/plugin.py @@ -1,5 +1,6 @@ from abc import ABC from typing import Generic +from typing import List from typing import Optional from typing import Type from typing import TypeVar @@ -19,6 +20,35 @@ class PluginOptions(StrictDictValidator): Class that defines the parameters to a plugin """ + # pylint: disable=no-self-use + def added_source_variables(self) -> List[str]: + """ + If the plugin adds source variables, list them here. + + Returns + ------- + List of added source variables this plugin creates + """ + return [] + + # pylint: disable=unused-argument + def validate_with_variables( + self, source_variables: List[str], override_variables: List[str] + ) -> None: + """ + Optional validation after init with the session's source and override variables. + + Parameters + ---------- + source_variables + Available source variables when running the plugin + override_variables + Available override variables when running the plugin + """ + return None + + # pylint: enable=no-self-use,unused-argument + PluginOptionsT = TypeVar("PluginOptionsT", bound=PluginOptions) @@ -43,6 +73,7 @@ class Plugin(DownloadArchiver, Generic[PluginOptionsT], ABC): # TODO pass yaml snake case name in the class somewhere, and use it for the logger self._logger = Logger.get(self.__class__.__name__) + # pylint: disable=no-self-use def modify_entry(self, entry: Entry) -> Optional[Entry]: """ For each entry downloaded, modify the entry in some way before sending it to @@ -57,6 +88,9 @@ class Plugin(DownloadArchiver, Generic[PluginOptionsT], ABC): ------- The entry or None, indicating not to move it to the output directory """ + return entry + + # pylint: enable=no-self-use def post_process_entry(self, entry: Entry) -> Optional[FileMetadata]: """ diff --git a/src/ytdl_sub/plugins/regex.py b/src/ytdl_sub/plugins/regex.py new file mode 100644 index 00000000..d584e058 --- /dev/null +++ b/src/ytdl_sub/plugins/regex.py @@ -0,0 +1,311 @@ +from typing import Dict +from typing import List +from typing import Optional + +from yt_dlp.utils import sanitize_filename + +from ytdl_sub.entries.entry import Entry +from ytdl_sub.plugins.plugin import Plugin +from ytdl_sub.plugins.plugin import PluginOptions +from ytdl_sub.utils.exceptions import RegexNoMatchException +from ytdl_sub.utils.logger import Logger +from ytdl_sub.validators.regex_validator import RegexListValidator +from ytdl_sub.validators.source_variable_validator import SourceVariableNameListValidator +from ytdl_sub.validators.strict_dict_validator import StrictDictValidator +from ytdl_sub.validators.string_formatter_validators import ListFormatterValidator +from ytdl_sub.validators.string_formatter_validators import StringFormatterValidator +from ytdl_sub.validators.validators import BoolValidator + +logger = Logger.get(name="regex") + + +class SourceVariableRegex(StrictDictValidator): + + _required_keys = {"match"} + _optional_keys = {"capture_group_defaults", "capture_group_names"} + + def __init__(self, name, value): + super().__init__(name, value) + self._match = self._validate_key(key="match", validator=RegexListValidator) + self._capture_group_defaults = self._validate_key_if_present( + key="capture_group_defaults", validator=ListFormatterValidator + ) + self._capture_group_names = self._validate_key_if_present( + key="capture_group_names", validator=SourceVariableNameListValidator, default=[] + ) + + # If defaults are to be used, ensure there are the same number of defaults as there are + # capture groups + if self._capture_group_defaults is not None and self._match.num_capture_groups != len( + self._capture_group_defaults.list + ): + raise self._validation_exception( + f"number of defaults must match number of capture groups, " + f"{len(self._capture_group_defaults.list)} != {self._match.num_capture_groups}" + ) + + # If there are capture groups, ensure there are capture group names + if len(self._capture_group_names.list) != self._match.num_capture_groups: + raise self._validation_exception( + f"number of capture group names must match number of capture groups, " + f"{len(self._capture_group_names.list)} != {self._match.num_capture_groups}" + ) + + @property + def match(self) -> RegexListValidator: + """ + Required. List of regex strings to try to match against a source variable. Each regex + string must have the same number of capture groups. + """ + return self._match + + @property + def capture_group_names(self) -> Optional[List[str]]: + """ + Optional (only when no capture groups are in the regex string). List of names to store the + capture group values to. These and ``_sanitized`` versions will be available to use as + source variables. The list's length must be equal to the number of match capture groups. + """ + return [validator.value for validator in self._capture_group_names.list] + + @property + def capture_group_defaults(self) -> Optional[List[StringFormatterValidator]]: + """ + Optional. List of string format validators to use for capture group defaults if a + source variable cannot be matched. The list's length must be equal to the number of match + capture groups. + """ + return self._capture_group_defaults.list if self.has_defaults else None + + @property + def has_defaults(self) -> bool: + """ + Returns + ------- + True if a validation exception should be raised if not captured. False otherwise. + """ + return self._capture_group_defaults is not None + + +class FromSourceVariablesRegex(StrictDictValidator): + + _optional_keys = Entry.source_variables() + _allow_extra_keys = True + + def __init__(self, name, value): + super().__init__(name, value) + self.source_variable_capture_dict: Dict[str, SourceVariableRegex] = { + key: self._validate_key(key=key, validator=SourceVariableRegex) for key in self._keys + } + + +class RegexOptions(PluginOptions): + r""" + Performs regex matching on an entry's source variables. Regex can be used to filter entries + from proceeding with download or capture groups to create new source variables. + + Usage: + + .. code-block:: yaml + + presets: + my_example_preset: + regex: + # By default, if any match fails and has no defaults, the entry will be skipped. + # If set to False, ytdl-sub will error and stop all downloads from proceeding. + skip_if_match_fails: True + + from: + # For each entry's `title` value... + title: + # Perform this regex match on it to act as a filter. + # This will only download videos with "Official Video" in it. + match: '\[Official Video\]' + + # For each entry's `description` value... + description: + # Match with capture groups and defaults. + # This tries to scrape a date from the description and produce new source variables + match: "([0-9]{4})-([0-9]{2})-([0-9]{2})" + + # Each capture group creates these new source variables, respectively, as well + # a sanitized version, i.e. `captured_upload_year_sanitized` + capture_group_names: + - "captured_upload_year" + - "captured_upload_month" + - "captured_upload_day" + + # And if the string does not match, use these as respective default values for the + # new source variables. + capture_group_defaults: + - "{upload_year}" + - "{upload_month}" + - "{upload_day}" + """ + + _required_keys = {"from"} + _optional_keys = {"skip_if_match_fails"} + + def __init__(self, name, value): + super().__init__(name, value) + self._from = self._validate_key(key="from", validator=FromSourceVariablesRegex) + self._skip_if_match_fails: bool = self._validate_key_if_present( + key="skip_if_match_fails", validator=BoolValidator, default=True + ).value + + @property + def skip_if_match_fails(self) -> Optional[bool]: + """ + Defaults to True. If True, when any match fails and has no defaults, the entry will be + skipped. If False, ytdl-sub will error and all downloads will not proceed. + """ + return self._skip_if_match_fails + + def validate_with_variables( + self, source_variables: List[str], override_variables: List[str] + ) -> None: + """ + Ensures each source variable capture group is valid + + Parameters + ---------- + source_variables + Available source variables when running the plugin + override_variables + Available override variables when running the plugin + """ + for key, regex_options in self.source_variable_capture_dict.items(): + # Ensure each variable getting captured is a source variable + if key not in source_variables: + raise self._validation_exception( + f"cannot regex capture '{key}' because it is not a source variable" + ) + + # Ensure the capture group names are not existing source/override variables + for capture_group_name in regex_options.capture_group_names: + if capture_group_name in source_variables: + raise self._validation_exception( + f"'{capture_group_name}' cannot be used as a capture group name because it " + f"is a source variable" + ) + if capture_group_name in override_variables: + raise self._validation_exception( + f"'{capture_group_name}' cannot be used as a capture group name because it " + f"is an override variable" + ) + + @property + def source_variable_capture_dict(self) -> Dict[str, SourceVariableRegex]: + """ + Returns + ------- + Dict of { source variable: capture options } + """ + return self._from.source_variable_capture_dict + + def added_source_variables(self) -> List[str]: + """ + Returns + ------- + List of new source variables created via regex capture + """ + added_source_vars: List[str] = [] + for regex_options in self.source_variable_capture_dict.values(): + added_source_vars.extend(regex_options.capture_group_names) + added_source_vars.extend( + f"{capture_group_name}_sanitized" + for capture_group_name in regex_options.capture_group_names + ) + + return added_source_vars + + +class RegexPlugin(Plugin[RegexOptions]): + plugin_options_type = RegexOptions + + def modify_entry(self, entry: Entry) -> Optional[Entry]: + """ + Parameters + ---------- + entry + Entry to add source variables to + + Returns + ------- + Entry with regex capture variables added to its source variables + + Raises + ------ + ValidationException + If no capture and no defaults + """ + entry_variable_dict = entry.to_dict() + + # Iterate each source var to capture and add to the entry + for source_var, regex_options in self.plugin_options.source_variable_capture_dict.items(): + maybe_capture = regex_options.match.match_any(input_str=entry_variable_dict[source_var]) + + # If no capture + if maybe_capture is None: + # and no defaults + if not regex_options.has_defaults: + # Skip the entry if toggled + if self.plugin_options.skip_if_match_fails: + logger.info( + "Regex failed to match '%s' from '%s', skipping.", + source_var, + entry.title, + ) + return None + + # Otherwise, error + raise RegexNoMatchException( + f"Regex failed to match '{source_var}' from '{entry.title}'" + ) + + # otherwise, use defaults (apply them using the original entry source dict) + source_variables_and_overrides_dict = dict( + entry_variable_dict, **self.overrides.dict_with_format_strings + ) + + # add both the default... + entry.add_variables( + variables_to_add={ + regex_options.capture_group_names[i]: default.apply_formatter( + variable_dict=source_variables_and_overrides_dict + ) + for i, default in enumerate(regex_options.capture_group_defaults) + }, + ) + # and sanitized default + entry.add_variables( + variables_to_add={ + f"{regex_options.capture_group_names[i]}_sanitized": sanitize_filename( + default.apply_formatter( + variable_dict=source_variables_and_overrides_dict + ) + ) + for i, default in enumerate(regex_options.capture_group_defaults) + }, + ) + # There is a capture, add the source variables to the entry as + # {source_var}_capture_1, {source_var}_capture_2, ... + else: + # Add the value... + entry.add_variables( + variables_to_add={ + regex_options.capture_group_names[i]: capture + for i, capture in enumerate(maybe_capture) + }, + ) + # And the sanitized value + entry.add_variables( + variables_to_add={ + f"{regex_options.capture_group_names[i]}_sanitized": sanitize_filename( + capture + ) + for i, capture in enumerate(maybe_capture) + }, + ) + + return entry diff --git a/src/ytdl_sub/subscriptions/subscription.py b/src/ytdl_sub/subscriptions/subscription.py index 6123a164..8dab4fdf 100644 --- a/src/ytdl_sub/subscriptions/subscription.py +++ b/src/ytdl_sub/subscriptions/subscription.py @@ -3,6 +3,7 @@ import copy import os import shutil from pathlib import Path +from typing import Dict from typing import List from typing import Optional from typing import Tuple @@ -271,6 +272,17 @@ class Subscription: if isinstance(entry, tuple): entry, entry_metadata = entry + # First, modify the entry with all plugins + for plugin in plugins: + # Break out of this plugin loop if entry is None, it is indicated to not DL it + if (entry := plugin.modify_entry(entry)) is None: + break + + # If entry is None from the broken out loop, continue over the other entries + if entry is None: + continue + + # Then, post-process the entry with all plugins for plugin in plugins: optional_plugin_entry_metadata = plugin.post_process_entry(entry) if optional_plugin_entry_metadata: @@ -307,3 +319,30 @@ class Subscription: preset_options=preset, config_options=config.config_options, ) + + @classmethod + def from_dict(cls, config: ConfigFile, preset_name: str, preset_dict: Dict) -> "Subscription": + """ + Creates a subscription from a preset dict + + Parameters + ---------- + config: + Validated instance of the config + preset_name: + Name of the preset + preset_dict: + The preset config in dict format + + Returns + ------- + Initialized subscription + """ + return cls.from_preset( + preset=Preset.from_dict( + config=config, + preset_name=preset_name, + preset_dict=preset_dict, + ), + config=config, + ) diff --git a/src/ytdl_sub/utils/exceptions.py b/src/ytdl_sub/utils/exceptions.py index 9574c430..57153cb2 100644 --- a/src/ytdl_sub/utils/exceptions.py +++ b/src/ytdl_sub/utils/exceptions.py @@ -10,6 +10,10 @@ class StringFormattingVariableNotFoundException(StringFormattingException): """Tried to format a string but the variable was not found""" +class InvalidVariableNameException(ValidationException): + """A user defined variable name is invalid""" + + class DownloadArchiveException(ValueError): """Any user or file errors caused by download archive or mapping files""" @@ -20,3 +24,7 @@ class FileNotFoundException(ValidationException): class InvalidYamlException(ValidationException): """User yaml that is invalid""" + + +class RegexNoMatchException(ValidationException): + """Regex failed to match during download""" diff --git a/src/ytdl_sub/validators/regex_validator.py b/src/ytdl_sub/validators/regex_validator.py index b6a2317f..c9087b20 100644 --- a/src/ytdl_sub/validators/regex_validator.py +++ b/src/ytdl_sub/validators/regex_validator.py @@ -28,33 +28,20 @@ class RegexValidator(StringValidator): """ return self._compiled_regex.groups - def is_match(self, input_str: str) -> bool: + def match(self, input_str: str) -> Optional[List[str]]: """ Parameters ---------- input_str - String to match against the regex + String to regex match Returns ------- - True if input_str matches. False otherwise. - """ - return self._compiled_regex.search(input_str) is not None - - def capture(self, input_str: str) -> Optional[List[str]]: - """ - Parameters - ---------- - input_str - String to try to regex capture from - - Returns - ------- - List of captures (will always be >= 1). None if there are no captures. + List of captures. If the regex has no capture groups, then the list will be emtpy. + None is returned if the input_str failed to match """ if match := self._compiled_regex.search(input_str): - if len(to_return := list(match.groups())) > 0: - return to_return + return list(match.groups()) return None @@ -70,20 +57,18 @@ class RegexListValidator(ListValidator[RegexValidator]): "each regex in a list must have the same number of capture groups" ) - def matches_any(self, input_str: str) -> bool: - """ - Parameters - ---------- - input_str - String to match against any regexes in the list + self._num_capture_groups = self._list[0].num_capture_groups + @property + def num_capture_groups(self) -> int: + """ Returns ------- - True if at least one matches. False if none match + Number of capture groups. All regexes in the list will have the same number. """ - return any(reg.is_match(input_str) for reg in self._list) + return self._num_capture_groups - def capture_any(self, input_str: str) -> Optional[List[str]]: + def match_any(self, input_str: str) -> Optional[List[str]]: """ Parameters ---------- @@ -92,10 +77,10 @@ class RegexListValidator(ListValidator[RegexValidator]): Returns ------- - List of captures (will always be >= 1) on the first regex that matches. None if - no regexes match. + List of captures on the first regex that matches. If the regex has no capture groups, then + the list will be emtpy. None is returned if the input_str failed to match """ for reg in self._list: - if maybe_capture := reg.capture(input_str): + if (maybe_capture := reg.match(input_str)) is not None: return maybe_capture return None diff --git a/src/ytdl_sub/validators/source_variable_validator.py b/src/ytdl_sub/validators/source_variable_validator.py new file mode 100644 index 00000000..0050779d --- /dev/null +++ b/src/ytdl_sub/validators/source_variable_validator.py @@ -0,0 +1,20 @@ +from ytdl_sub.utils.exceptions import InvalidVariableNameException +from ytdl_sub.validators.string_formatter_validators import is_valid_source_variable_name +from ytdl_sub.validators.validators import ListValidator +from ytdl_sub.validators.validators import StringValidator + + +class SourceVariableNameValidator(StringValidator): + _expected_value_type_name = "source variable name" + + def __init__(self, name, value): + super().__init__(name, value) + try: + _ = is_valid_source_variable_name(self.value, raise_exception=True) + except InvalidVariableNameException as exc: + raise self._validation_exception(exc) from exc + + +class SourceVariableNameListValidator(ListValidator[SourceVariableNameValidator]): + _inner_list_type = SourceVariableNameValidator + _expected_value_type_name = "source variable name list" diff --git a/src/ytdl_sub/validators/string_formatter_validators.py b/src/ytdl_sub/validators/string_formatter_validators.py index 7b03dc17..c22cb49a 100644 --- a/src/ytdl_sub/validators/string_formatter_validators.py +++ b/src/ytdl_sub/validators/string_formatter_validators.py @@ -5,11 +5,45 @@ from typing import Dict from typing import List from typing import final +from ytdl_sub.utils.exceptions import InvalidVariableNameException from ytdl_sub.utils.exceptions import StringFormattingException from ytdl_sub.utils.exceptions import StringFormattingVariableNotFoundException +from ytdl_sub.validators.validators import ListValidator from ytdl_sub.validators.validators import LiteralDictValidator from ytdl_sub.validators.validators import Validator +_fields_validator = re.compile(r"{([a-z][a-z0-9_]+?)}") + +_fields_validator_exception_message: str = ( + "{variable_names} must start with a lowercase letter, should only contain lowercase letters, " + "numbers, underscores, and have a single open and close bracket." +) + + +def is_valid_source_variable_name(input_str: str, raise_exception: bool = False) -> bool: + """ + Parameters + ---------- + input_str + String to see if it can be a source variable + raise_exception + Raise InvalidVariableNameException False. + + Returns + ------- + True if it is. False otherwise. + + Raises + ------ + InvalidVariableNameException + If raise_exception and output is False + """ + # Add brackets around it to pretend its a StringFormatter, see if it captures + is_source_variable_name = len(re.findall(_fields_validator, f"{{{input_str}}}")) > 0 + if not is_source_variable_name and raise_exception: + raise InvalidVariableNameException(_fields_validator_exception_message) + return is_source_variable_name + class StringFormatterValidator(Validator): """ @@ -43,8 +77,6 @@ class StringFormatterValidator(Validator): "Format variable '{variable_name}' does not exist. Available variables: {available_fields}" ) - __fields_validator = re.compile(r"{([a-z_]+?)}") - __max_format_recursion = 3 def __validate_and_get_format_variables(self) -> List[str]: @@ -69,14 +101,11 @@ class StringFormatterValidator(Validator): exception_class=StringFormattingException, ) - format_variables: List[str] = list( - re.findall(StringFormatterValidator.__fields_validator, self.format_string) - ) + format_variables: List[str] = list(re.findall(_fields_validator, self.format_string)) if len(format_variables) != open_bracket_count: raise self._validation_exception( - "{variable_names} should only contain " - "lowercase letters and underscores with a single open and close bracket.", + error_message=_fields_validator_exception_message, exception_class=StringFormattingException, ) @@ -177,6 +206,10 @@ class OverridesStringFormatterValidator(StringFormatterValidator): # pylint: enable=line-too-long +class ListFormatterValidator(ListValidator[StringFormatterValidator]): + _inner_list_type = StringFormatterValidator + + class DictFormatterValidator(LiteralDictValidator): """ A dict made up of diff --git a/src/ytdl_sub/validators/validators.py b/src/ytdl_sub/validators/validators.py index e684cb8c..6f57a6e6 100644 --- a/src/ytdl_sub/validators/validators.py +++ b/src/ytdl_sub/validators/validators.py @@ -112,6 +112,15 @@ class ListValidator(Validator, ABC, Generic[T]): for i, val in enumerate(self._value) ] + @property + def list(self) -> List[T]: + """ + Returns + ------- + The list + """ + return self._list + class DictValidator(Validator): """ diff --git a/tests/e2e/conftest.py b/tests/e2e/conftest.py index 07af7b6b..c80ba719 100644 --- a/tests/e2e/conftest.py +++ b/tests/e2e/conftest.py @@ -2,8 +2,20 @@ import tempfile import pytest +from ytdl_sub.config.config_file import ConfigFile + @pytest.fixture() def output_directory(): with tempfile.TemporaryDirectory() as temp_dir: yield temp_dir + + +@pytest.fixture() +def music_video_config(): + return ConfigFile.from_file_path(config_path="examples/kodi_music_videos_config.yaml") + + +@pytest.fixture() +def channel_as_tv_show_config(): + return ConfigFile.from_file_path(config_path="examples/kodi_tv_shows_config.yaml") diff --git a/tests/e2e/plugins/__init__.py b/tests/e2e/plugins/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/e2e/plugins/test_regex.py b/tests/e2e/plugins/test_regex.py new file mode 100644 index 00000000..1b12e8ac --- /dev/null +++ b/tests/e2e/plugins/test_regex.py @@ -0,0 +1,208 @@ +import copy +import re + +import pytest +from e2e.expected_transaction_log import assert_transaction_log_matches + +from ytdl_sub.config.preset import Preset +from ytdl_sub.subscriptions.subscription import Subscription +from ytdl_sub.utils.exceptions import RegexNoMatchException +from ytdl_sub.utils.exceptions import ValidationException + + +@pytest.fixture +def regex_subscription_dict(output_directory): + return { + "preset": "yt_music_video_playlist", + "youtube": {"playlist_url": "https://youtube.com/playlist?list=PL5BC0FC26BECA5A35"}, + # override the output directory with our fixture-generated dir + "output_options": {"output_directory": output_directory}, + # download the worst format so it is fast + "ytdl_options": { + "format": "best[height<=480]", + }, + "regex": { + # tests that skip_if_match_fails defaults to True + "from": { + "title": { + "match": [ + "should not cap (.+) - (.+)", + ".*\\[(.+) - (Feb.+)]", # should filter out march video + ], + "capture_group_names": ["title_type", "title_date"], + }, + "description": { + "match": [".*http:\\/\\/(.+).com.*"], + "capture_group_names": ["description_website"], + }, + "upload_date_standardized": { + "match": ["([0-9]+)-([0-9]+)-27"], + "capture_group_names": [ + "upload_captured_year", + "upload_captured_month", + ], + "capture_group_defaults": [ + "First", + "Second containing {in_regex_default}", + ], + }, + "artist": { + "match": ["Never (.*) capture"], + "capture_group_names": ["always_default"], + "capture_group_defaults": ["Always default"], + }, + }, + }, + "nfo_tags": { + "tags": { + "title_cap_1": "{title_type}", + "title_cap_1_sanitized": "{title_type_sanitized}", + "title_cap_2": "{title_date}", + "desc_cap": "{description_website}", + "upload_date_both_caps": "{upload_captured_year} and {upload_captured_month}", + "artist_cap_always_default": "{always_default}", + "artist_cap_always_default_sanitized": "{always_default_sanitized}", + "override_with_capture_variable": "{contains_regex_default}", + "override_with_capture_variable_sanitized": "{contains_regex_sanitized_default}", + } + }, + "overrides": { + "in_regex_default": "in regex default", + "contains_regex_default": "contains {title_type}", + "contains_regex_sanitized_default": "contains {title_type_sanitized}", + }, + } + + +@pytest.fixture +def regex_subscription_dict_no_match_fails(regex_subscription_dict): + regex_subscription_dict["regex"]["skip_if_match_fails"] = False + return regex_subscription_dict + + +@pytest.fixture +def playlist_subscription(music_video_config, regex_subscription_dict): + playlist_preset = Preset.from_dict( + config=music_video_config, + preset_name="regex_capture_playlist_test", + preset_dict=regex_subscription_dict, + ) + + return Subscription.from_preset( + preset=playlist_preset, + config=music_video_config, + ) + + +@pytest.fixture +def playlist_subscription_no_match_fails( + music_video_config, regex_subscription_dict_no_match_fails +): + playlist_preset = Preset.from_dict( + config=music_video_config, + preset_name="regex_capture_playlist_test", + preset_dict=regex_subscription_dict_no_match_fails, + ) + + return Subscription.from_preset( + preset=playlist_preset, + config=music_video_config, + ) + + +class TestRegex: + def test_regex_success(self, playlist_subscription, output_directory): + # Only dry run is needed to see if capture variables are created + transaction_log = playlist_subscription.download(dry_run=True) + assert_transaction_log_matches( + output_directory=output_directory, + transaction_log=transaction_log, + transaction_log_summary_file_name="plugins/test_regex.txt", + ) + + def test_regex_fails_no_match(self, playlist_subscription_no_match_fails, output_directory): + with pytest.raises( + RegexNoMatchException, + match=re.escape( + "Regex failed to match 'title' from 'Jesse's Minecraft Server [Trailer - Mar.21]'" + ), + ): + _ = playlist_subscription_no_match_fails.download(dry_run=True) + + def test_regex_fails_capture_group_is_source_variable( + self, regex_subscription_dict, music_video_config + ): + regex_subscription_dict["regex"]["from"]["title"]["capture_group_names"][0] = "uid" + with pytest.raises( + ValidationException, + match=re.escape( + "'uid' cannot be used as a capture group name because it is a source variable" + ), + ): + _ = Subscription.from_dict( + config=music_video_config, + preset_name="test_regex_fails_capture_group_is_source_variable", + preset_dict=regex_subscription_dict, + ) + + def test_regex_fails_capture_group_is_override_variable( + self, regex_subscription_dict, music_video_config + ): + regex_subscription_dict["regex"]["from"]["title"]["capture_group_names"][ + 0 + ] = "in_regex_default" + with pytest.raises( + ValidationException, + match=re.escape( + "'in_regex_default' cannot be used as a capture group name because it is an override variable" + ), + ): + _ = Subscription.from_dict( + config=music_video_config, + preset_name="test_regex_fails_capture_group_is_override_variable", + preset_dict=regex_subscription_dict, + ) + + def test_regex_fails_source_variable_does_not_exist( + self, regex_subscription_dict, music_video_config + ): + regex_subscription_dict["regex"]["from"]["dne"] = copy.deepcopy( + regex_subscription_dict["regex"]["from"]["title"] + ) + with pytest.raises( + ValidationException, + match=re.escape("cannot regex capture 'dne' because it is not a source variable"), + ): + _ = Subscription.from_dict( + config=music_video_config, + preset_name="test_regex_fails_source_variable_does_not_exist", + preset_dict=regex_subscription_dict, + ) + + def test_regex_fails_unequal_defaults(self, regex_subscription_dict, music_video_config): + regex_subscription_dict["regex"]["from"]["title"]["capture_group_defaults"] = ["1 != 2"] + with pytest.raises( + ValidationException, + match=re.escape("number of defaults must match number of capture groups, 1 != 2"), + ): + _ = Subscription.from_dict( + config=music_video_config, + preset_name="test_regex_fails_unequal_defaults", + preset_dict=regex_subscription_dict, + ) + + def test_regex_fails_unequal_capture_group_names( + self, regex_subscription_dict, music_video_config + ): + regex_subscription_dict["regex"]["from"]["title"]["capture_group_names"].append("unequal") + with pytest.raises( + ValidationException, + match=re.escape( + "number of capture group names must match number of capture groups, 3 != 2" + ), + ): + _ = Subscription.from_dict( + config=music_video_config, + preset_name="test_regex_fails_unequal_capture_group_names", + preset_dict=regex_subscription_dict, + ) diff --git a/tests/e2e/resources/transaction_log_summaries/plugins/test_regex.txt b/tests/e2e/resources/transaction_log_summaries/plugins/test_regex.txt new file mode 100644 index 00000000..97fd64dc --- /dev/null +++ b/tests/e2e/resources/transaction_log_summaries/plugins/test_regex.txt @@ -0,0 +1,39 @@ +Files created in '{output_directory}' +---------------------------------------- +.ytdl-sub-regex_capture_playlist_test-download-archive.json +Project Zombie - Jesse's Minecraft Server [Trailer - Feb.1]-thumb.jpg +Project Zombie - Jesse's Minecraft Server [Trailer - Feb.1].mp4 +Project Zombie - Jesse's Minecraft Server [Trailer - Feb.1].nfo + NFO tags: + musicvideo: + album: Music Videos + artist: Project Zombie + artist_cap_always_default: Always default + artist_cap_always_default_sanitized: Always default + desc_cap: www.jesseminecraft.webs + override_with_capture_variable: contains Trailer + override_with_capture_variable_sanitized: contains Trailer + title: Jesse's Minecraft Server [Trailer - Feb.1] + title_cap_1: Trailer + title_cap_1_sanitized: Trailer + title_cap_2: Feb.1 + upload_date_both_caps: First and Second containing in regex default + year: 2011 +Project Zombie - Jesse's Minecraft Server [Trailer - Feb.27]-thumb.jpg +Project Zombie - Jesse's Minecraft Server [Trailer - Feb.27].mp4 +Project Zombie - Jesse's Minecraft Server [Trailer - Feb.27].nfo + NFO tags: + musicvideo: + album: Music Videos + artist: Project Zombie + artist_cap_always_default: Always default + artist_cap_always_default_sanitized: Always default + desc_cap: jesseminecraft.webs + override_with_capture_variable: contains Trailer + override_with_capture_variable_sanitized: contains Trailer + title: Jesse's Minecraft Server [Trailer - Feb.27] + title_cap_1: Trailer + title_cap_1_sanitized: Trailer + title_cap_2: Feb.27 + upload_date_both_caps: 2011 and 02 + year: 2011 \ No newline at end of file diff --git a/tests/e2e/resources/transaction_log_summaries/youtube/test_playlist.txt b/tests/e2e/resources/transaction_log_summaries/youtube/test_playlist.txt index 3d31bea7..603789f6 100644 --- a/tests/e2e/resources/transaction_log_summaries/youtube/test_playlist.txt +++ b/tests/e2e/resources/transaction_log_summaries/youtube/test_playlist.txt @@ -1,6 +1,6 @@ Files created in '{output_directory}' ---------------------------------------- -.ytdl-sub-jmc-download-archive.json +.ytdl-sub-music_video_playlist_test-download-archive.json JMC - Jesse's Minecraft Server [Trailer - Feb.1]-thumb.jpg JMC - Jesse's Minecraft Server [Trailer - Feb.1].mp4 JMC - Jesse's Minecraft Server [Trailer - Feb.1].nfo diff --git a/tests/e2e/youtube/test_playlist_as_kodi_music_videos.py b/tests/e2e/youtube/test_playlist_as_kodi_music_videos.py index 76111d1b..a8e0e797 100644 --- a/tests/e2e/youtube/test_playlist_as_kodi_music_videos.py +++ b/tests/e2e/youtube/test_playlist_as_kodi_music_videos.py @@ -8,28 +8,12 @@ from e2e.expected_download import ExpectedDownloads from e2e.expected_transaction_log import assert_transaction_log_matches import ytdl_sub.downloaders.downloader -from ytdl_sub.config.config_file import ConfigFile from ytdl_sub.config.preset import Preset from ytdl_sub.subscriptions.subscription import Subscription @pytest.fixture -def config_path(): - return "examples/kodi_music_videos_config.yaml" - - -@pytest.fixture -def subscription_name(): - return "jmc" - - -@pytest.fixture -def config(config_path): - return ConfigFile.from_file_path(config_path=config_path) - - -@pytest.fixture -def subscription_dict(output_directory, subscription_name): +def subscription_dict(output_directory): return { "preset": "yt_music_video_playlist", "youtube": {"playlist_url": "https://youtube.com/playlist?list=PL5BC0FC26BECA5A35"}, @@ -48,16 +32,16 @@ def subscription_dict(output_directory, subscription_name): @pytest.fixture -def playlist_subscription(config, subscription_name, subscription_dict): +def playlist_subscription(music_video_config, subscription_dict): playlist_preset = Preset.from_dict( - config=config, - preset_name=subscription_name, + config=music_video_config, + preset_name="music_video_playlist_test", preset_dict=subscription_dict, ) return Subscription.from_preset( preset=playlist_preset, - config=config, + config=music_video_config, ) @@ -68,7 +52,7 @@ def expected_playlist_download(): return ExpectedDownloads( expected_downloads=[ # Download mapping - ExpectedDownloadFile(path=Path(".ytdl-sub-jmc-download-archive.json"), md5="9f785c29194a6ecfba6a6b4018763ddc"), + ExpectedDownloadFile(path=Path(".ytdl-sub-music_video_playlist_test-download-archive.json"), md5="9f785c29194a6ecfba6a6b4018763ddc"), # Entry files ExpectedDownloadFile(path=Path("JMC - Jesse's Minecraft Server [Trailer - Feb.1]-thumb.jpg"), md5="b232d253df621aa770b780c1301d364d"), @@ -103,16 +87,16 @@ def single_video_subscription_dict(subscription_dict): @pytest.fixture -def single_video_subscription(config, subscription_name, single_video_subscription_dict): +def single_video_subscription(music_video_config, single_video_subscription_dict): single_video_preset = Preset.from_dict( - config=config, - preset_name=subscription_name, + config=music_video_config, + preset_name="music_video_single_video_test", preset_dict=single_video_subscription_dict, ) return Subscription.from_preset( preset=single_video_preset, - config=config, + config=music_video_config, ) diff --git a/tests/e2e/youtube/test_split_video.py b/tests/e2e/youtube/test_split_video.py index cb324ce6..0773ce3a 100644 --- a/tests/e2e/youtube/test_split_video.py +++ b/tests/e2e/youtube/test_split_video.py @@ -1,24 +1,15 @@ from pathlib import Path from tempfile import NamedTemporaryFile -import mergedeep import pytest -from conftest import assert_debug_log from e2e.expected_download import ExpectedDownloadFile from e2e.expected_download import ExpectedDownloads from e2e.expected_transaction_log import assert_transaction_log_matches -import ytdl_sub.downloaders.downloader -from ytdl_sub.config.config_file import ConfigFile from ytdl_sub.config.preset import Preset from ytdl_sub.subscriptions.subscription import Subscription -@pytest.fixture -def config_path(): - return "examples/kodi_music_videos_config.yaml" - - @pytest.fixture def split_timestamps_file_path(): timestamps = [ @@ -37,17 +28,7 @@ def split_timestamps_file_path(): @pytest.fixture -def subscription_name(): - return "jmc" - - -@pytest.fixture -def config(config_path): - return ConfigFile.from_file_path(config_path=config_path) - - -@pytest.fixture -def subscription_dict(output_directory, subscription_name, split_timestamps_file_path): +def subscription_dict(output_directory, split_timestamps_file_path): return { "preset": "yt_music_video", "youtube": { @@ -73,16 +54,16 @@ def subscription_dict(output_directory, subscription_name, split_timestamps_file @pytest.fixture -def single_video_subscription(config, subscription_name, subscription_dict): +def single_video_subscription(music_video_config, subscription_dict): single_video_preset = Preset.from_dict( - config=config, - preset_name=subscription_name, + config=music_video_config, + preset_name="split_video_test", preset_dict=subscription_dict, ) return Subscription.from_preset( preset=single_video_preset, - config=config, + config=music_video_config, ) diff --git a/tests/unit/plugins/__init__.py b/tests/unit/plugins/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/unit/plugins/test_regex_capture.py b/tests/unit/plugins/test_regex_capture.py new file mode 100644 index 00000000..8b137891 --- /dev/null +++ b/tests/unit/plugins/test_regex_capture.py @@ -0,0 +1 @@ + diff --git a/tests/unit/validators/test_regex_validator.py b/tests/unit/validators/test_regex_validator.py index ab49af6c..87ecfb39 100644 --- a/tests/unit/validators/test_regex_validator.py +++ b/tests/unit/validators/test_regex_validator.py @@ -6,21 +6,20 @@ from ytdl_sub.validators.regex_validator import RegexValidator @pytest.mark.parametrize( - "regex_value, input_str, matches, captures", + "regex_value, input_str, expected_output", [ - ("^match this$", "match this", True, None), - ("^my (.+) cap", "my first cap", True, ["first"]), - (".* (.+) - (.+) two", "my other - capped two", True, ["other", "capped"]), - ("failed match", "nope", False, None), + ("^match this$", "match this", []), + ("^my (.+) cap", "my first cap", ["first"]), + (".* (.+) - (.+) two", "my other - capped two", ["other", "capped"]), + ("failed match", "nope", None), ], ) -def test_regex_validator(regex_value, input_str, matches, captures): +def test_regex_validator(regex_value, input_str, expected_output): regex_validator = RegexValidator(name="good_regex_validator", value=regex_value) assert regex_validator._name == "good_regex_validator" assert regex_validator._value == regex_value - assert regex_validator.is_match(input_str=input_str) is matches - assert regex_validator.capture(input_str=input_str) == captures + assert regex_validator.match(input_str=input_str) == expected_output @pytest.mark.parametrize("regex_value", ["(", "??"]) @@ -29,17 +28,16 @@ def test_regex_validator_fails_bad_value(regex_value): _ = RegexValidator(name="fail", value=regex_value) -def test_regex_list_validator(): +def test_regex_list_validator_matches(): regex_list_validator_raw_value = ["try matching this", "try matching that", "how about this"] regex_list = RegexListValidator(name="list val", value=regex_list_validator_raw_value) for raw_val in regex_list_validator_raw_value: - assert regex_list.matches_any(raw_val) - assert regex_list.capture_any(raw_val) is None + assert regex_list.match_any(raw_val) == [] -def test_regex_list_validator_capture(): +def test_regex_list_validator_captures(): regex_list_validator_raw_value = ["try (.+) this", "try (.+) that", "how (.+) this"] regex_list = RegexListValidator(name="list val", value=regex_list_validator_raw_value) @@ -50,8 +48,7 @@ def test_regex_list_validator_capture(): ] for capture_str, expected_capture in captures_test: - assert regex_list.matches_any(capture_str) - assert regex_list.capture_any(capture_str) == [expected_capture] + assert regex_list.match_any(capture_str) == [expected_capture] def test_regex_list_validator_invalid_regex(): diff --git a/tests/unit/validators/test_string_formatter_validator.py b/tests/unit/validators/test_string_formatter_validator.py index d838da34..c325c8a1 100644 --- a/tests/unit/validators/test_string_formatter_validator.py +++ b/tests/unit/validators/test_string_formatter_validator.py @@ -19,8 +19,8 @@ def error_message_unequal_brackets_str(): @pytest.fixture def error_message_unequal_regex_matches_str(): return ( - "{variable_names} should only contain lowercase letters and " - "underscores with a single open and close bracket." + "{variable_names} must start with a lowercase letter, should only contain lowercase " + "letters, numbers, underscores, and have a single open and close bracket." ) @@ -29,11 +29,11 @@ def error_message_unequal_regex_matches_str(): ) class TestStringFormatterValidator(object): def test_parse(self, string_formatter_class): - format_string = "Here is my {var_one} and {var_two} 💩" + format_string = "Here is my {var_one1} and {var_two} 💩" validator = string_formatter_class(name="test_format_variables", value=format_string) assert validator.format_string == format_string - assert validator.format_variables == ["var_one", "var_two"] + assert validator.format_variables == ["var_one1", "var_two"] def test_format_variables(self, string_formatter_class): format_string = "No vars 💩" @@ -64,7 +64,8 @@ class TestStringFormatterValidator(object): @pytest.mark.parametrize( "format_string", [ - "Try {var1} numbers", + "Try {1starting_number}", + "Try {_underscore_first}", "Try {VAR1} caps w/numbers", "Try {internal{bracket}}", "Try }backwards{ facing",