diff --git a/src/ytdl_sub/plugins/regex.py b/src/ytdl_sub/plugins/regex.py index c4611d6c..8deec694 100644 --- a/src/ytdl_sub/plugins/regex.py +++ b/src/ytdl_sub/plugins/regex.py @@ -8,6 +8,7 @@ from ytdl_sub.plugins.plugin import PluginOptions from ytdl_sub.utils.exceptions import RegexNoMatchException from ytdl_sub.utils.logger import Logger from ytdl_sub.validators.regex_validator import RegexListValidator +from ytdl_sub.validators.source_variable_validator import SourceVariableNameListValidator from ytdl_sub.validators.strict_dict_validator import StrictDictValidator from ytdl_sub.validators.string_formatter_validators import ListFormatterValidator from ytdl_sub.validators.string_formatter_validators import StringFormatterValidator @@ -16,14 +17,10 @@ from ytdl_sub.validators.validators import BoolValidator logger = Logger.get(name="regex") -def _source_var_name(source_variable: str, capture_group_idx: int) -> str: - return f"{source_variable}_capture_{capture_group_idx+1}" - - class SourceVariableRegex(StrictDictValidator): _required_keys = {"match"} - _optional_keys = {"defaults"} + _optional_keys = {"defaults", "capture_group_names"} def __init__(self, name, value): super().__init__(name, value) @@ -31,6 +28,9 @@ class SourceVariableRegex(StrictDictValidator): self._defaults = self._validate_key_if_present( key="defaults", validator=ListFormatterValidator ) + self._capture_group_names = self._validate_key_if_present( + key="capture_group_names", validator=SourceVariableNameListValidator, default=[] + ) # If defaults are to be used, ensure there are the same number of defaults as there are # capture groups @@ -42,6 +42,13 @@ class SourceVariableRegex(StrictDictValidator): f"{len(self._defaults.list)} != {self._match.num_capture_groups}" ) + # If there are capture groups, ensure there are capture group names + if len(self._capture_group_names.list) != self._match.num_capture_groups: + raise self._validation_exception( + f"Number of capture group names must match number of capture groups, " + f"{len(self._capture_group_names.list)} != {self._match.num_capture_groups}" + ) + @property def capture_list(self) -> RegexListValidator: """ @@ -69,6 +76,15 @@ class SourceVariableRegex(StrictDictValidator): """ return self._defaults.list if self.has_defaults else None + @property + def capture_group_names(self) -> List[str]: + """ + Returns + ------- + List of new capture group names + """ + return [validator.value for validator in self._capture_group_names.list] + class FromSourceVariablesRegex(StrictDictValidator): @@ -83,9 +99,9 @@ class FromSourceVariablesRegex(StrictDictValidator): class RegexOptions(PluginOptions): - """ - Performs regex matching on an entry's source variables. Regex can be used to either capture - groups to create new source variables or filter entries from proceeding with download. + r""" + Performs regex matching on an entry's source variables. Regex can be used to filter entries + from proceeding with download or capture groups to create new source variables. Usage: @@ -94,25 +110,32 @@ class RegexOptions(PluginOptions): presets: my_example_preset: regex: - skip_if_match_fails: False - from: - title: - # Match with no defaults act as a filter. - # This will only download videos with "Official Video" in it. - match: "\[Official Video\]" - description: - # Match with capture groups and defaults. - # This tries to scape a date from the description and produce new source variables - # {description_capture_1}, {description_capture_2}, {description_capture_3} - match: "([0-9]{4})-([0-9]{2})-([0-9]{2})" - capture_group_defaults: - - "{upload_year}" - - "{upload_month}" - - "{upload_day}" + # By default, if any match fails and has no defaults, the entry will be skipped. + # If set to False, ytdl-sub will error and stop all downloads from proceeding. + skip_if_match_fails: True - TODO: add test with override variables in the capture group defaults, and override - variable - referencing a capture variable + from: + title: + # Match with capture groups act as a filter. + # This will only download videos with "Official Video" in it. + match: "\[Official Video\]" + description: + # Match with capture groups and defaults. + # This tries to scrape a date from the description and produce new source variables + match: "([0-9]{4})-([0-9]{2})-([0-9]{2})" + + # Each capture group creates these new source variables, respectively + capture_group_names: + - "captured_upload_year" + - "captured_upload_month" + - "captured_upload_day" + + # And if the string does not match, use these as respective default values for the + # new source variables. + capture_group_defaults: + - "{upload_year}" + - "{upload_month}" + - "{upload_day}" """ _required_keys = {"from"} @@ -156,11 +179,8 @@ class RegexOptions(PluginOptions): List of new source variables created via regex capture """ added_source_vars: List[str] = [] - for source_var, regex_options in self.source_variable_capture_dict.items(): - added_source_vars.extend( - _source_var_name(source_var, idx) - for idx in range(regex_options.capture_list.num_capture_groups) - ) + for regex_options in self.source_variable_capture_dict.values(): + added_source_vars.extend(regex_options.capture_group_names) return added_source_vars @@ -216,7 +236,7 @@ class RegexPlugin(Plugin[RegexOptions]): ) entry.add_variables( variables_to_add={ - _source_var_name(source_var, i): default.apply_formatter( + regex_options.capture_group_names[i]: default.apply_formatter( variable_dict=source_variables_and_overrides_dict ) for i, default in enumerate(regex_options.defaults) @@ -227,7 +247,7 @@ class RegexPlugin(Plugin[RegexOptions]): else: entry.add_variables( variables_to_add={ - _source_var_name(source_var, i): capture + regex_options.capture_group_names[i]: capture for i, capture in enumerate(maybe_capture) }, ) diff --git a/src/ytdl_sub/utils/exceptions.py b/src/ytdl_sub/utils/exceptions.py index 1a8ac4ff..57153cb2 100644 --- a/src/ytdl_sub/utils/exceptions.py +++ b/src/ytdl_sub/utils/exceptions.py @@ -10,6 +10,10 @@ class StringFormattingVariableNotFoundException(StringFormattingException): """Tried to format a string but the variable was not found""" +class InvalidVariableNameException(ValidationException): + """A user defined variable name is invalid""" + + class DownloadArchiveException(ValueError): """Any user or file errors caused by download archive or mapping files""" diff --git a/src/ytdl_sub/validators/source_variable_validator.py b/src/ytdl_sub/validators/source_variable_validator.py new file mode 100644 index 00000000..0050779d --- /dev/null +++ b/src/ytdl_sub/validators/source_variable_validator.py @@ -0,0 +1,20 @@ +from ytdl_sub.utils.exceptions import InvalidVariableNameException +from ytdl_sub.validators.string_formatter_validators import is_valid_source_variable_name +from ytdl_sub.validators.validators import ListValidator +from ytdl_sub.validators.validators import StringValidator + + +class SourceVariableNameValidator(StringValidator): + _expected_value_type_name = "source variable name" + + def __init__(self, name, value): + super().__init__(name, value) + try: + _ = is_valid_source_variable_name(self.value, raise_exception=True) + except InvalidVariableNameException as exc: + raise self._validation_exception(exc) from exc + + +class SourceVariableNameListValidator(ListValidator[SourceVariableNameValidator]): + _inner_list_type = SourceVariableNameValidator + _expected_value_type_name = "source variable name list" diff --git a/src/ytdl_sub/validators/string_formatter_validators.py b/src/ytdl_sub/validators/string_formatter_validators.py index 8f3b2a09..c22cb49a 100644 --- a/src/ytdl_sub/validators/string_formatter_validators.py +++ b/src/ytdl_sub/validators/string_formatter_validators.py @@ -5,12 +5,45 @@ from typing import Dict from typing import List from typing import final +from ytdl_sub.utils.exceptions import InvalidVariableNameException from ytdl_sub.utils.exceptions import StringFormattingException from ytdl_sub.utils.exceptions import StringFormattingVariableNotFoundException from ytdl_sub.validators.validators import ListValidator from ytdl_sub.validators.validators import LiteralDictValidator from ytdl_sub.validators.validators import Validator +_fields_validator = re.compile(r"{([a-z][a-z0-9_]+?)}") + +_fields_validator_exception_message: str = ( + "{variable_names} must start with a lowercase letter, should only contain lowercase letters, " + "numbers, underscores, and have a single open and close bracket." +) + + +def is_valid_source_variable_name(input_str: str, raise_exception: bool = False) -> bool: + """ + Parameters + ---------- + input_str + String to see if it can be a source variable + raise_exception + Raise InvalidVariableNameException False. + + Returns + ------- + True if it is. False otherwise. + + Raises + ------ + InvalidVariableNameException + If raise_exception and output is False + """ + # Add brackets around it to pretend its a StringFormatter, see if it captures + is_source_variable_name = len(re.findall(_fields_validator, f"{{{input_str}}}")) > 0 + if not is_source_variable_name and raise_exception: + raise InvalidVariableNameException(_fields_validator_exception_message) + return is_source_variable_name + class StringFormatterValidator(Validator): """ @@ -44,8 +77,6 @@ class StringFormatterValidator(Validator): "Format variable '{variable_name}' does not exist. Available variables: {available_fields}" ) - __fields_validator = re.compile(r"{([a-z][a-z0-9_]+?)}") - __max_format_recursion = 3 def __validate_and_get_format_variables(self) -> List[str]: @@ -70,14 +101,11 @@ class StringFormatterValidator(Validator): exception_class=StringFormattingException, ) - format_variables: List[str] = list( - re.findall(StringFormatterValidator.__fields_validator, self.format_string) - ) + format_variables: List[str] = list(re.findall(_fields_validator, self.format_string)) if len(format_variables) != open_bracket_count: raise self._validation_exception( - "{variable_names} must start with a lowercase letter, should only contain lowercase" - "letters, numbers, underscores, and have a single open and close bracket.", + error_message=_fields_validator_exception_message, exception_class=StringFormattingException, ) diff --git a/tests/e2e/plugins/test_regex.py b/tests/e2e/plugins/test_regex.py index f2f0dcc5..fc5100cb 100644 --- a/tests/e2e/plugins/test_regex.py +++ b/tests/e2e/plugins/test_regex.py @@ -27,32 +27,44 @@ def regex_subscription_dict(output_directory): "should not cap (.+) - (.+)", ".*\\[(.+) - (Feb.+)]", # should filter out march video ], + "capture_group_names": ["title_type", "title_date"], + }, + "description": { + "match": [".*http:\\/\\/(.+).com.*"], + "capture_group_names": ["description_website"], }, - "description": {"match": [".*http:\\/\\/(.+).com.*"]}, "upload_date_standardized": { "match": ["([0-9]+)-([0-9]+)-27"], + "capture_group_names": [ + "upload_captured_year", + "upload_captured_month", + ], "defaults": [ "First", "Second containing {in_regex_default}", ], }, - "artist": {"match": ["Never (.*) capture"], "defaults": ["Always default"]}, + "artist": { + "match": ["Never (.*) capture"], + "capture_group_names": ["always_default"], + "defaults": ["Always default"], + }, }, }, "nfo_tags": { "tags": { - "title_cap_1": "{title_capture_1}", - "title_cap_2": "{title_capture_2}", - "desc_cap": "{description_capture_1}", - "upload_date_both_caps": "{upload_date_standardized_capture_1} and {upload_date_standardized_capture_2}", - "artist_cap_always_default": "{artist_capture_1}", - "override_with_capture_variable": "{contains_regex_default}" + "title_cap_1": "{title_type}", + "title_cap_2": "{title_date}", + "desc_cap": "{description_website}", + "upload_date_both_caps": "{upload_captured_year} and {upload_captured_month}", + "artist_cap_always_default": "{always_default}", + "override_with_capture_variable": "{contains_regex_default}", } }, "overrides": { "in_regex_default": "in regex default", - "contains_regex_default": "contains {title_capture_1}" - } + "contains_regex_default": "contains {title_type}", + }, } diff --git a/tests/e2e/resources/transaction_log_summaries/plugins/test_regex.txt b/tests/e2e/resources/transaction_log_summaries/plugins/test_regex.txt index 2565c3a6..7465ae89 100644 --- a/tests/e2e/resources/transaction_log_summaries/plugins/test_regex.txt +++ b/tests/e2e/resources/transaction_log_summaries/plugins/test_regex.txt @@ -10,10 +10,11 @@ Project Zombie - Jesse's Minecraft Server [Trailer - Feb.1].nfo artist: Project Zombie artist_cap_always_default: Always default desc_cap: www.jesseminecraft.webs + override_with_capture_variable: contains Trailer title: Jesse's Minecraft Server [Trailer - Feb.1] title_cap_1: Trailer title_cap_2: Feb.1 - upload_date_both_caps: First and Second + upload_date_both_caps: First and Second containing in regex default year: 2011 Project Zombie - Jesse's Minecraft Server [Trailer - Feb.27]-thumb.jpg Project Zombie - Jesse's Minecraft Server [Trailer - Feb.27].mp4 @@ -24,6 +25,7 @@ Project Zombie - Jesse's Minecraft Server [Trailer - Feb.27].nfo artist: Project Zombie artist_cap_always_default: Always default desc_cap: jesseminecraft.webs + override_with_capture_variable: contains Trailer title: Jesse's Minecraft Server [Trailer - Feb.27] title_cap_1: Trailer title_cap_2: Feb.27