e2e tested, need more failure tests, no dupe variable names, add docs

src/ytdl_sub/plugins/regex.py

@@ -8,6 +8,7 @@ from ytdl_sub.plugins.plugin import PluginOptions
 from ytdl_sub.utils.exceptions import RegexNoMatchException
 from ytdl_sub.utils.logger import Logger
 from ytdl_sub.validators.regex_validator import RegexListValidator
+from ytdl_sub.validators.source_variable_validator import SourceVariableNameListValidator
 from ytdl_sub.validators.strict_dict_validator import StrictDictValidator
 from ytdl_sub.validators.string_formatter_validators import ListFormatterValidator
 from ytdl_sub.validators.string_formatter_validators import StringFormatterValidator
@@ -16,14 +17,10 @@ from ytdl_sub.validators.validators import BoolValidator
 logger = Logger.get(name="regex")
 
 
-def _source_var_name(source_variable: str, capture_group_idx: int) -> str:
-    return f"{source_variable}_capture_{capture_group_idx+1}"
-
-
 class SourceVariableRegex(StrictDictValidator):
 
     _required_keys = {"match"}
-    _optional_keys = {"defaults"}
+    _optional_keys = {"defaults", "capture_group_names"}
 
     def __init__(self, name, value):
         super().__init__(name, value)
@@ -31,6 +28,9 @@ class SourceVariableRegex(StrictDictValidator):
         self._defaults = self._validate_key_if_present(
             key="defaults", validator=ListFormatterValidator
         )
+        self._capture_group_names = self._validate_key_if_present(
+            key="capture_group_names", validator=SourceVariableNameListValidator, default=[]
+        )
 
         # If defaults are to be used, ensure there are the same number of defaults as there are
         # capture groups
@@ -42,6 +42,13 @@ class SourceVariableRegex(StrictDictValidator):
                 f"{len(self._defaults.list)} != {self._match.num_capture_groups}"
             )
 
+        # If there are capture groups, ensure there are capture group names
+        if len(self._capture_group_names.list) != self._match.num_capture_groups:
+            raise self._validation_exception(
+                f"Number of capture group names must match number of capture groups, "
+                f"{len(self._capture_group_names.list)} != {self._match.num_capture_groups}"
+            )
+
     @property
     def capture_list(self) -> RegexListValidator:
         """
@@ -69,6 +76,15 @@ class SourceVariableRegex(StrictDictValidator):
         """
         return self._defaults.list if self.has_defaults else None
 
+    @property
+    def capture_group_names(self) -> List[str]:
+        """
+        Returns
+        -------
+        List of new capture group names
+        """
+        return [validator.value for validator in self._capture_group_names.list]
+
 
 class FromSourceVariablesRegex(StrictDictValidator):
 
@@ -83,9 +99,9 @@ class FromSourceVariablesRegex(StrictDictValidator):
 
 
 class RegexOptions(PluginOptions):
-    """
+    r"""
-    Performs regex matching on an entry's source variables. Regex can be used to either capture
+    Performs regex matching on an entry's source variables. Regex can be used to filter entries
-    groups to create new source variables or filter entries from proceeding with download.
+    from proceeding with download or capture groups to create new source variables.
 
     Usage:
 
@@ -94,25 +110,32 @@ class RegexOptions(PluginOptions):
        presets:
          my_example_preset:
            regex:
-            skip_if_match_fails: False
+             # By default, if any match fails and has no defaults, the entry will be skipped.
-            from:
+             # If set to False, ytdl-sub will error and stop all downloads from proceeding.
-              title:
+             skip_if_match_fails: True
-                # Match with no defaults act as a filter.
-                # This will only download videos with "Official Video" in it.
-                match: "\[Official Video\]"
-              description:
-                # Match with capture groups and defaults.
-                # This tries to scape a date from the description and produce new source variables
-                # {description_capture_1}, {description_capture_2}, {description_capture_3}
-                match: "([0-9]{4})-([0-9]{2})-([0-9]{2})"
-                capture_group_defaults:
-                  - "{upload_year}"
-                  - "{upload_month}"
-                  - "{upload_day}"
 
-            TODO: add test with override variables in the capture group defaults, and override
+             from:
-            variable
+               title:
-            referencing a capture variable
+                 # Match with capture groups act as a filter.
+                 # This will only download videos with "Official Video" in it.
+                 match: "\[Official Video\]"
+               description:
+                 # Match with capture groups and defaults.
+                 # This tries to scrape a date from the description and produce new source variables
+                 match: "([0-9]{4})-([0-9]{2})-([0-9]{2})"
+
+                 # Each capture group creates these new source variables, respectively
+                 capture_group_names:
+                  - "captured_upload_year"
+                  - "captured_upload_month"
+                  - "captured_upload_day"
+
+                 # And if the string does not match, use these as respective default values for the
+                 # new source variables.
+                 capture_group_defaults:
+                   - "{upload_year}"
+                   - "{upload_month}"
+                   - "{upload_day}"
     """
 
     _required_keys = {"from"}
@@ -156,11 +179,8 @@ class RegexOptions(PluginOptions):
         List of new source variables created via regex capture
         """
         added_source_vars: List[str] = []
-        for source_var, regex_options in self.source_variable_capture_dict.items():
+        for regex_options in self.source_variable_capture_dict.values():
-            added_source_vars.extend(
+            added_source_vars.extend(regex_options.capture_group_names)
-                _source_var_name(source_var, idx)
-                for idx in range(regex_options.capture_list.num_capture_groups)
-            )
 
         return added_source_vars
 
@@ -216,7 +236,7 @@ class RegexPlugin(Plugin[RegexOptions]):
                 )
                 entry.add_variables(
                     variables_to_add={
-                        _source_var_name(source_var, i): default.apply_formatter(
+                        regex_options.capture_group_names[i]: default.apply_formatter(
                             variable_dict=source_variables_and_overrides_dict
                         )
                         for i, default in enumerate(regex_options.defaults)
@@ -227,7 +247,7 @@ class RegexPlugin(Plugin[RegexOptions]):
             else:
                 entry.add_variables(
                     variables_to_add={
-                        _source_var_name(source_var, i): capture
+                        regex_options.capture_group_names[i]: capture
                         for i, capture in enumerate(maybe_capture)
                     },
                 )

src/ytdl_sub/utils/exceptions.py

@@ -10,6 +10,10 @@ class StringFormattingVariableNotFoundException(StringFormattingException):
     """Tried to format a string but the variable was not found"""
 
 
+class InvalidVariableNameException(ValidationException):
+    """A user defined variable name is invalid"""
+
+
 class DownloadArchiveException(ValueError):
     """Any user or file errors caused by download archive or mapping files"""
 

src/ytdl_sub/validators/source_variable_validator.py

@@ -0,0 +1,20 @@
+from ytdl_sub.utils.exceptions import InvalidVariableNameException
+from ytdl_sub.validators.string_formatter_validators import is_valid_source_variable_name
+from ytdl_sub.validators.validators import ListValidator
+from ytdl_sub.validators.validators import StringValidator
+
+
+class SourceVariableNameValidator(StringValidator):
+    _expected_value_type_name = "source variable name"
+
+    def __init__(self, name, value):
+        super().__init__(name, value)
+        try:
+            _ = is_valid_source_variable_name(self.value, raise_exception=True)
+        except InvalidVariableNameException as exc:
+            raise self._validation_exception(exc) from exc
+
+
+class SourceVariableNameListValidator(ListValidator[SourceVariableNameValidator]):
+    _inner_list_type = SourceVariableNameValidator
+    _expected_value_type_name = "source variable name list"

src/ytdl_sub/validators/string_formatter_validators.py

@@ -5,12 +5,45 @@ from typing import Dict
 from typing import List
 from typing import final
 
+from ytdl_sub.utils.exceptions import InvalidVariableNameException
 from ytdl_sub.utils.exceptions import StringFormattingException
 from ytdl_sub.utils.exceptions import StringFormattingVariableNotFoundException
 from ytdl_sub.validators.validators import ListValidator
 from ytdl_sub.validators.validators import LiteralDictValidator
 from ytdl_sub.validators.validators import Validator
 
+_fields_validator = re.compile(r"{([a-z][a-z0-9_]+?)}")
+
+_fields_validator_exception_message: str = (
+    "{variable_names} must start with a lowercase letter, should only contain lowercase letters, "
+    "numbers, underscores, and have a single open and close bracket."
+)
+
+
+def is_valid_source_variable_name(input_str: str, raise_exception: bool = False) -> bool:
+    """
+    Parameters
+    ----------
+    input_str
+        String to see if it can be a source variable
+    raise_exception
+        Raise InvalidVariableNameException False.
+
+    Returns
+    -------
+    True if it is. False otherwise.
+
+    Raises
+    ------
+    InvalidVariableNameException
+        If raise_exception and output is False
+    """
+    # Add brackets around it to pretend its a StringFormatter, see if it captures
+    is_source_variable_name = len(re.findall(_fields_validator, f"{{{input_str}}}")) > 0
+    if not is_source_variable_name and raise_exception:
+        raise InvalidVariableNameException(_fields_validator_exception_message)
+    return is_source_variable_name
+
 
 class StringFormatterValidator(Validator):
     """
@@ -44,8 +77,6 @@ class StringFormatterValidator(Validator):
         "Format variable '{variable_name}' does not exist. Available variables: {available_fields}"
     )
 
-    __fields_validator = re.compile(r"{([a-z][a-z0-9_]+?)}")
-
     __max_format_recursion = 3
 
     def __validate_and_get_format_variables(self) -> List[str]:
@@ -70,14 +101,11 @@ class StringFormatterValidator(Validator):
                 exception_class=StringFormattingException,
             )
 
-        format_variables: List[str] = list(
+        format_variables: List[str] = list(re.findall(_fields_validator, self.format_string))
-            re.findall(StringFormatterValidator.__fields_validator, self.format_string)
-        )
 
         if len(format_variables) != open_bracket_count:
             raise self._validation_exception(
-                "{variable_names} must start with a lowercase letter, should only contain lowercase"
+                error_message=_fields_validator_exception_message,
-                "letters, numbers, underscores, and have a single open and close bracket.",
                 exception_class=StringFormattingException,
             )
 

tests/e2e/plugins/test_regex.py

@@ -27,32 +27,44 @@ def regex_subscription_dict(output_directory):
                         "should not cap (.+) - (.+)",
                         ".*\\[(.+) - (Feb.+)]",  # should filter out march video
                     ],
+                    "capture_group_names": ["title_type", "title_date"],
+                },
+                "description": {
+                    "match": [".*http:\\/\\/(.+).com.*"],
+                    "capture_group_names": ["description_website"],
                 },
-                "description": {"match": [".*http:\\/\\/(.+).com.*"]},
                 "upload_date_standardized": {
                     "match": ["([0-9]+)-([0-9]+)-27"],
+                    "capture_group_names": [
+                        "upload_captured_year",
+                        "upload_captured_month",
+                    ],
                     "defaults": [
                         "First",
                         "Second containing {in_regex_default}",
                     ],
                 },
-                "artist": {"match": ["Never (.*) capture"], "defaults": ["Always default"]},
+                "artist": {
+                    "match": ["Never (.*) capture"],
+                    "capture_group_names": ["always_default"],
+                    "defaults": ["Always default"],
+                },
             },
         },
         "nfo_tags": {
             "tags": {
-                "title_cap_1": "{title_capture_1}",
+                "title_cap_1": "{title_type}",
-                "title_cap_2": "{title_capture_2}",
+                "title_cap_2": "{title_date}",
-                "desc_cap": "{description_capture_1}",
+                "desc_cap": "{description_website}",
-                "upload_date_both_caps": "{upload_date_standardized_capture_1} and {upload_date_standardized_capture_2}",
+                "upload_date_both_caps": "{upload_captured_year} and {upload_captured_month}",
-                "artist_cap_always_default": "{artist_capture_1}",
+                "artist_cap_always_default": "{always_default}",
-                "override_with_capture_variable": "{contains_regex_default}"
+                "override_with_capture_variable": "{contains_regex_default}",
             }
         },
         "overrides": {
             "in_regex_default": "in regex default",
-            "contains_regex_default": "contains {title_capture_1}"
+            "contains_regex_default": "contains {title_type}",
-        }
+        },
     }
 
 

tests/e2e/resources/transaction_log_summaries/plugins/test_regex.txt

@@ -10,10 +10,11 @@ Project Zombie - Jesse's Minecraft Server [Trailer - Feb.1].nfo
       artist: Project Zombie
       artist_cap_always_default: Always default
       desc_cap: www.jesseminecraft.webs
+      override_with_capture_variable: contains Trailer
       title: Jesse's Minecraft Server [Trailer - Feb.1]
       title_cap_1: Trailer
       title_cap_2: Feb.1
-      upload_date_both_caps: First and Second
+      upload_date_both_caps: First and Second containing in regex default
       year: 2011
 Project Zombie - Jesse's Minecraft Server [Trailer - Feb.27]-thumb.jpg
 Project Zombie - Jesse's Minecraft Server [Trailer - Feb.27].mp4
@@ -24,6 +25,7 @@ Project Zombie - Jesse's Minecraft Server [Trailer - Feb.27].nfo
       artist: Project Zombie
       artist_cap_always_default: Always default
       desc_cap: jesseminecraft.webs
+      override_with_capture_variable: contains Trailer
       title: Jesse's Minecraft Server [Trailer - Feb.27]
       title_cap_1: Trailer
       title_cap_2: Feb.27