[FEATURE] Regex capture and filtering on source variables plugin (#100)

* actually working

* changed plugin format. Need unit tests, documentation, update e2e to filter

* refactor plugin name, matching now working

* description work in progress, need more intricate testing

* tests looks good, regenerate output

* e2e tested, need more failure tests, no dupe variable names, add docs

* unit tests passing again

* more fail tests

* updated with name, default, better docs

* sanitized vars created

docs/config.rst

@@ -192,6 +192,17 @@ nfo_output_directory
 
 -------------------------------------------------------------------------------
 
+regex
+'''''
+.. autoclass:: ytdl_sub.plugins.regex.RegexOptions()
+  :members: skip_if_match_fails
+
+.. autoclass:: ytdl_sub.plugins.regex.SourceVariableRegex()
+  :members: match, capture_group_names, capture_group_defaults
+  :member-order: bysource
+
+-------------------------------------------------------------------------------
+
 .. _subscription_yaml:
 
 subscription.yaml

src/ytdl_sub/config/preset.py

@@ -83,6 +83,10 @@ class Preset(StrictDictValidator):
     # and ensure required keys are present.
     _optional_keys = PRESET_KEYS
 
+    @property
+    def _source_variables(self) -> List[str]:
+        return self.downloader.downloader_entry_type.source_variables()
+
     def __validate_and_get_downloader(self, downloader_source: str) -> Type[Downloader]:
         return self._validate_key(key=downloader_source, validator=DownloadStrategyValidator).get(
             downloader_source=downloader_source
@@ -141,6 +145,9 @@ class Preset(StrictDictValidator):
 
             plugin = PluginMapping.get(plugin=key)
             plugin_options = self._validate_key(key=key, validator=plugin.plugin_options_type)
+            plugin_options.validate_with_variables(
+                source_variables=self._source_variables, override_variables=self.overrides.keys
+            )
 
             plugins.append((plugin, plugin_options))
 
@@ -156,12 +163,20 @@ class Preset(StrictDictValidator):
         # If the formatter supports source variables, set the formatter variables to include
         # both source and override variables
         if not isinstance(formatter_validator, OverridesStringFormatterValidator):
-            source_variables = {
+            source_variables = {source_var: "dummy_string" for source_var in self._source_variables}
-                source_var: "dummy_string"
-                for source_var in self.downloader.downloader_entry_type.source_variables()
-            }
             variable_dict = dict(source_variables, **variable_dict)
 
+        # For all plugins, add in any extra added source variables
+        for _, plugin_options in self.plugins:
+            added_plugin_variables = {
+                source_var: "dummy_string" for source_var in plugin_options.added_source_variables()
+            }
+            # sanity check plugin variables do not override source variables
+            expected_len = len(variable_dict) + len(added_plugin_variables)
+            variable_dict = dict(variable_dict, **added_plugin_variables)
+
+            assert len(variable_dict) == expected_len, "plugin variables overwrote source variables"
+
         _ = formatter_validator.apply_formatter(variable_dict=variable_dict)
 
     def __recursive_preset_validate(

src/ytdl_sub/config/preset_class_mappings.py

@@ -13,6 +13,7 @@ from ytdl_sub.plugins.music_tags import MusicTagsPlugin
 from ytdl_sub.plugins.nfo_tags import NfoTagsPlugin
 from ytdl_sub.plugins.output_directory_nfo_tags import OutputDirectoryNfoTagsPlugin
 from ytdl_sub.plugins.plugin import Plugin
+from ytdl_sub.plugins.regex import RegexPlugin
 
 
 class DownloadStrategyMapping:
@@ -107,6 +108,7 @@ class PluginMapping:
         "music_tags": MusicTagsPlugin,
         "nfo_tags": NfoTagsPlugin,
         "output_directory_nfo_tags": OutputDirectoryNfoTagsPlugin,
+        "regex": RegexPlugin,
     }
 
     @classmethod

src/ytdl_sub/entries/base_entry.py

@@ -22,6 +22,8 @@ class BaseEntry(ABC):
         self._working_directory = working_directory
         self._kwargs = entry_dict
 
+        self._additional_variables: Dict[str, str] = {}
+
     def kwargs_contains(self, key: str) -> bool:
         """Returns whether internal kwargs contains the specified key"""
         return key in self._kwargs
@@ -39,3 +41,28 @@ class BaseEntry(ABC):
         The working directory
         """
         return self._working_directory
+
+    def add_variables(self, variables_to_add: Dict[str, str]) -> "BaseEntry":
+        """
+        Parameters
+        ----------
+        variables_to_add
+            Variables to add to this entry
+
+        Returns
+        -------
+        self
+
+        Raises
+        ------
+        ValueError
+            If a variable trying to be added already exists as a source variable
+        """
+        for variable_name in variables_to_add.keys():
+            if self.kwargs_contains(variable_name):
+                raise ValueError(
+                    f"Cannot add variable '{variable_name}': already exists in the kwargs"
+                )
+
+        self._additional_variables = dict(self._additional_variables, **variables_to_add)
+        return self

src/ytdl_sub/entries/variables/entry_variables.py

@@ -37,6 +37,14 @@ class SourceVariables:
         """
         return self.kwargs("extractor")
 
+    def _added_variables(self: BaseEntry) -> Dict[str, str]:
+        """
+        Returns
+        -------
+        Dict of variables added to this entry
+        """
+        return self._additional_variables
+
     @classmethod
     def source_variables(cls) -> List[str]:
         """
@@ -54,7 +62,10 @@ class SourceVariables:
         -------
         Dictionary containing all variables
         """
-        return {source_var: getattr(self, source_var) for source_var in self.source_variables()}
+        source_variable_dict = {
+            source_var: getattr(self, source_var) for source_var in self.source_variables()
+        }
+        return dict(source_variable_dict, **self._added_variables())
 
 
 class EntryVariables(SourceVariables):

src/ytdl_sub/plugins/plugin.py

@@ -1,5 +1,6 @@
 from abc import ABC
 from typing import Generic
+from typing import List
 from typing import Optional
 from typing import Type
 from typing import TypeVar
@@ -19,6 +20,35 @@ class PluginOptions(StrictDictValidator):
     Class that defines the parameters to a plugin
     """
 
+    # pylint: disable=no-self-use
+    def added_source_variables(self) -> List[str]:
+        """
+        If the plugin adds source variables, list them here.
+
+        Returns
+        -------
+        List of added source variables this plugin creates
+        """
+        return []
+
+    # pylint: disable=unused-argument
+    def validate_with_variables(
+        self, source_variables: List[str], override_variables: List[str]
+    ) -> None:
+        """
+        Optional validation after init with the session's source and override variables.
+
+        Parameters
+        ----------
+        source_variables
+            Available source variables when running the plugin
+        override_variables
+            Available override variables when running the plugin
+        """
+        return None
+
+    # pylint: enable=no-self-use,unused-argument
+
 
 PluginOptionsT = TypeVar("PluginOptionsT", bound=PluginOptions)
 
@@ -43,6 +73,7 @@ class Plugin(DownloadArchiver, Generic[PluginOptionsT], ABC):
         # TODO pass yaml snake case name in the class somewhere, and use it for the logger
         self._logger = Logger.get(self.__class__.__name__)
 
+    # pylint: disable=no-self-use
     def modify_entry(self, entry: Entry) -> Optional[Entry]:
         """
         For each entry downloaded, modify the entry in some way before sending it to
@@ -57,6 +88,9 @@ class Plugin(DownloadArchiver, Generic[PluginOptionsT], ABC):
         -------
         The entry or None, indicating not to move it to the output directory
         """
+        return entry
+
+    # pylint: enable=no-self-use
 
     def post_process_entry(self, entry: Entry) -> Optional[FileMetadata]:
         """

src/ytdl_sub/plugins/regex.py

@@ -0,0 +1,311 @@
+from typing import Dict
+from typing import List
+from typing import Optional
+
+from yt_dlp.utils import sanitize_filename
+
+from ytdl_sub.entries.entry import Entry
+from ytdl_sub.plugins.plugin import Plugin
+from ytdl_sub.plugins.plugin import PluginOptions
+from ytdl_sub.utils.exceptions import RegexNoMatchException
+from ytdl_sub.utils.logger import Logger
+from ytdl_sub.validators.regex_validator import RegexListValidator
+from ytdl_sub.validators.source_variable_validator import SourceVariableNameListValidator
+from ytdl_sub.validators.strict_dict_validator import StrictDictValidator
+from ytdl_sub.validators.string_formatter_validators import ListFormatterValidator
+from ytdl_sub.validators.string_formatter_validators import StringFormatterValidator
+from ytdl_sub.validators.validators import BoolValidator
+
+logger = Logger.get(name="regex")
+
+
+class SourceVariableRegex(StrictDictValidator):
+
+    _required_keys = {"match"}
+    _optional_keys = {"capture_group_defaults", "capture_group_names"}
+
+    def __init__(self, name, value):
+        super().__init__(name, value)
+        self._match = self._validate_key(key="match", validator=RegexListValidator)
+        self._capture_group_defaults = self._validate_key_if_present(
+            key="capture_group_defaults", validator=ListFormatterValidator
+        )
+        self._capture_group_names = self._validate_key_if_present(
+            key="capture_group_names", validator=SourceVariableNameListValidator, default=[]
+        )
+
+        # If defaults are to be used, ensure there are the same number of defaults as there are
+        # capture groups
+        if self._capture_group_defaults is not None and self._match.num_capture_groups != len(
+            self._capture_group_defaults.list
+        ):
+            raise self._validation_exception(
+                f"number of defaults must match number of capture groups, "
+                f"{len(self._capture_group_defaults.list)} != {self._match.num_capture_groups}"
+            )
+
+        # If there are capture groups, ensure there are capture group names
+        if len(self._capture_group_names.list) != self._match.num_capture_groups:
+            raise self._validation_exception(
+                f"number of capture group names must match number of capture groups, "
+                f"{len(self._capture_group_names.list)} != {self._match.num_capture_groups}"
+            )
+
+    @property
+    def match(self) -> RegexListValidator:
+        """
+        Required. List of regex strings to try to match against a source variable. Each regex
+        string must have the same number of capture groups.
+        """
+        return self._match
+
+    @property
+    def capture_group_names(self) -> Optional[List[str]]:
+        """
+        Optional (only when no capture groups are in the regex string). List of names to store the
+        capture group values to. These and ``_sanitized`` versions will be available to use as
+        source variables. The list's length must be equal to the number of match capture groups.
+        """
+        return [validator.value for validator in self._capture_group_names.list]
+
+    @property
+    def capture_group_defaults(self) -> Optional[List[StringFormatterValidator]]:
+        """
+        Optional. List of string format validators to use for capture group defaults if a
+        source variable cannot be matched. The list's length must be equal to the number of match
+        capture groups.
+        """
+        return self._capture_group_defaults.list if self.has_defaults else None
+
+    @property
+    def has_defaults(self) -> bool:
+        """
+        Returns
+        -------
+        True if a validation exception should be raised if not captured. False otherwise.
+        """
+        return self._capture_group_defaults is not None
+
+
+class FromSourceVariablesRegex(StrictDictValidator):
+
+    _optional_keys = Entry.source_variables()
+    _allow_extra_keys = True
+
+    def __init__(self, name, value):
+        super().__init__(name, value)
+        self.source_variable_capture_dict: Dict[str, SourceVariableRegex] = {
+            key: self._validate_key(key=key, validator=SourceVariableRegex) for key in self._keys
+        }
+
+
+class RegexOptions(PluginOptions):
+    r"""
+    Performs regex matching on an entry's source variables. Regex can be used to filter entries
+    from proceeding with download or capture groups to create new source variables.
+
+    Usage:
+
+    .. code-block:: yaml
+
+       presets:
+         my_example_preset:
+           regex:
+             # By default, if any match fails and has no defaults, the entry will be skipped.
+             # If set to False, ytdl-sub will error and stop all downloads from proceeding.
+             skip_if_match_fails: True
+
+             from:
+               # For each entry's `title` value...
+               title:
+                 # Perform this regex match on it to act as a filter.
+                 # This will only download videos with "Official Video" in it.
+                 match: '\[Official Video\]'
+
+               # For each entry's `description` value...
+               description:
+                 # Match with capture groups and defaults.
+                 # This tries to scrape a date from the description and produce new source variables
+                 match: "([0-9]{4})-([0-9]{2})-([0-9]{2})"
+
+                 # Each capture group creates these new source variables, respectively, as well
+                 # a sanitized version, i.e. `captured_upload_year_sanitized`
+                 capture_group_names:
+                  - "captured_upload_year"
+                  - "captured_upload_month"
+                  - "captured_upload_day"
+
+                 # And if the string does not match, use these as respective default values for the
+                 # new source variables.
+                 capture_group_defaults:
+                   - "{upload_year}"
+                   - "{upload_month}"
+                   - "{upload_day}"
+    """
+
+    _required_keys = {"from"}
+    _optional_keys = {"skip_if_match_fails"}
+
+    def __init__(self, name, value):
+        super().__init__(name, value)
+        self._from = self._validate_key(key="from", validator=FromSourceVariablesRegex)
+        self._skip_if_match_fails: bool = self._validate_key_if_present(
+            key="skip_if_match_fails", validator=BoolValidator, default=True
+        ).value
+
+    @property
+    def skip_if_match_fails(self) -> Optional[bool]:
+        """
+        Defaults to True. If True, when any match fails and has no defaults, the entry will be
+        skipped. If False, ytdl-sub will error and all downloads will not proceed.
+        """
+        return self._skip_if_match_fails
+
+    def validate_with_variables(
+        self, source_variables: List[str], override_variables: List[str]
+    ) -> None:
+        """
+        Ensures each source variable capture group is valid
+
+        Parameters
+        ----------
+        source_variables
+            Available source variables when running the plugin
+        override_variables
+            Available override variables when running the plugin
+        """
+        for key, regex_options in self.source_variable_capture_dict.items():
+            # Ensure each variable getting captured is a source variable
+            if key not in source_variables:
+                raise self._validation_exception(
+                    f"cannot regex capture '{key}' because it is not a source variable"
+                )
+
+            # Ensure the capture group names are not existing source/override variables
+            for capture_group_name in regex_options.capture_group_names:
+                if capture_group_name in source_variables:
+                    raise self._validation_exception(
+                        f"'{capture_group_name}' cannot be used as a capture group name because it "
+                        f"is a source variable"
+                    )
+                if capture_group_name in override_variables:
+                    raise self._validation_exception(
+                        f"'{capture_group_name}' cannot be used as a capture group name because it "
+                        f"is an override variable"
+                    )
+
+    @property
+    def source_variable_capture_dict(self) -> Dict[str, SourceVariableRegex]:
+        """
+        Returns
+        -------
+        Dict of { source variable: capture options }
+        """
+        return self._from.source_variable_capture_dict
+
+    def added_source_variables(self) -> List[str]:
+        """
+        Returns
+        -------
+        List of new source variables created via regex capture
+        """
+        added_source_vars: List[str] = []
+        for regex_options in self.source_variable_capture_dict.values():
+            added_source_vars.extend(regex_options.capture_group_names)
+            added_source_vars.extend(
+                f"{capture_group_name}_sanitized"
+                for capture_group_name in regex_options.capture_group_names
+            )
+
+        return added_source_vars
+
+
+class RegexPlugin(Plugin[RegexOptions]):
+    plugin_options_type = RegexOptions
+
+    def modify_entry(self, entry: Entry) -> Optional[Entry]:
+        """
+        Parameters
+        ----------
+        entry
+            Entry to add source variables to
+
+        Returns
+        -------
+        Entry with regex capture variables added to its source variables
+
+        Raises
+        ------
+        ValidationException
+            If no capture and no defaults
+        """
+        entry_variable_dict = entry.to_dict()
+
+        # Iterate each source var to capture and add to the entry
+        for source_var, regex_options in self.plugin_options.source_variable_capture_dict.items():
+            maybe_capture = regex_options.match.match_any(input_str=entry_variable_dict[source_var])
+
+            # If no capture
+            if maybe_capture is None:
+                # and no defaults
+                if not regex_options.has_defaults:
+                    # Skip the entry if toggled
+                    if self.plugin_options.skip_if_match_fails:
+                        logger.info(
+                            "Regex failed to match '%s' from '%s', skipping.",
+                            source_var,
+                            entry.title,
+                        )
+                        return None
+
+                    # Otherwise, error
+                    raise RegexNoMatchException(
+                        f"Regex failed to match '{source_var}' from '{entry.title}'"
+                    )
+
+                # otherwise, use defaults (apply them using the original entry source dict)
+                source_variables_and_overrides_dict = dict(
+                    entry_variable_dict, **self.overrides.dict_with_format_strings
+                )
+
+                # add both the default...
+                entry.add_variables(
+                    variables_to_add={
+                        regex_options.capture_group_names[i]: default.apply_formatter(
+                            variable_dict=source_variables_and_overrides_dict
+                        )
+                        for i, default in enumerate(regex_options.capture_group_defaults)
+                    },
+                )
+                # and sanitized default
+                entry.add_variables(
+                    variables_to_add={
+                        f"{regex_options.capture_group_names[i]}_sanitized": sanitize_filename(
+                            default.apply_formatter(
+                                variable_dict=source_variables_and_overrides_dict
+                            )
+                        )
+                        for i, default in enumerate(regex_options.capture_group_defaults)
+                    },
+                )
+            # There is a capture, add the source variables to the entry as
+            # {source_var}_capture_1, {source_var}_capture_2, ...
+            else:
+                # Add the value...
+                entry.add_variables(
+                    variables_to_add={
+                        regex_options.capture_group_names[i]: capture
+                        for i, capture in enumerate(maybe_capture)
+                    },
+                )
+                # And the sanitized value
+                entry.add_variables(
+                    variables_to_add={
+                        f"{regex_options.capture_group_names[i]}_sanitized": sanitize_filename(
+                            capture
+                        )
+                        for i, capture in enumerate(maybe_capture)
+                    },
+                )
+
+        return entry

src/ytdl_sub/subscriptions/subscription.py

@@ -3,6 +3,7 @@ import copy
 import os
 import shutil
 from pathlib import Path
+from typing import Dict
 from typing import List
 from typing import Optional
 from typing import Tuple
@@ -271,6 +272,17 @@ class Subscription:
                 if isinstance(entry, tuple):
                     entry, entry_metadata = entry
 
+                # First, modify the entry with all plugins
+                for plugin in plugins:
+                    # Break out of this plugin loop if entry is None, it is indicated to not DL it
+                    if (entry := plugin.modify_entry(entry)) is None:
+                        break
+
+                # If entry is None from the broken out loop, continue over the other entries
+                if entry is None:
+                    continue
+
+                # Then, post-process the entry with all plugins
                 for plugin in plugins:
                     optional_plugin_entry_metadata = plugin.post_process_entry(entry)
                     if optional_plugin_entry_metadata:
@@ -307,3 +319,30 @@ class Subscription:
             preset_options=preset,
             config_options=config.config_options,
         )
+
+    @classmethod
+    def from_dict(cls, config: ConfigFile, preset_name: str, preset_dict: Dict) -> "Subscription":
+        """
+        Creates a subscription from a preset dict
+
+        Parameters
+        ----------
+        config:
+            Validated instance of the config
+        preset_name:
+            Name of the preset
+        preset_dict:
+            The preset config in dict format
+
+        Returns
+        -------
+        Initialized subscription
+        """
+        return cls.from_preset(
+            preset=Preset.from_dict(
+                config=config,
+                preset_name=preset_name,
+                preset_dict=preset_dict,
+            ),
+            config=config,
+        )

src/ytdl_sub/utils/exceptions.py

@@ -10,6 +10,10 @@ class StringFormattingVariableNotFoundException(StringFormattingException):
     """Tried to format a string but the variable was not found"""
 
 
+class InvalidVariableNameException(ValidationException):
+    """A user defined variable name is invalid"""
+
+
 class DownloadArchiveException(ValueError):
     """Any user or file errors caused by download archive or mapping files"""
 
@@ -20,3 +24,7 @@ class FileNotFoundException(ValidationException):
 
 class InvalidYamlException(ValidationException):
     """User yaml that is invalid"""
+
+
+class RegexNoMatchException(ValidationException):
+    """Regex failed to match during download"""

src/ytdl_sub/validators/regex_validator.py

@@ -28,33 +28,20 @@ class RegexValidator(StringValidator):
         """
         return self._compiled_regex.groups
 
-    def is_match(self, input_str: str) -> bool:
+    def match(self, input_str: str) -> Optional[List[str]]:
         """
         Parameters
         ----------
         input_str
-            String to match against the regex
+            String to regex match
 
         Returns
         -------
-        True if input_str matches. False otherwise.
+        List of captures. If the regex has no capture groups, then the list will be emtpy.
-        """
+        None is returned if the input_str failed to match
-        return self._compiled_regex.search(input_str) is not None
-
-    def capture(self, input_str: str) -> Optional[List[str]]:
-        """
-        Parameters
-        ----------
-        input_str
-            String to try to regex capture from
-
-        Returns
-        -------
-        List of captures (will always be >= 1). None if there are no captures.
         """
         if match := self._compiled_regex.search(input_str):
-            if len(to_return := list(match.groups())) > 0:
+            return list(match.groups())
-                return to_return
         return None
 
 
@@ -70,20 +57,18 @@ class RegexListValidator(ListValidator[RegexValidator]):
                 "each regex in a list must have the same number of capture groups"
             )
 
-    def matches_any(self, input_str: str) -> bool:
+        self._num_capture_groups = self._list[0].num_capture_groups
-        """
-        Parameters
-        ----------
-        input_str
-            String to match against any regexes in the list
 
+    @property
+    def num_capture_groups(self) -> int:
+        """
         Returns
         -------
-        True if at least one matches. False if none match
+        Number of capture groups. All regexes in the list will have the same number.
         """
-        return any(reg.is_match(input_str) for reg in self._list)
+        return self._num_capture_groups
 
-    def capture_any(self, input_str: str) -> Optional[List[str]]:
+    def match_any(self, input_str: str) -> Optional[List[str]]:
         """
         Parameters
         ----------
@@ -92,10 +77,10 @@ class RegexListValidator(ListValidator[RegexValidator]):
 
         Returns
         -------
-        List of captures (will always be >= 1) on the first regex that matches. None if
+        List of captures on the first regex that matches. If the regex has no capture groups, then
-        no regexes match.
+        the list will be emtpy. None is returned if the input_str failed to match
         """
         for reg in self._list:
-            if maybe_capture := reg.capture(input_str):
+            if (maybe_capture := reg.match(input_str)) is not None:
                 return maybe_capture
         return None

src/ytdl_sub/validators/source_variable_validator.py

@@ -0,0 +1,20 @@
+from ytdl_sub.utils.exceptions import InvalidVariableNameException
+from ytdl_sub.validators.string_formatter_validators import is_valid_source_variable_name
+from ytdl_sub.validators.validators import ListValidator
+from ytdl_sub.validators.validators import StringValidator
+
+
+class SourceVariableNameValidator(StringValidator):
+    _expected_value_type_name = "source variable name"
+
+    def __init__(self, name, value):
+        super().__init__(name, value)
+        try:
+            _ = is_valid_source_variable_name(self.value, raise_exception=True)
+        except InvalidVariableNameException as exc:
+            raise self._validation_exception(exc) from exc
+
+
+class SourceVariableNameListValidator(ListValidator[SourceVariableNameValidator]):
+    _inner_list_type = SourceVariableNameValidator
+    _expected_value_type_name = "source variable name list"

src/ytdl_sub/validators/string_formatter_validators.py

@@ -5,11 +5,45 @@ from typing import Dict
 from typing import List
 from typing import final
 
+from ytdl_sub.utils.exceptions import InvalidVariableNameException
 from ytdl_sub.utils.exceptions import StringFormattingException
 from ytdl_sub.utils.exceptions import StringFormattingVariableNotFoundException
+from ytdl_sub.validators.validators import ListValidator
 from ytdl_sub.validators.validators import LiteralDictValidator
 from ytdl_sub.validators.validators import Validator
 
+_fields_validator = re.compile(r"{([a-z][a-z0-9_]+?)}")
+
+_fields_validator_exception_message: str = (
+    "{variable_names} must start with a lowercase letter, should only contain lowercase letters, "
+    "numbers, underscores, and have a single open and close bracket."
+)
+
+
+def is_valid_source_variable_name(input_str: str, raise_exception: bool = False) -> bool:
+    """
+    Parameters
+    ----------
+    input_str
+        String to see if it can be a source variable
+    raise_exception
+        Raise InvalidVariableNameException False.
+
+    Returns
+    -------
+    True if it is. False otherwise.
+
+    Raises
+    ------
+    InvalidVariableNameException
+        If raise_exception and output is False
+    """
+    # Add brackets around it to pretend its a StringFormatter, see if it captures
+    is_source_variable_name = len(re.findall(_fields_validator, f"{{{input_str}}}")) > 0
+    if not is_source_variable_name and raise_exception:
+        raise InvalidVariableNameException(_fields_validator_exception_message)
+    return is_source_variable_name
+
 
 class StringFormatterValidator(Validator):
     """
@@ -43,8 +77,6 @@ class StringFormatterValidator(Validator):
         "Format variable '{variable_name}' does not exist. Available variables: {available_fields}"
     )
 
-    __fields_validator = re.compile(r"{([a-z_]+?)}")
-
     __max_format_recursion = 3
 
     def __validate_and_get_format_variables(self) -> List[str]:
@@ -69,14 +101,11 @@ class StringFormatterValidator(Validator):
                 exception_class=StringFormattingException,
             )
 
-        format_variables: List[str] = list(
+        format_variables: List[str] = list(re.findall(_fields_validator, self.format_string))
-            re.findall(StringFormatterValidator.__fields_validator, self.format_string)
-        )
 
         if len(format_variables) != open_bracket_count:
             raise self._validation_exception(
-                "{variable_names} should only contain "
+                error_message=_fields_validator_exception_message,
-                "lowercase letters and underscores with a single open and close bracket.",
                 exception_class=StringFormattingException,
             )
 
@@ -177,6 +206,10 @@ class OverridesStringFormatterValidator(StringFormatterValidator):
 # pylint: enable=line-too-long
 
 
+class ListFormatterValidator(ListValidator[StringFormatterValidator]):
+    _inner_list_type = StringFormatterValidator
+
+
 class DictFormatterValidator(LiteralDictValidator):
     """
     A dict made up of

src/ytdl_sub/validators/validators.py

@@ -112,6 +112,15 @@ class ListValidator(Validator, ABC, Generic[T]):
             for i, val in enumerate(self._value)
         ]
 
+    @property
+    def list(self) -> List[T]:
+        """
+        Returns
+        -------
+        The list
+        """
+        return self._list
+
 
 class DictValidator(Validator):
     """

tests/e2e/conftest.py

@@ -2,8 +2,20 @@ import tempfile
 
 import pytest
 
+from ytdl_sub.config.config_file import ConfigFile
+
 
 @pytest.fixture()
 def output_directory():
     with tempfile.TemporaryDirectory() as temp_dir:
         yield temp_dir
+
+
+@pytest.fixture()
+def music_video_config():
+    return ConfigFile.from_file_path(config_path="examples/kodi_music_videos_config.yaml")
+
+
+@pytest.fixture()
+def channel_as_tv_show_config():
+    return ConfigFile.from_file_path(config_path="examples/kodi_tv_shows_config.yaml")

tests/e2e/plugins/test_regex.py

@@ -0,0 +1,208 @@
+import copy
+import re
+
+import pytest
+from e2e.expected_transaction_log import assert_transaction_log_matches
+
+from ytdl_sub.config.preset import Preset
+from ytdl_sub.subscriptions.subscription import Subscription
+from ytdl_sub.utils.exceptions import RegexNoMatchException
+from ytdl_sub.utils.exceptions import ValidationException
+
+
+@pytest.fixture
+def regex_subscription_dict(output_directory):
+    return {
+        "preset": "yt_music_video_playlist",
+        "youtube": {"playlist_url": "https://youtube.com/playlist?list=PL5BC0FC26BECA5A35"},
+        # override the output directory with our fixture-generated dir
+        "output_options": {"output_directory": output_directory},
+        # download the worst format so it is fast
+        "ytdl_options": {
+            "format": "best[height<=480]",
+        },
+        "regex": {
+            # tests that skip_if_match_fails defaults to True
+            "from": {
+                "title": {
+                    "match": [
+                        "should not cap (.+) - (.+)",
+                        ".*\\[(.+) - (Feb.+)]",  # should filter out march video
+                    ],
+                    "capture_group_names": ["title_type", "title_date"],
+                },
+                "description": {
+                    "match": [".*http:\\/\\/(.+).com.*"],
+                    "capture_group_names": ["description_website"],
+                },
+                "upload_date_standardized": {
+                    "match": ["([0-9]+)-([0-9]+)-27"],
+                    "capture_group_names": [
+                        "upload_captured_year",
+                        "upload_captured_month",
+                    ],
+                    "capture_group_defaults": [
+                        "First",
+                        "Second containing {in_regex_default}",
+                    ],
+                },
+                "artist": {
+                    "match": ["Never (.*) capture"],
+                    "capture_group_names": ["always_default"],
+                    "capture_group_defaults": ["Always default"],
+                },
+            },
+        },
+        "nfo_tags": {
+            "tags": {
+                "title_cap_1": "{title_type}",
+                "title_cap_1_sanitized": "{title_type_sanitized}",
+                "title_cap_2": "{title_date}",
+                "desc_cap": "{description_website}",
+                "upload_date_both_caps": "{upload_captured_year} and {upload_captured_month}",
+                "artist_cap_always_default": "{always_default}",
+                "artist_cap_always_default_sanitized": "{always_default_sanitized}",
+                "override_with_capture_variable": "{contains_regex_default}",
+                "override_with_capture_variable_sanitized": "{contains_regex_sanitized_default}",
+            }
+        },
+        "overrides": {
+            "in_regex_default": "in regex default",
+            "contains_regex_default": "contains {title_type}",
+            "contains_regex_sanitized_default": "contains {title_type_sanitized}",
+        },
+    }
+
+
+@pytest.fixture
+def regex_subscription_dict_no_match_fails(regex_subscription_dict):
+    regex_subscription_dict["regex"]["skip_if_match_fails"] = False
+    return regex_subscription_dict
+
+
+@pytest.fixture
+def playlist_subscription(music_video_config, regex_subscription_dict):
+    playlist_preset = Preset.from_dict(
+        config=music_video_config,
+        preset_name="regex_capture_playlist_test",
+        preset_dict=regex_subscription_dict,
+    )
+
+    return Subscription.from_preset(
+        preset=playlist_preset,
+        config=music_video_config,
+    )
+
+
+@pytest.fixture
+def playlist_subscription_no_match_fails(
+    music_video_config, regex_subscription_dict_no_match_fails
+):
+    playlist_preset = Preset.from_dict(
+        config=music_video_config,
+        preset_name="regex_capture_playlist_test",
+        preset_dict=regex_subscription_dict_no_match_fails,
+    )
+
+    return Subscription.from_preset(
+        preset=playlist_preset,
+        config=music_video_config,
+    )
+
+
+class TestRegex:
+    def test_regex_success(self, playlist_subscription, output_directory):
+        # Only dry run is needed to see if capture variables are created
+        transaction_log = playlist_subscription.download(dry_run=True)
+        assert_transaction_log_matches(
+            output_directory=output_directory,
+            transaction_log=transaction_log,
+            transaction_log_summary_file_name="plugins/test_regex.txt",
+        )
+
+    def test_regex_fails_no_match(self, playlist_subscription_no_match_fails, output_directory):
+        with pytest.raises(
+            RegexNoMatchException,
+            match=re.escape(
+                "Regex failed to match 'title' from 'Jesse's Minecraft Server [Trailer - Mar.21]'"
+            ),
+        ):
+            _ = playlist_subscription_no_match_fails.download(dry_run=True)
+
+    def test_regex_fails_capture_group_is_source_variable(
+        self, regex_subscription_dict, music_video_config
+    ):
+        regex_subscription_dict["regex"]["from"]["title"]["capture_group_names"][0] = "uid"
+        with pytest.raises(
+            ValidationException,
+            match=re.escape(
+                "'uid' cannot be used as a capture group name because it is a source variable"
+            ),
+        ):
+            _ = Subscription.from_dict(
+                config=music_video_config,
+                preset_name="test_regex_fails_capture_group_is_source_variable",
+                preset_dict=regex_subscription_dict,
+            )
+
+    def test_regex_fails_capture_group_is_override_variable(
+        self, regex_subscription_dict, music_video_config
+    ):
+        regex_subscription_dict["regex"]["from"]["title"]["capture_group_names"][
+            0
+        ] = "in_regex_default"
+        with pytest.raises(
+            ValidationException,
+            match=re.escape(
+                "'in_regex_default' cannot be used as a capture group name because it is an override variable"
+            ),
+        ):
+            _ = Subscription.from_dict(
+                config=music_video_config,
+                preset_name="test_regex_fails_capture_group_is_override_variable",
+                preset_dict=regex_subscription_dict,
+            )
+
+    def test_regex_fails_source_variable_does_not_exist(
+        self, regex_subscription_dict, music_video_config
+    ):
+        regex_subscription_dict["regex"]["from"]["dne"] = copy.deepcopy(
+            regex_subscription_dict["regex"]["from"]["title"]
+        )
+        with pytest.raises(
+            ValidationException,
+            match=re.escape("cannot regex capture 'dne' because it is not a source variable"),
+        ):
+            _ = Subscription.from_dict(
+                config=music_video_config,
+                preset_name="test_regex_fails_source_variable_does_not_exist",
+                preset_dict=regex_subscription_dict,
+            )
+
+    def test_regex_fails_unequal_defaults(self, regex_subscription_dict, music_video_config):
+        regex_subscription_dict["regex"]["from"]["title"]["capture_group_defaults"] = ["1 != 2"]
+        with pytest.raises(
+            ValidationException,
+            match=re.escape("number of defaults must match number of capture groups, 1 != 2"),
+        ):
+            _ = Subscription.from_dict(
+                config=music_video_config,
+                preset_name="test_regex_fails_unequal_defaults",
+                preset_dict=regex_subscription_dict,
+            )
+
+    def test_regex_fails_unequal_capture_group_names(
+        self, regex_subscription_dict, music_video_config
+    ):
+        regex_subscription_dict["regex"]["from"]["title"]["capture_group_names"].append("unequal")
+        with pytest.raises(
+            ValidationException,
+            match=re.escape(
+                "number of capture group names must match number of capture groups, 3 != 2"
+            ),
+        ):
+            _ = Subscription.from_dict(
+                config=music_video_config,
+                preset_name="test_regex_fails_unequal_capture_group_names",
+                preset_dict=regex_subscription_dict,
+            )

tests/e2e/resources/transaction_log_summaries/plugins/test_regex.txt

@@ -0,0 +1,39 @@
+Files created in '{output_directory}'
+----------------------------------------
+.ytdl-sub-regex_capture_playlist_test-download-archive.json
+Project Zombie - Jesse's Minecraft Server [Trailer - Feb.1]-thumb.jpg
+Project Zombie - Jesse's Minecraft Server [Trailer - Feb.1].mp4
+Project Zombie - Jesse's Minecraft Server [Trailer - Feb.1].nfo
+  NFO tags:
+    musicvideo:
+      album: Music Videos
+      artist: Project Zombie
+      artist_cap_always_default: Always default
+      artist_cap_always_default_sanitized: Always default
+      desc_cap: www.jesseminecraft.webs
+      override_with_capture_variable: contains Trailer
+      override_with_capture_variable_sanitized: contains Trailer
+      title: Jesse's Minecraft Server [Trailer - Feb.1]
+      title_cap_1: Trailer
+      title_cap_1_sanitized: Trailer
+      title_cap_2: Feb.1
+      upload_date_both_caps: First and Second containing in regex default
+      year: 2011
+Project Zombie - Jesse's Minecraft Server [Trailer - Feb.27]-thumb.jpg
+Project Zombie - Jesse's Minecraft Server [Trailer - Feb.27].mp4
+Project Zombie - Jesse's Minecraft Server [Trailer - Feb.27].nfo
+  NFO tags:
+    musicvideo:
+      album: Music Videos
+      artist: Project Zombie
+      artist_cap_always_default: Always default
+      artist_cap_always_default_sanitized: Always default
+      desc_cap: jesseminecraft.webs
+      override_with_capture_variable: contains Trailer
+      override_with_capture_variable_sanitized: contains Trailer
+      title: Jesse's Minecraft Server [Trailer - Feb.27]
+      title_cap_1: Trailer
+      title_cap_1_sanitized: Trailer
+      title_cap_2: Feb.27
+      upload_date_both_caps: 2011 and 02
+      year: 2011

tests/e2e/resources/transaction_log_summaries/youtube/test_playlist.txt

@@ -1,6 +1,6 @@
 Files created in '{output_directory}'
 ----------------------------------------
-.ytdl-sub-jmc-download-archive.json
+.ytdl-sub-music_video_playlist_test-download-archive.json
 JMC - Jesse's Minecraft Server [Trailer - Feb.1]-thumb.jpg
 JMC - Jesse's Minecraft Server [Trailer - Feb.1].mp4
 JMC - Jesse's Minecraft Server [Trailer - Feb.1].nfo

tests/e2e/youtube/test_playlist_as_kodi_music_videos.py

@@ -8,28 +8,12 @@ from e2e.expected_download import ExpectedDownloads
 from e2e.expected_transaction_log import assert_transaction_log_matches
 
 import ytdl_sub.downloaders.downloader
-from ytdl_sub.config.config_file import ConfigFile
 from ytdl_sub.config.preset import Preset
 from ytdl_sub.subscriptions.subscription import Subscription
 
 
 @pytest.fixture
-def config_path():
+def subscription_dict(output_directory):
-    return "examples/kodi_music_videos_config.yaml"
-
-
-@pytest.fixture
-def subscription_name():
-    return "jmc"
-
-
-@pytest.fixture
-def config(config_path):
-    return ConfigFile.from_file_path(config_path=config_path)
-
-
-@pytest.fixture
-def subscription_dict(output_directory, subscription_name):
     return {
         "preset": "yt_music_video_playlist",
         "youtube": {"playlist_url": "https://youtube.com/playlist?list=PL5BC0FC26BECA5A35"},
@@ -48,16 +32,16 @@ def subscription_dict(output_directory, subscription_name):
 
 
 @pytest.fixture
-def playlist_subscription(config, subscription_name, subscription_dict):
+def playlist_subscription(music_video_config, subscription_dict):
     playlist_preset = Preset.from_dict(
-        config=config,
+        config=music_video_config,
-        preset_name=subscription_name,
+        preset_name="music_video_playlist_test",
         preset_dict=subscription_dict,
     )
 
     return Subscription.from_preset(
         preset=playlist_preset,
-        config=config,
+        config=music_video_config,
     )
 
 
@@ -68,7 +52,7 @@ def expected_playlist_download():
     return ExpectedDownloads(
         expected_downloads=[
             # Download mapping
-            ExpectedDownloadFile(path=Path(".ytdl-sub-jmc-download-archive.json"), md5="9f785c29194a6ecfba6a6b4018763ddc"),
+            ExpectedDownloadFile(path=Path(".ytdl-sub-music_video_playlist_test-download-archive.json"), md5="9f785c29194a6ecfba6a6b4018763ddc"),
 
             # Entry files
             ExpectedDownloadFile(path=Path("JMC - Jesse's Minecraft Server [Trailer - Feb.1]-thumb.jpg"), md5="b232d253df621aa770b780c1301d364d"),
@@ -103,16 +87,16 @@ def single_video_subscription_dict(subscription_dict):
 
 
 @pytest.fixture
-def single_video_subscription(config, subscription_name, single_video_subscription_dict):
+def single_video_subscription(music_video_config, single_video_subscription_dict):
     single_video_preset = Preset.from_dict(
-        config=config,
+        config=music_video_config,
-        preset_name=subscription_name,
+        preset_name="music_video_single_video_test",
         preset_dict=single_video_subscription_dict,
     )
 
     return Subscription.from_preset(
         preset=single_video_preset,
-        config=config,
+        config=music_video_config,
     )
 
 

tests/e2e/youtube/test_split_video.py

@@ -1,24 +1,15 @@
 from pathlib import Path
 from tempfile import NamedTemporaryFile
 
-import mergedeep
 import pytest
-from conftest import assert_debug_log
 from e2e.expected_download import ExpectedDownloadFile
 from e2e.expected_download import ExpectedDownloads
 from e2e.expected_transaction_log import assert_transaction_log_matches
 
-import ytdl_sub.downloaders.downloader
-from ytdl_sub.config.config_file import ConfigFile
 from ytdl_sub.config.preset import Preset
 from ytdl_sub.subscriptions.subscription import Subscription
 
 
-@pytest.fixture
-def config_path():
-    return "examples/kodi_music_videos_config.yaml"
-
-
 @pytest.fixture
 def split_timestamps_file_path():
     timestamps = [
@@ -37,17 +28,7 @@ def split_timestamps_file_path():
 
 
 @pytest.fixture
-def subscription_name():
+def subscription_dict(output_directory, split_timestamps_file_path):
-    return "jmc"
-
-
-@pytest.fixture
-def config(config_path):
-    return ConfigFile.from_file_path(config_path=config_path)
-
-
-@pytest.fixture
-def subscription_dict(output_directory, subscription_name, split_timestamps_file_path):
     return {
         "preset": "yt_music_video",
         "youtube": {
@@ -73,16 +54,16 @@ def subscription_dict(output_directory, subscription_name, split_timestamps_file
 
 
 @pytest.fixture
-def single_video_subscription(config, subscription_name, subscription_dict):
+def single_video_subscription(music_video_config, subscription_dict):
     single_video_preset = Preset.from_dict(
-        config=config,
+        config=music_video_config,
-        preset_name=subscription_name,
+        preset_name="split_video_test",
         preset_dict=subscription_dict,
     )
 
     return Subscription.from_preset(
         preset=single_video_preset,
-        config=config,
+        config=music_video_config,
     )
 
 

tests/unit/plugins/test_regex_capture.py

@@ -0,0 +1 @@
+

tests/unit/validators/test_regex_validator.py

@@ -6,21 +6,20 @@ from ytdl_sub.validators.regex_validator import RegexValidator
 
 
 @pytest.mark.parametrize(
-    "regex_value, input_str, matches, captures",
+    "regex_value, input_str, expected_output",
     [
-        ("^match this$", "match this", True, None),
+        ("^match this$", "match this", []),
-        ("^my (.+) cap", "my first cap", True, ["first"]),
+        ("^my (.+) cap", "my first cap", ["first"]),
-        (".* (.+) - (.+) two", "my other - capped two", True, ["other", "capped"]),
+        (".* (.+) - (.+) two", "my other - capped two", ["other", "capped"]),
-        ("failed match", "nope", False, None),
+        ("failed match", "nope", None),
     ],
 )
-def test_regex_validator(regex_value, input_str, matches, captures):
+def test_regex_validator(regex_value, input_str, expected_output):
     regex_validator = RegexValidator(name="good_regex_validator", value=regex_value)
     assert regex_validator._name == "good_regex_validator"
     assert regex_validator._value == regex_value
 
-    assert regex_validator.is_match(input_str=input_str) is matches
+    assert regex_validator.match(input_str=input_str) == expected_output
-    assert regex_validator.capture(input_str=input_str) == captures
 
 
 @pytest.mark.parametrize("regex_value", ["(", "??"])
@@ -29,17 +28,16 @@ def test_regex_validator_fails_bad_value(regex_value):
         _ = RegexValidator(name="fail", value=regex_value)
 
 
-def test_regex_list_validator():
+def test_regex_list_validator_matches():
     regex_list_validator_raw_value = ["try matching this", "try matching that", "how about this"]
 
     regex_list = RegexListValidator(name="list val", value=regex_list_validator_raw_value)
 
     for raw_val in regex_list_validator_raw_value:
-        assert regex_list.matches_any(raw_val)
+        assert regex_list.match_any(raw_val) == []
-        assert regex_list.capture_any(raw_val) is None
 
 
-def test_regex_list_validator_capture():
+def test_regex_list_validator_captures():
     regex_list_validator_raw_value = ["try (.+) this", "try (.+) that", "how (.+) this"]
 
     regex_list = RegexListValidator(name="list val", value=regex_list_validator_raw_value)
@@ -50,8 +48,7 @@ def test_regex_list_validator_capture():
     ]
 
     for capture_str, expected_capture in captures_test:
-        assert regex_list.matches_any(capture_str)
+        assert regex_list.match_any(capture_str) == [expected_capture]
-        assert regex_list.capture_any(capture_str) == [expected_capture]
 
 
 def test_regex_list_validator_invalid_regex():

tests/unit/validators/test_string_formatter_validator.py

@@ -19,8 +19,8 @@ def error_message_unequal_brackets_str():
 @pytest.fixture
 def error_message_unequal_regex_matches_str():
     return (
-        "{variable_names} should only contain lowercase letters and "
+        "{variable_names} must start with a lowercase letter, should only contain lowercase "
-        "underscores with a single open and close bracket."
+        "letters, numbers, underscores, and have a single open and close bracket."
     )
 
 
@@ -29,11 +29,11 @@ def error_message_unequal_regex_matches_str():
 )
 class TestStringFormatterValidator(object):
     def test_parse(self, string_formatter_class):
-        format_string = "Here is my {var_one} and {var_two} 💩"
+        format_string = "Here is my {var_one1} and {var_two} 💩"
         validator = string_formatter_class(name="test_format_variables", value=format_string)
 
         assert validator.format_string == format_string
-        assert validator.format_variables == ["var_one", "var_two"]
+        assert validator.format_variables == ["var_one1", "var_two"]
 
     def test_format_variables(self, string_formatter_class):
         format_string = "No vars 💩"
@@ -64,7 +64,8 @@ class TestStringFormatterValidator(object):
     @pytest.mark.parametrize(
         "format_string",
         [
-            "Try {var1} numbers",
+            "Try {1starting_number}",
+            "Try {_underscore_first}",
             "Try {VAR1} caps w/numbers",
             "Try {internal{bracket}}",
             "Try }backwards{ facing",