[FEATURE] Ability to use overrides in regex plugin (#631)

* [FEATURE] Ability to use overrides in regex plugin

* lint

* more tests

* more docs

* docs

docs/config.rst

@@ -213,7 +213,7 @@ regex
 .. autoclass:: ytdl_sub.plugins.regex.RegexOptions()
   :members: skip_if_match_fails
 
-.. autoclass:: ytdl_sub.plugins.regex.SourceVariableRegex()
+.. autoclass:: ytdl_sub.plugins.regex.VariableRegex()
   :members: match, capture_group_names, capture_group_defaults, exclude
   :member-order: bysource
   :exclude-members: partial_validate

src/ytdl_sub/plugins/regex.py

@@ -11,6 +11,7 @@ from ytdl_sub.plugins.plugin import Plugin
 from ytdl_sub.plugins.plugin import PluginOptions
 from ytdl_sub.plugins.plugin import PluginPriority
 from ytdl_sub.utils.exceptions import RegexNoMatchException
+from ytdl_sub.utils.exceptions import StringFormattingVariableNotFoundException
 from ytdl_sub.utils.logger import Logger
 from ytdl_sub.validators.regex_validator import RegexListValidator
 from ytdl_sub.validators.source_variable_validator import SourceVariableNameListValidator
@@ -22,7 +23,7 @@ from ytdl_sub.validators.validators import BoolValidator
 logger = Logger.get(name="regex")
 
 
-class SourceVariableRegex(StrictDictValidator):
+class VariableRegex(StrictDictValidator):
 
     _optional_keys = {"match", "exclude", "capture_group_defaults", "capture_group_names"}
 
@@ -65,7 +66,7 @@ class SourceVariableRegex(StrictDictValidator):
     @property
     def match(self) -> Optional[RegexListValidator]:
         """
-        List of regex strings to try to match against a source variable. Each regex
+        List of regex strings to try to match against a source or override variable. Each regex
         string must have the same number of capture groups.
         """
         return self._match
@@ -73,9 +74,10 @@ class SourceVariableRegex(StrictDictValidator):
     @property
     def exclude(self) -> Optional[RegexListValidator]:
         """
-        List of regex strings to try to match against a source variable. If one of the regex strings
+        List of regex strings to try to match against a source or override variable. If one of the
-        match, then the entry will be skipped. If both ``exclude`` and ``match`` are specified,
+        regex strings match, then the entry will be skipped. If both ``exclude`` and ``match`` are
-        entries will get skipped if the regex matches against both ``exclude`` and ``match``.
+        specified, entries will get skipped if the regex matches against both ``exclude`` and
+        ``match``.
         """
         return self._exclude
 
@@ -114,19 +116,25 @@ class FromSourceVariablesRegex(StrictDictValidator):
 
     def __init__(self, name, value):
         super().__init__(name, value)
-        self.source_variable_capture_dict: Dict[str, SourceVariableRegex] = {
+        self.variable_capture_dict: Dict[str, VariableRegex] = {
-            key: self._validate_key(key=key, validator=SourceVariableRegex) for key in self._keys
+            key: self._validate_key(key=key, validator=VariableRegex) for key in self._keys
         }
 
 
 class RegexOptions(PluginOptions):
     r"""
-    Performs regex matching on an entry's source variables. Regex can be used to filter entries
+    Performs regex matching on an entry's source or override variables. Regex can be used to filter
-    from proceeding with download or capture groups to create new source variables. NOTE to
+    entries from proceeding with download or capture groups to create new source variables. NOTE to
     use backslashes anywhere in your regex, i.e. ``\d``, you must add another backslash escape. This
     means ``\d`` should be written as ``\\d``. This is because YAML requires an escape for any
     backslash usage.
 
+    If you want to regex-search multiple source variables to create a logical OR effect, you can
+    create an override variable that contains the concatenation of them, and search that with regex.
+    For example, creating the override variable ``"title_and_description": "{title} {description}"``
+    and using ``title_and_description`` can regex match/exclude from either ``title`` or
+    ``description``.
+
     Usage:
 
     .. code-block:: yaml
@@ -216,9 +224,9 @@ class RegexOptions(PluginOptions):
         """
         for key, regex_options in self.source_variable_capture_dict.items():
             # Ensure each variable getting captured is a source variable
-            if key not in source_variables:
+            if key not in source_variables and key not in override_variables:
                 raise self._validation_exception(
-                    f"cannot regex capture '{key}' because it is not a source variable"
+                    f"cannot regex capture '{key}' because it is not a source or override variable"
                 )
 
             # Ensure the capture group names are not existing source/override variables
@@ -235,13 +243,13 @@ class RegexOptions(PluginOptions):
                     )
 
     @property
-    def source_variable_capture_dict(self) -> Dict[str, SourceVariableRegex]:
+    def source_variable_capture_dict(self) -> Dict[str, VariableRegex]:
         """
         Returns
         -------
         Dict of { source variable: capture options }
         """
-        return self._from.source_variable_capture_dict
+        return self._from.variable_capture_dict
 
     def added_source_variables(self) -> List[str]:
         """
@@ -267,28 +275,56 @@ class RegexPlugin(Plugin[RegexOptions]):
     )
 
     @classmethod
-    def _add_processed_regex_source_var(cls, entry: Entry, source_var: str) -> None:
+    def _add_processed_regex_variable_name(cls, entry: Entry, source_var: str) -> None:
         if not entry.kwargs_contains(YTDL_SUB_REGEX_SOURCE_VARS):
             entry.add_kwargs({YTDL_SUB_REGEX_SOURCE_VARS: []})
 
         entry.kwargs(YTDL_SUB_REGEX_SOURCE_VARS).append(source_var)
 
     @classmethod
-    def _contains_processed_regex_source_var(cls, entry: Entry, source_var: str) -> bool:
+    def _contains_processed_regex_variable(cls, entry: Entry, variable_name: str) -> bool:
-        return source_var in entry.kwargs_get(YTDL_SUB_REGEX_SOURCE_VARS, [])
+        return variable_name in entry.kwargs_get(YTDL_SUB_REGEX_SOURCE_VARS, [])
 
-    def _try_skip_entry(self, entry: Entry, source_var: str) -> None:
+    def _try_skip_entry(self, entry: Entry, variable_name: str) -> None:
         # Skip the entry if toggled
         if self.plugin_options.skip_if_match_fails:
             logger.info(
                 "Regex failed to match '%s' from '%s', skipping.",
-                source_var,
+                variable_name,
                 entry.title,
             )
             return None
 
         # Otherwise, error
-        raise RegexNoMatchException(f"Regex failed to match '{source_var}' from '{entry.title}'")
+        raise RegexNoMatchException(f"Regex failed to match '{variable_name}' from '{entry.title}'")
+
+    def _can_process_at_metadata_stage(self, entry: Entry, variable_name: str) -> bool:
+        # If the variable is an override...
+        if variable_name in self.overrides.dict:
+            # Try to see if it can resolve
+            try:
+                self.overrides.apply_formatter(
+                    formatter=self.overrides.dict[variable_name],
+                    entry=entry,
+                )
+            # If it can not from missing variables (from post-metadata stage), return False
+            except StringFormattingVariableNotFoundException:
+                return False
+        # If it is a source variable and not present, return false
+        elif variable_name not in entry.to_dict():
+            return False
+
+        return True
+
+    def _get_regex_input_string(self, entry: Entry, variable_name: str) -> str:
+        # Apply override formatter if it's an override
+        if variable_name in self.overrides.dict:
+            return self.overrides.apply_formatter(
+                formatter=self.overrides.dict[variable_name],
+                entry=entry,
+            )
+        # Otherwise pluck from the entry's source variable
+        return entry.to_dict()[variable_name]
 
     def _modify_entry_metadata(self, entry: Entry, is_metadata_stage: bool) -> Optional[Entry]:
         """
@@ -308,46 +344,50 @@ class RegexPlugin(Plugin[RegexOptions]):
         ValidationException
             If no capture and no defaults
         """
-        entry_variable_dict = entry.to_dict()
-
         # Iterate each source var to capture and add to the entry
-        for source_var, regex_options in self.plugin_options.source_variable_capture_dict.items():
+        for (
+            variable_name,
+            regex_options,
+        ) in self.plugin_options.source_variable_capture_dict.items():
 
             # Record which regex source variables are processed, to
             # process as many variables as possible in the metadata stage, then the rest
             # after the media file has been downloaded.
-            if self._contains_processed_regex_source_var(entry, source_var):
+            if self._contains_processed_regex_variable(entry, variable_name):
                 continue
 
-            # Continue if the source var isn't in the dict since entry variables could be added
+            # If it's the metadata stage, and it can't be processed, skip until post-metadata
-            # after the metadata stage
+            if is_metadata_stage and not self._can_process_at_metadata_stage(
-            if is_metadata_stage and source_var not in entry_variable_dict:
+                entry=entry, variable_name=variable_name
+            ):
                 continue
 
-            self._add_processed_regex_source_var(entry, source_var)
+            self._add_processed_regex_variable_name(entry, variable_name)
+
+            regex_input_str = self._get_regex_input_string(
+                entry=entry,
+                variable_name=variable_name,
+            )
 
             if (
                 regex_options.exclude is not None
-                and regex_options.exclude.match_any(input_str=entry_variable_dict[source_var])
+                and regex_options.exclude.match_any(input_str=regex_input_str) is not None
-                is not None
             ):
-                return self._try_skip_entry(entry=entry, source_var=source_var)
+                return self._try_skip_entry(entry=entry, variable_name=variable_name)
 
             # If match is present
             if regex_options.match is not None:
-                maybe_capture = regex_options.match.match_any(
+                maybe_capture = regex_options.match.match_any(input_str=regex_input_str)
-                    input_str=entry_variable_dict[source_var]
-                )
 
                 # And nothing matched
                 if maybe_capture is None:
                     # and no defaults
                     if not regex_options.has_defaults:
-                        return self._try_skip_entry(entry=entry, source_var=source_var)
+                        return self._try_skip_entry(entry=entry, variable_name=variable_name)
 
                     # otherwise, use defaults (apply them using the original entry source dict)
                     source_variables_and_overrides_dict = dict(
-                        entry_variable_dict, **self.overrides.dict_with_format_strings
+                        entry.to_dict(), **self.overrides.dict_with_format_strings
                     )
 
                     # add both the default...

tests/e2e/plugins/test_music_tags.py

@@ -1,8 +1,6 @@
 import re
 
 import pytest
-from expected_download import assert_expected_downloads
-from expected_transaction_log import assert_transaction_log_matches
 
 from ytdl_sub.subscriptions.subscription import Subscription
 from ytdl_sub.utils.exceptions import ValidationException

tests/e2e/plugins/test_regex.py

@@ -146,6 +146,33 @@ def regex_subscription_dict_match_and_exclude(regex_subscription_dict_base, outp
     )
 
 
+@pytest.fixture
+def regex_subscription_dict_match_and_exclude_override_variable(
+    regex_subscription_dict_base, output_directory
+):
+    return mergedeep.merge(
+        regex_subscription_dict_base,
+        {
+            "regex": {
+                # tests that skip_if_match_fails defaults to True
+                "from": {
+                    "override_title": {
+                        "exclude": [
+                            "should not cap",
+                            ".*Feb.*",  # should filter out march video
+                        ],
+                    },
+                    "override_description": {
+                        "match": [".*http:\\/\\/(.+).com.*"],
+                        "capture_group_names": ["override_description_website"],
+                    },
+                },
+            },
+            "overrides": {"override_title": "{title}", "override_description": "{description}"},
+        },
+    )
+
+
 @pytest.fixture
 def playlist_subscription(music_video_config, regex_subscription_dict):
     return Subscription.from_dict(
@@ -179,6 +206,18 @@ def playlist_subscription_exclude(
     )
 
 
+@pytest.fixture
+def playlist_subscription_overrides(
+    music_video_config: ConfigFile,
+    regex_subscription_dict_match_and_exclude_override_variable: Dict[str, Any],
+) -> Subscription:
+    return Subscription.from_dict(
+        config=music_video_config,
+        preset_name="regex_using_overrides_test",
+        preset_dict=regex_subscription_dict_match_and_exclude_override_variable,
+    )
+
+
 @pytest.fixture
 def playlist_subscription_match_and_exclude(
     music_video_config: ConfigFile, regex_subscription_dict_match_and_exclude: Dict[str, Any]
@@ -221,6 +260,16 @@ class TestRegex:
             regenerate_transaction_log=True,
         )
 
+    def test_regex_using_overrides_success(self, playlist_subscription_overrides, output_directory):
+        # Should only contain the march video
+        transaction_log = playlist_subscription_overrides.download(dry_run=True)
+        assert_transaction_log_matches(
+            output_directory=output_directory,
+            transaction_log=transaction_log,
+            transaction_log_summary_file_name="plugins/test_regex_overrides.txt",
+            regenerate_transaction_log=True,
+        )
+
     def test_regex_fails_no_match(self, playlist_subscription_no_match_fails, output_directory):
         with pytest.raises(
             RegexNoMatchException,
@@ -300,7 +349,9 @@ class TestRegex:
         )
         with pytest.raises(
             ValidationException,
-            match=re.escape("cannot regex capture 'dne' because it is not a source variable"),
+            match=re.escape(
+                "cannot regex capture 'dne' because it is not a source or override variable"
+            ),
         ):
             _ = Subscription.from_dict(
                 config=music_video_config,

tests/e2e/plugins/test_split_by_chapters.py

@@ -32,7 +32,9 @@ def yt_album_as_chapters_with_regex_preset_dict(yt_album_as_chapters_preset_dict
         {
             "regex": {
                 "from": {
-                    "chapter_title": {
+                    # Ensure regex can handle override variables that come from the
+                    # post-metadata stage
+                    "override_chapter_title": {
                         "match": r"\d+\. (.+)",
                         "capture_group_names": "captured_track_title",
                         "capture_group_defaults": "{chapter_title}",
@@ -54,6 +56,7 @@ def yt_album_as_chapters_with_regex_preset_dict(yt_album_as_chapters_preset_dict
                 }
             },
             "overrides": {
+                "override_chapter_title": "{chapter_title}",
                 "track_title": "{captured_track_title}",
                 "track_album": "{captured_track_album}",
                 "track_artist": "{captured_track_artist}",

tests/resources/transaction_log_summaries/plugins/test_regex_overrides.txt

@@ -0,0 +1,15 @@
+Files created:
+----------------------------------------
+{output_directory}
+  .ytdl-sub-regex_using_overrides_test-download-archive.json
+{output_directory}/Project Zombie
+  Project Zombie - Jesse's Minecraft Server [Trailer - Mar.21]-thumb.jpg
+  Project Zombie - Jesse's Minecraft Server [Trailer - Mar.21].info.json
+  Project Zombie - Jesse's Minecraft Server [Trailer - Mar.21].mp4
+  Project Zombie - Jesse's Minecraft Server [Trailer - Mar.21].nfo
+    NFO tags:
+      musicvideo:
+        album: Music Videos
+        artist: Project Zombie
+        title: Jesse's Minecraft Server [Trailer - Mar.21]
+        year: 2011