[FEATURE] exclude field in regex plugin (#620)

* [FEATURE] `exclude` field in regex plugin

* tests

* excludes doc

docs/config.rst

@@ -214,7 +214,7 @@ regex
   :members: skip_if_match_fails
 
 .. autoclass:: ytdl_sub.plugins.regex.SourceVariableRegex()
-  :members: match, capture_group_names, capture_group_defaults
+  :members: match, capture_group_names, capture_group_defaults, exclude
   :member-order: bysource
   :exclude-members: partial_validate
 

src/ytdl_sub/plugins/regex.py

@@ -24,12 +24,12 @@ logger = Logger.get(name="regex")
 
 class SourceVariableRegex(StrictDictValidator):
 
-    _required_keys = {"match"}
+    _optional_keys = {"match", "exclude", "capture_group_defaults", "capture_group_names"}
-    _optional_keys = {"capture_group_defaults", "capture_group_names"}
 
     def __init__(self, name, value):
         super().__init__(name, value)
-        self._match = self._validate_key(key="match", validator=RegexListValidator)
+        self._match = self._validate_key_if_present(key="match", validator=RegexListValidator)
+        self._exclude = self._validate_key_if_present(key="exclude", validator=RegexListValidator)
         self._capture_group_defaults = self._validate_key_if_present(
             key="capture_group_defaults", validator=ListFormatterValidator
         )
@@ -37,6 +37,14 @@ class SourceVariableRegex(StrictDictValidator):
             key="capture_group_names", validator=SourceVariableNameListValidator, default=[]
         )
 
+        if self._match is None and self._exclude is None:
+            raise self._validation_exception("must specify either `match` or `exclude`")
+
+        if self._match is None and (self._capture_group_defaults or self._capture_group_names.list):
+            raise self._validation_exception(
+                "capture group parameters requires at least one `match` to be specified"
+            )
+
         # If defaults are to be used, ensure there are the same number of defaults as there are
         # capture groups
         if self._capture_group_defaults is not None and self._match.num_capture_groups != len(
@@ -48,20 +56,29 @@ class SourceVariableRegex(StrictDictValidator):
             )
 
         # If there are capture groups, ensure there are capture group names
-        if len(self._capture_group_names.list) != self._match.num_capture_groups:
+        if self._match and (len(self._capture_group_names.list) != self._match.num_capture_groups):
             raise self._validation_exception(
                 f"number of capture group names must match number of capture groups, "
                 f"{len(self._capture_group_names.list)} != {self._match.num_capture_groups}"
             )
 
     @property
-    def match(self) -> RegexListValidator:
+    def match(self) -> Optional[RegexListValidator]:
         """
-        Required. List of regex strings to try to match against a source variable. Each regex
+        List of regex strings to try to match against a source variable. Each regex
         string must have the same number of capture groups.
         """
         return self._match
 
+    @property
+    def exclude(self) -> Optional[RegexListValidator]:
+        """
+        List of regex strings to try to match against a source variable. If one of the regex strings
+        match, then the entry will be skipped. If both ``exclude`` and ``match`` are specified,
+        entries will get skipped if the regex matches against both ``exclude`` and ``match``.
+        """
+        return self._exclude
+
     @property
     def capture_group_names(self) -> Optional[List[str]]:
         """
@@ -129,7 +146,7 @@ class RegexOptions(PluginOptions):
                  # This will only download videos with "[Official Video]" in it. Note that we
                  # double backslash to make YAML happy
                  match:
-                  - '\\[Official Video\\]'
+                   - '\\[Official Video\\]'
 
                # For each entry's `description` value...
                description:
@@ -137,14 +154,17 @@ class RegexOptions(PluginOptions):
                  # This tries to scrape a date from the description and produce new
                  # source variables
                  match:
-                  - "([0-9]{4})-([0-9]{2})-([0-9]{2})"
+                   - "([0-9]{4})-([0-9]{2})-([0-9]{2})"
+                 # Exclude any entry where the description contains #short
+                 exclude:
+                   - "#short"
 
                  # Each capture group creates these new source variables, respectively,
                  # as well a sanitized version, i.e. `captured_upload_year_sanitized`
                  capture_group_names:
-                  - "captured_upload_year"
+                   - "captured_upload_year"
-                  - "captured_upload_month"
+                   - "captured_upload_month"
-                  - "captured_upload_day"
+                   - "captured_upload_day"
 
                  # And if the string does not match, use these as respective default
                  # values for the new source variables.
@@ -257,6 +277,19 @@ class RegexPlugin(Plugin[RegexOptions]):
     def _contains_processed_regex_source_var(cls, entry: Entry, source_var: str) -> bool:
         return source_var in entry.kwargs_get(YTDL_SUB_REGEX_SOURCE_VARS, [])
 
+    def _try_skip_entry(self, entry: Entry, source_var: str) -> None:
+        # Skip the entry if toggled
+        if self.plugin_options.skip_if_match_fails:
+            logger.info(
+                "Regex failed to match '%s' from '%s', skipping.",
+                source_var,
+                entry.title,
+            )
+            return None
+
+        # Otherwise, error
+        raise RegexNoMatchException(f"Regex failed to match '{source_var}' from '{entry.title}'")
+
     def _modify_entry_metadata(self, entry: Entry, is_metadata_stage: bool) -> Optional[Entry]:
         """
         Parameters
@@ -292,70 +325,70 @@ class RegexPlugin(Plugin[RegexOptions]):
                 continue
 
             self._add_processed_regex_source_var(entry, source_var)
-            maybe_capture = regex_options.match.match_any(input_str=entry_variable_dict[source_var])
 
-            # If no capture
+            if (
-            if maybe_capture is None:
+                regex_options.exclude is not None
-                # and no defaults
+                and regex_options.exclude.match_any(input_str=entry_variable_dict[source_var])
-                if not regex_options.has_defaults:
+                is not None
-                    # Skip the entry if toggled
+            ):
-                    if self.plugin_options.skip_if_match_fails:
+                return self._try_skip_entry(entry=entry, source_var=source_var)
-                        logger.info(
-                            "Regex failed to match '%s' from '%s', skipping.",
-                            source_var,
-                            entry.title,
-                        )
-                        return None
 
-                    # Otherwise, error
+            # If match is present
-                    raise RegexNoMatchException(
+            if regex_options.match is not None:
-                        f"Regex failed to match '{source_var}' from '{entry.title}'"
+                maybe_capture = regex_options.match.match_any(
+                    input_str=entry_variable_dict[source_var]
+                )
+
+                # And nothing matched
+                if maybe_capture is None:
+                    # and no defaults
+                    if not regex_options.has_defaults:
+                        return self._try_skip_entry(entry=entry, source_var=source_var)
+
+                    # otherwise, use defaults (apply them using the original entry source dict)
+                    source_variables_and_overrides_dict = dict(
+                        entry_variable_dict, **self.overrides.dict_with_format_strings
                     )
 
-                # otherwise, use defaults (apply them using the original entry source dict)
+                    # add both the default...
-                source_variables_and_overrides_dict = dict(
+                    entry.add_variables(
-                    entry_variable_dict, **self.overrides.dict_with_format_strings
+                        variables_to_add={
-                )
+                            regex_options.capture_group_names[i]: default.apply_formatter(
-
-                # add both the default...
-                entry.add_variables(
-                    variables_to_add={
-                        regex_options.capture_group_names[i]: default.apply_formatter(
-                            variable_dict=source_variables_and_overrides_dict
-                        )
-                        for i, default in enumerate(regex_options.capture_group_defaults)
-                    },
-                )
-                # and sanitized default
-                entry.add_variables(
-                    variables_to_add={
-                        f"{regex_options.capture_group_names[i]}_sanitized": sanitize_filename(
-                            default.apply_formatter(
                                 variable_dict=source_variables_and_overrides_dict
                             )
-                        )
+                            for i, default in enumerate(regex_options.capture_group_defaults)
-                        for i, default in enumerate(regex_options.capture_group_defaults)
+                        },
-                    },
+                    )
-                )
+                    # and sanitized default
-            # There is a capture, add the source variables to the entry as
+                    entry.add_variables(
-            # {source_var}_capture_1, {source_var}_capture_2, ...
+                        variables_to_add={
-            else:
+                            f"{regex_options.capture_group_names[i]}_sanitized": sanitize_filename(
-                # Add the value...
+                                default.apply_formatter(
-                entry.add_variables(
+                                    variable_dict=source_variables_and_overrides_dict
-                    variables_to_add={
+                                )
-                        regex_options.capture_group_names[i]: capture
+                            )
-                        for i, capture in enumerate(maybe_capture)
+                            for i, default in enumerate(regex_options.capture_group_defaults)
-                    },
+                        },
-                )
+                    )
-                # And the sanitized value
+                # There is a capture, add the source variables to the entry as
-                entry.add_variables(
+                # {source_var}_capture_1, {source_var}_capture_2, ...
-                    variables_to_add={
+                else:
-                        f"{regex_options.capture_group_names[i]}_sanitized": sanitize_filename(
+                    # Add the value...
-                            capture
+                    entry.add_variables(
-                        )
+                        variables_to_add={
-                        for i, capture in enumerate(maybe_capture)
+                            regex_options.capture_group_names[i]: capture
-                    },
+                            for i, capture in enumerate(maybe_capture)
-                )
+                        },
+                    )
+                    # And the sanitized value
+                    entry.add_variables(
+                        variables_to_add={
+                            f"{regex_options.capture_group_names[i]}_sanitized": sanitize_filename(
+                                capture
+                            )
+                            for i, capture in enumerate(maybe_capture)
+                        },
+                    )
 
         return entry
 

tests/e2e/plugins/test_regex.py

@@ -1,17 +1,20 @@
 import copy
 import re
+from typing import Any
+from typing import Dict
 
+import mergedeep
 import pytest
 from expected_transaction_log import assert_transaction_log_matches
 
-from ytdl_sub.config.preset import Preset
+from ytdl_sub.config.config_file import ConfigFile
 from ytdl_sub.subscriptions.subscription import Subscription
 from ytdl_sub.utils.exceptions import RegexNoMatchException
 from ytdl_sub.utils.exceptions import ValidationException
 
 
 @pytest.fixture
-def regex_subscription_dict(output_directory):
+def regex_subscription_dict_base(output_directory):
     return {
         "preset": "music_video",
         "download": {"url": "https://youtube.com/playlist?list=PL5BC0FC26BECA5A35"},
@@ -24,13 +27,6 @@ def regex_subscription_dict(output_directory):
         "regex": {
             # tests that skip_if_match_fails defaults to True
             "from": {
-                "title": {
-                    "match": [
-                        "should not cap (.+) - (.+)",
-                        ".*\\[(.+) - (Feb.+)]",  # should filter out march video
-                    ],
-                    "capture_group_names": ["title_type", "title_date"],
-                },
                 "description": {
                     "match": [".*http:\\/\\/(.+).com.*"],
                     "capture_group_names": ["description_website"],
@@ -48,58 +44,149 @@ def regex_subscription_dict(output_directory):
                 },
             },
         },
-        "nfo_tags": {
-            "tags": {
-                "title_cap_1": "{title_type}",
-                "title_cap_1_sanitized": "{title_type_sanitized}",
-                "title_cap_2": "{title_date}",
-                "desc_cap": "{description_website}",
-                "upload_date_both_caps": "{upload_captured_year} and {upload_captured_month}",
-                "override_with_capture_variable": "{contains_regex_default}",
-                "override_with_capture_variable_sanitized": "{contains_regex_sanitized_default}",
-            }
-        },
         "overrides": {
             "in_regex_default": "in regex default",
-            "contains_regex_default": "contains {title_type}",
-            "contains_regex_sanitized_default": "contains {title_type_sanitized}",
         },
     }
 
 
 @pytest.fixture
-def regex_subscription_dict_no_match_fails(regex_subscription_dict):
+def regex_subscription_dict(regex_subscription_dict_base, output_directory):
-    regex_subscription_dict["regex"]["skip_if_match_fails"] = False
+    return mergedeep.merge(
-    return regex_subscription_dict
+        regex_subscription_dict_base,
+        {
+            "regex": {
+                # tests that skip_if_match_fails defaults to True
+                "from": {
+                    "title": {
+                        "match": [
+                            "should not cap (.+) - (.+)",
+                            ".*\\[(.+) - (Feb.+)]",  # should filter out march video
+                        ],
+                        "capture_group_names": ["title_type", "title_date"],
+                    },
+                },
+            },
+            "nfo_tags": {
+                "tags": {
+                    "title_cap_1": "{title_type}",
+                    "title_cap_1_sanitized": "{title_type_sanitized}",
+                    "title_cap_2": "{title_date}",
+                    "desc_cap": "{description_website}",
+                    "upload_date_both_caps": "{upload_captured_year} and {upload_captured_month}",
+                    "override_with_capture_variable": "{contains_regex_default}",
+                    "override_with_capture_variable_sanitized": "{contains_regex_sanitized_default}",
+                }
+            },
+            "overrides": {
+                "contains_regex_default": "contains {title_type}",
+                "contains_regex_sanitized_default": "contains {title_type_sanitized}",
+            },
+        },
+    )
+
+
+@pytest.fixture
+def regex_subscription_dict_exclude(regex_subscription_dict_base, output_directory):
+    return mergedeep.merge(
+        regex_subscription_dict_base,
+        {
+            "regex": {
+                # tests that skip_if_match_fails defaults to True
+                "from": {
+                    "title": {
+                        "exclude": [
+                            "should not cap",
+                            ".*Feb.*",  # should filter out march video
+                        ],
+                    },
+                },
+            },
+        },
+    )
+
+
+@pytest.fixture
+def regex_subscription_dict_match_and_exclude(regex_subscription_dict_base, output_directory):
+    return mergedeep.merge(
+        regex_subscription_dict_base,
+        {
+            "regex": {
+                # tests that skip_if_match_fails defaults to True
+                "from": {
+                    "title": {
+                        "match": [
+                            "should not cap (.+) - (.+)",
+                            ".*\\[(.+) - (Feb.+)]",  # should filter out march video
+                        ],
+                        "capture_group_names": ["title_type", "title_date"],
+                        "exclude": [
+                            "should not cap",
+                            ".*27.*",  # should filter out Feb 27th video
+                        ],
+                    },
+                },
+            },
+            "nfo_tags": {
+                "tags": {
+                    "title_cap_1": "{title_type}",
+                    "title_cap_1_sanitized": "{title_type_sanitized}",
+                    "title_cap_2": "{title_date}",
+                    "desc_cap": "{description_website}",
+                    "upload_date_both_caps": "{upload_captured_year} and {upload_captured_month}",
+                    "override_with_capture_variable": "{contains_regex_default}",
+                    "override_with_capture_variable_sanitized": "{contains_regex_sanitized_default}",
+                }
+            },
+            "overrides": {
+                "contains_regex_default": "contains {title_type}",
+                "contains_regex_sanitized_default": "contains {title_type_sanitized}",
+            },
+        },
+    )
 
 
 @pytest.fixture
 def playlist_subscription(music_video_config, regex_subscription_dict):
-    playlist_preset = Preset.from_dict(
+    return Subscription.from_dict(
         config=music_video_config,
         preset_name="regex_capture_playlist_test",
         preset_dict=regex_subscription_dict,
     )
 
-    return Subscription.from_preset(
+
-        preset=playlist_preset,
+@pytest.fixture
+def playlist_subscription_no_match_fails(
+    music_video_config: ConfigFile, regex_subscription_dict: Dict[str, Any]
+):
+    regex_subscription_dict["regex"]["skip_if_match_fails"] = False
+
+    return Subscription.from_dict(
         config=music_video_config,
+        preset_name="regex_capture_playlist_test",
+        preset_dict=regex_subscription_dict,
     )
 
 
 @pytest.fixture
-def playlist_subscription_no_match_fails(
+def playlist_subscription_exclude(
-    music_video_config, regex_subscription_dict_no_match_fails
+    music_video_config: ConfigFile, regex_subscription_dict_exclude: Dict[str, Any]
-):
+) -> Subscription:
-    playlist_preset = Preset.from_dict(
+    return Subscription.from_dict(
         config=music_video_config,
-        preset_name="regex_capture_playlist_test",
+        preset_name="regex_exclude_playlist_test",
-        preset_dict=regex_subscription_dict_no_match_fails,
+        preset_dict=regex_subscription_dict_exclude,
     )
 
-    return Subscription.from_preset(
+
-        preset=playlist_preset,
+@pytest.fixture
+def playlist_subscription_match_and_exclude(
+    music_video_config: ConfigFile, regex_subscription_dict_match_and_exclude: Dict[str, Any]
+) -> Subscription:
+    return Subscription.from_dict(
         config=music_video_config,
+        preset_name="regex_match_and_exclude_playlist_test",
+        preset_dict=regex_subscription_dict_match_and_exclude,
     )
 
 
@@ -113,6 +200,27 @@ class TestRegex:
             transaction_log_summary_file_name="plugins/test_regex.txt",
         )
 
+    def test_regex_excludes_success(self, playlist_subscription_exclude, output_directory):
+        # Should only contain the march video
+        transaction_log = playlist_subscription_exclude.download(dry_run=True)
+        assert_transaction_log_matches(
+            output_directory=output_directory,
+            transaction_log=transaction_log,
+            transaction_log_summary_file_name="plugins/test_regex_exclude.txt",
+        )
+
+    def test_regex_match_and_excludes_success(
+        self, playlist_subscription_match_and_exclude, output_directory
+    ):
+        # Should only contain the Feb 1st video
+        transaction_log = playlist_subscription_match_and_exclude.download(dry_run=True)
+        assert_transaction_log_matches(
+            output_directory=output_directory,
+            transaction_log=transaction_log,
+            transaction_log_summary_file_name="plugins/test_regex_match_and_exclude.txt",
+            regenerate_transaction_log=True,
+        )
+
     def test_regex_fails_no_match(self, playlist_subscription_no_match_fails, output_directory):
         with pytest.raises(
             RegexNoMatchException,
@@ -122,6 +230,34 @@ class TestRegex:
         ):
             _ = playlist_subscription_no_match_fails.download(dry_run=True)
 
+    def test_regex_fails_capture_group_with_only_excludes(
+        self, regex_subscription_dict_exclude, music_video_config
+    ):
+        regex_subscription_dict_exclude["regex"]["from"]["title"]["capture_group_names"] = ["uid"]
+        with pytest.raises(
+            ValidationException,
+            match=re.escape(
+                "capture group parameters requires at least one `match` to be specified"
+            ),
+        ):
+            _ = Subscription.from_dict(
+                config=music_video_config,
+                preset_name="test_regex_fails_capture_group_is_source_variable",
+                preset_dict=regex_subscription_dict_exclude,
+            )
+
+    def test_regex_fails_no_match_or_exclude(self, regex_subscription_dict, music_video_config):
+        del regex_subscription_dict["regex"]["from"]["title"]["match"]
+        with pytest.raises(
+            ValidationException,
+            match=re.escape("must specify either `match` or `exclude`"),
+        ):
+            _ = Subscription.from_dict(
+                config=music_video_config,
+                preset_name="test_regex_fails_capture_group_is_source_variable",
+                preset_dict=regex_subscription_dict,
+            )
+
     def test_regex_fails_capture_group_is_source_variable(
         self, regex_subscription_dict, music_video_config
     ):

tests/resources/transaction_log_summaries/plugins/test_regex_exclude.txt

@@ -0,0 +1,15 @@
+Files created:
+----------------------------------------
+{output_directory}
+  .ytdl-sub-regex_exclude_playlist_test-download-archive.json
+{output_directory}/Project Zombie
+  Project Zombie - Jesse's Minecraft Server [Trailer - Mar.21]-thumb.jpg
+  Project Zombie - Jesse's Minecraft Server [Trailer - Mar.21].info.json
+  Project Zombie - Jesse's Minecraft Server [Trailer - Mar.21].mp4
+  Project Zombie - Jesse's Minecraft Server [Trailer - Mar.21].nfo
+    NFO tags:
+      musicvideo:
+        album: Music Videos
+        artist: Project Zombie
+        title: Jesse's Minecraft Server [Trailer - Mar.21]
+        year: 2011

tests/resources/transaction_log_summaries/plugins/test_regex_match_and_exclude.txt

@@ -0,0 +1,22 @@
+Files created:
+----------------------------------------
+{output_directory}
+  .ytdl-sub-regex_match_and_exclude_playlist_test-download-archive.json
+{output_directory}/Project Zombie
+  Project Zombie - Jesse's Minecraft Server [Trailer - Feb.1]-thumb.jpg
+  Project Zombie - Jesse's Minecraft Server [Trailer - Feb.1].info.json
+  Project Zombie - Jesse's Minecraft Server [Trailer - Feb.1].mp4
+  Project Zombie - Jesse's Minecraft Server [Trailer - Feb.1].nfo
+    NFO tags:
+      musicvideo:
+        album: Music Videos
+        artist: Project Zombie
+        desc_cap: www.jesseminecraft.webs
+        override_with_capture_variable: contains Trailer
+        override_with_capture_variable_sanitized: contains Trailer
+        title: Jesse's Minecraft Server [Trailer - Feb.1]
+        title_cap_1: Trailer
+        title_cap_1_sanitized: Trailer
+        title_cap_2: Feb.1
+        upload_date_both_caps: First and Second containing in regex default
+        year: 2011