diff --git a/docs/config.rst b/docs/config.rst index 0725a0ef..56438f4d 100644 --- a/docs/config.rst +++ b/docs/config.rst @@ -214,7 +214,7 @@ regex :members: skip_if_match_fails .. autoclass:: ytdl_sub.plugins.regex.SourceVariableRegex() - :members: match, capture_group_names, capture_group_defaults + :members: match, capture_group_names, capture_group_defaults, exclude :member-order: bysource :exclude-members: partial_validate diff --git a/src/ytdl_sub/plugins/regex.py b/src/ytdl_sub/plugins/regex.py index 015580dd..1e7a5b03 100644 --- a/src/ytdl_sub/plugins/regex.py +++ b/src/ytdl_sub/plugins/regex.py @@ -24,12 +24,12 @@ logger = Logger.get(name="regex") class SourceVariableRegex(StrictDictValidator): - _required_keys = {"match"} - _optional_keys = {"capture_group_defaults", "capture_group_names"} + _optional_keys = {"match", "exclude", "capture_group_defaults", "capture_group_names"} def __init__(self, name, value): super().__init__(name, value) - self._match = self._validate_key(key="match", validator=RegexListValidator) + self._match = self._validate_key_if_present(key="match", validator=RegexListValidator) + self._exclude = self._validate_key_if_present(key="exclude", validator=RegexListValidator) self._capture_group_defaults = self._validate_key_if_present( key="capture_group_defaults", validator=ListFormatterValidator ) @@ -37,6 +37,14 @@ class SourceVariableRegex(StrictDictValidator): key="capture_group_names", validator=SourceVariableNameListValidator, default=[] ) + if self._match is None and self._exclude is None: + raise self._validation_exception("must specify either `match` or `exclude`") + + if self._match is None and (self._capture_group_defaults or self._capture_group_names.list): + raise self._validation_exception( + "capture group parameters requires at least one `match` to be specified" + ) + # If defaults are to be used, ensure there are the same number of defaults as there are # capture groups if self._capture_group_defaults is not None and self._match.num_capture_groups != len( @@ -48,20 +56,29 @@ class SourceVariableRegex(StrictDictValidator): ) # If there are capture groups, ensure there are capture group names - if len(self._capture_group_names.list) != self._match.num_capture_groups: + if self._match and (len(self._capture_group_names.list) != self._match.num_capture_groups): raise self._validation_exception( f"number of capture group names must match number of capture groups, " f"{len(self._capture_group_names.list)} != {self._match.num_capture_groups}" ) @property - def match(self) -> RegexListValidator: + def match(self) -> Optional[RegexListValidator]: """ - Required. List of regex strings to try to match against a source variable. Each regex + List of regex strings to try to match against a source variable. Each regex string must have the same number of capture groups. """ return self._match + @property + def exclude(self) -> Optional[RegexListValidator]: + """ + List of regex strings to try to match against a source variable. If one of the regex strings + match, then the entry will be skipped. If both ``exclude`` and ``match`` are specified, + entries will get skipped if the regex matches against both ``exclude`` and ``match``. + """ + return self._exclude + @property def capture_group_names(self) -> Optional[List[str]]: """ @@ -129,7 +146,7 @@ class RegexOptions(PluginOptions): # This will only download videos with "[Official Video]" in it. Note that we # double backslash to make YAML happy match: - - '\\[Official Video\\]' + - '\\[Official Video\\]' # For each entry's `description` value... description: @@ -137,14 +154,17 @@ class RegexOptions(PluginOptions): # This tries to scrape a date from the description and produce new # source variables match: - - "([0-9]{4})-([0-9]{2})-([0-9]{2})" + - "([0-9]{4})-([0-9]{2})-([0-9]{2})" + # Exclude any entry where the description contains #short + exclude: + - "#short" # Each capture group creates these new source variables, respectively, # as well a sanitized version, i.e. `captured_upload_year_sanitized` capture_group_names: - - "captured_upload_year" - - "captured_upload_month" - - "captured_upload_day" + - "captured_upload_year" + - "captured_upload_month" + - "captured_upload_day" # And if the string does not match, use these as respective default # values for the new source variables. @@ -257,6 +277,19 @@ class RegexPlugin(Plugin[RegexOptions]): def _contains_processed_regex_source_var(cls, entry: Entry, source_var: str) -> bool: return source_var in entry.kwargs_get(YTDL_SUB_REGEX_SOURCE_VARS, []) + def _try_skip_entry(self, entry: Entry, source_var: str) -> None: + # Skip the entry if toggled + if self.plugin_options.skip_if_match_fails: + logger.info( + "Regex failed to match '%s' from '%s', skipping.", + source_var, + entry.title, + ) + return None + + # Otherwise, error + raise RegexNoMatchException(f"Regex failed to match '{source_var}' from '{entry.title}'") + def _modify_entry_metadata(self, entry: Entry, is_metadata_stage: bool) -> Optional[Entry]: """ Parameters @@ -292,70 +325,70 @@ class RegexPlugin(Plugin[RegexOptions]): continue self._add_processed_regex_source_var(entry, source_var) - maybe_capture = regex_options.match.match_any(input_str=entry_variable_dict[source_var]) - # If no capture - if maybe_capture is None: - # and no defaults - if not regex_options.has_defaults: - # Skip the entry if toggled - if self.plugin_options.skip_if_match_fails: - logger.info( - "Regex failed to match '%s' from '%s', skipping.", - source_var, - entry.title, - ) - return None + if ( + regex_options.exclude is not None + and regex_options.exclude.match_any(input_str=entry_variable_dict[source_var]) + is not None + ): + return self._try_skip_entry(entry=entry, source_var=source_var) - # Otherwise, error - raise RegexNoMatchException( - f"Regex failed to match '{source_var}' from '{entry.title}'" + # If match is present + if regex_options.match is not None: + maybe_capture = regex_options.match.match_any( + input_str=entry_variable_dict[source_var] + ) + + # And nothing matched + if maybe_capture is None: + # and no defaults + if not regex_options.has_defaults: + return self._try_skip_entry(entry=entry, source_var=source_var) + + # otherwise, use defaults (apply them using the original entry source dict) + source_variables_and_overrides_dict = dict( + entry_variable_dict, **self.overrides.dict_with_format_strings ) - # otherwise, use defaults (apply them using the original entry source dict) - source_variables_and_overrides_dict = dict( - entry_variable_dict, **self.overrides.dict_with_format_strings - ) - - # add both the default... - entry.add_variables( - variables_to_add={ - regex_options.capture_group_names[i]: default.apply_formatter( - variable_dict=source_variables_and_overrides_dict - ) - for i, default in enumerate(regex_options.capture_group_defaults) - }, - ) - # and sanitized default - entry.add_variables( - variables_to_add={ - f"{regex_options.capture_group_names[i]}_sanitized": sanitize_filename( - default.apply_formatter( + # add both the default... + entry.add_variables( + variables_to_add={ + regex_options.capture_group_names[i]: default.apply_formatter( variable_dict=source_variables_and_overrides_dict ) - ) - for i, default in enumerate(regex_options.capture_group_defaults) - }, - ) - # There is a capture, add the source variables to the entry as - # {source_var}_capture_1, {source_var}_capture_2, ... - else: - # Add the value... - entry.add_variables( - variables_to_add={ - regex_options.capture_group_names[i]: capture - for i, capture in enumerate(maybe_capture) - }, - ) - # And the sanitized value - entry.add_variables( - variables_to_add={ - f"{regex_options.capture_group_names[i]}_sanitized": sanitize_filename( - capture - ) - for i, capture in enumerate(maybe_capture) - }, - ) + for i, default in enumerate(regex_options.capture_group_defaults) + }, + ) + # and sanitized default + entry.add_variables( + variables_to_add={ + f"{regex_options.capture_group_names[i]}_sanitized": sanitize_filename( + default.apply_formatter( + variable_dict=source_variables_and_overrides_dict + ) + ) + for i, default in enumerate(regex_options.capture_group_defaults) + }, + ) + # There is a capture, add the source variables to the entry as + # {source_var}_capture_1, {source_var}_capture_2, ... + else: + # Add the value... + entry.add_variables( + variables_to_add={ + regex_options.capture_group_names[i]: capture + for i, capture in enumerate(maybe_capture) + }, + ) + # And the sanitized value + entry.add_variables( + variables_to_add={ + f"{regex_options.capture_group_names[i]}_sanitized": sanitize_filename( + capture + ) + for i, capture in enumerate(maybe_capture) + }, + ) return entry diff --git a/tests/e2e/plugins/test_regex.py b/tests/e2e/plugins/test_regex.py index fa32d5ab..fc4bd6b4 100644 --- a/tests/e2e/plugins/test_regex.py +++ b/tests/e2e/plugins/test_regex.py @@ -1,17 +1,20 @@ import copy import re +from typing import Any +from typing import Dict +import mergedeep import pytest from expected_transaction_log import assert_transaction_log_matches -from ytdl_sub.config.preset import Preset +from ytdl_sub.config.config_file import ConfigFile from ytdl_sub.subscriptions.subscription import Subscription from ytdl_sub.utils.exceptions import RegexNoMatchException from ytdl_sub.utils.exceptions import ValidationException @pytest.fixture -def regex_subscription_dict(output_directory): +def regex_subscription_dict_base(output_directory): return { "preset": "music_video", "download": {"url": "https://youtube.com/playlist?list=PL5BC0FC26BECA5A35"}, @@ -24,13 +27,6 @@ def regex_subscription_dict(output_directory): "regex": { # tests that skip_if_match_fails defaults to True "from": { - "title": { - "match": [ - "should not cap (.+) - (.+)", - ".*\\[(.+) - (Feb.+)]", # should filter out march video - ], - "capture_group_names": ["title_type", "title_date"], - }, "description": { "match": [".*http:\\/\\/(.+).com.*"], "capture_group_names": ["description_website"], @@ -48,58 +44,149 @@ def regex_subscription_dict(output_directory): }, }, }, - "nfo_tags": { - "tags": { - "title_cap_1": "{title_type}", - "title_cap_1_sanitized": "{title_type_sanitized}", - "title_cap_2": "{title_date}", - "desc_cap": "{description_website}", - "upload_date_both_caps": "{upload_captured_year} and {upload_captured_month}", - "override_with_capture_variable": "{contains_regex_default}", - "override_with_capture_variable_sanitized": "{contains_regex_sanitized_default}", - } - }, "overrides": { "in_regex_default": "in regex default", - "contains_regex_default": "contains {title_type}", - "contains_regex_sanitized_default": "contains {title_type_sanitized}", }, } @pytest.fixture -def regex_subscription_dict_no_match_fails(regex_subscription_dict): - regex_subscription_dict["regex"]["skip_if_match_fails"] = False - return regex_subscription_dict +def regex_subscription_dict(regex_subscription_dict_base, output_directory): + return mergedeep.merge( + regex_subscription_dict_base, + { + "regex": { + # tests that skip_if_match_fails defaults to True + "from": { + "title": { + "match": [ + "should not cap (.+) - (.+)", + ".*\\[(.+) - (Feb.+)]", # should filter out march video + ], + "capture_group_names": ["title_type", "title_date"], + }, + }, + }, + "nfo_tags": { + "tags": { + "title_cap_1": "{title_type}", + "title_cap_1_sanitized": "{title_type_sanitized}", + "title_cap_2": "{title_date}", + "desc_cap": "{description_website}", + "upload_date_both_caps": "{upload_captured_year} and {upload_captured_month}", + "override_with_capture_variable": "{contains_regex_default}", + "override_with_capture_variable_sanitized": "{contains_regex_sanitized_default}", + } + }, + "overrides": { + "contains_regex_default": "contains {title_type}", + "contains_regex_sanitized_default": "contains {title_type_sanitized}", + }, + }, + ) + + +@pytest.fixture +def regex_subscription_dict_exclude(regex_subscription_dict_base, output_directory): + return mergedeep.merge( + regex_subscription_dict_base, + { + "regex": { + # tests that skip_if_match_fails defaults to True + "from": { + "title": { + "exclude": [ + "should not cap", + ".*Feb.*", # should filter out march video + ], + }, + }, + }, + }, + ) + + +@pytest.fixture +def regex_subscription_dict_match_and_exclude(regex_subscription_dict_base, output_directory): + return mergedeep.merge( + regex_subscription_dict_base, + { + "regex": { + # tests that skip_if_match_fails defaults to True + "from": { + "title": { + "match": [ + "should not cap (.+) - (.+)", + ".*\\[(.+) - (Feb.+)]", # should filter out march video + ], + "capture_group_names": ["title_type", "title_date"], + "exclude": [ + "should not cap", + ".*27.*", # should filter out Feb 27th video + ], + }, + }, + }, + "nfo_tags": { + "tags": { + "title_cap_1": "{title_type}", + "title_cap_1_sanitized": "{title_type_sanitized}", + "title_cap_2": "{title_date}", + "desc_cap": "{description_website}", + "upload_date_both_caps": "{upload_captured_year} and {upload_captured_month}", + "override_with_capture_variable": "{contains_regex_default}", + "override_with_capture_variable_sanitized": "{contains_regex_sanitized_default}", + } + }, + "overrides": { + "contains_regex_default": "contains {title_type}", + "contains_regex_sanitized_default": "contains {title_type_sanitized}", + }, + }, + ) @pytest.fixture def playlist_subscription(music_video_config, regex_subscription_dict): - playlist_preset = Preset.from_dict( + return Subscription.from_dict( config=music_video_config, preset_name="regex_capture_playlist_test", preset_dict=regex_subscription_dict, ) - return Subscription.from_preset( - preset=playlist_preset, + +@pytest.fixture +def playlist_subscription_no_match_fails( + music_video_config: ConfigFile, regex_subscription_dict: Dict[str, Any] +): + regex_subscription_dict["regex"]["skip_if_match_fails"] = False + + return Subscription.from_dict( config=music_video_config, + preset_name="regex_capture_playlist_test", + preset_dict=regex_subscription_dict, ) @pytest.fixture -def playlist_subscription_no_match_fails( - music_video_config, regex_subscription_dict_no_match_fails -): - playlist_preset = Preset.from_dict( +def playlist_subscription_exclude( + music_video_config: ConfigFile, regex_subscription_dict_exclude: Dict[str, Any] +) -> Subscription: + return Subscription.from_dict( config=music_video_config, - preset_name="regex_capture_playlist_test", - preset_dict=regex_subscription_dict_no_match_fails, + preset_name="regex_exclude_playlist_test", + preset_dict=regex_subscription_dict_exclude, ) - return Subscription.from_preset( - preset=playlist_preset, + +@pytest.fixture +def playlist_subscription_match_and_exclude( + music_video_config: ConfigFile, regex_subscription_dict_match_and_exclude: Dict[str, Any] +) -> Subscription: + return Subscription.from_dict( config=music_video_config, + preset_name="regex_match_and_exclude_playlist_test", + preset_dict=regex_subscription_dict_match_and_exclude, ) @@ -113,6 +200,27 @@ class TestRegex: transaction_log_summary_file_name="plugins/test_regex.txt", ) + def test_regex_excludes_success(self, playlist_subscription_exclude, output_directory): + # Should only contain the march video + transaction_log = playlist_subscription_exclude.download(dry_run=True) + assert_transaction_log_matches( + output_directory=output_directory, + transaction_log=transaction_log, + transaction_log_summary_file_name="plugins/test_regex_exclude.txt", + ) + + def test_regex_match_and_excludes_success( + self, playlist_subscription_match_and_exclude, output_directory + ): + # Should only contain the Feb 1st video + transaction_log = playlist_subscription_match_and_exclude.download(dry_run=True) + assert_transaction_log_matches( + output_directory=output_directory, + transaction_log=transaction_log, + transaction_log_summary_file_name="plugins/test_regex_match_and_exclude.txt", + regenerate_transaction_log=True, + ) + def test_regex_fails_no_match(self, playlist_subscription_no_match_fails, output_directory): with pytest.raises( RegexNoMatchException, @@ -122,6 +230,34 @@ class TestRegex: ): _ = playlist_subscription_no_match_fails.download(dry_run=True) + def test_regex_fails_capture_group_with_only_excludes( + self, regex_subscription_dict_exclude, music_video_config + ): + regex_subscription_dict_exclude["regex"]["from"]["title"]["capture_group_names"] = ["uid"] + with pytest.raises( + ValidationException, + match=re.escape( + "capture group parameters requires at least one `match` to be specified" + ), + ): + _ = Subscription.from_dict( + config=music_video_config, + preset_name="test_regex_fails_capture_group_is_source_variable", + preset_dict=regex_subscription_dict_exclude, + ) + + def test_regex_fails_no_match_or_exclude(self, regex_subscription_dict, music_video_config): + del regex_subscription_dict["regex"]["from"]["title"]["match"] + with pytest.raises( + ValidationException, + match=re.escape("must specify either `match` or `exclude`"), + ): + _ = Subscription.from_dict( + config=music_video_config, + preset_name="test_regex_fails_capture_group_is_source_variable", + preset_dict=regex_subscription_dict, + ) + def test_regex_fails_capture_group_is_source_variable( self, regex_subscription_dict, music_video_config ): diff --git a/tests/resources/transaction_log_summaries/plugins/test_regex_exclude.txt b/tests/resources/transaction_log_summaries/plugins/test_regex_exclude.txt new file mode 100644 index 00000000..f82556b6 --- /dev/null +++ b/tests/resources/transaction_log_summaries/plugins/test_regex_exclude.txt @@ -0,0 +1,15 @@ +Files created: +---------------------------------------- +{output_directory} + .ytdl-sub-regex_exclude_playlist_test-download-archive.json +{output_directory}/Project Zombie + Project Zombie - Jesse's Minecraft Server [Trailer - Mar.21]-thumb.jpg + Project Zombie - Jesse's Minecraft Server [Trailer - Mar.21].info.json + Project Zombie - Jesse's Minecraft Server [Trailer - Mar.21].mp4 + Project Zombie - Jesse's Minecraft Server [Trailer - Mar.21].nfo + NFO tags: + musicvideo: + album: Music Videos + artist: Project Zombie + title: Jesse's Minecraft Server [Trailer - Mar.21] + year: 2011 \ No newline at end of file diff --git a/tests/resources/transaction_log_summaries/plugins/test_regex_match_and_exclude.txt b/tests/resources/transaction_log_summaries/plugins/test_regex_match_and_exclude.txt new file mode 100644 index 00000000..6a23ffab --- /dev/null +++ b/tests/resources/transaction_log_summaries/plugins/test_regex_match_and_exclude.txt @@ -0,0 +1,22 @@ +Files created: +---------------------------------------- +{output_directory} + .ytdl-sub-regex_match_and_exclude_playlist_test-download-archive.json +{output_directory}/Project Zombie + Project Zombie - Jesse's Minecraft Server [Trailer - Feb.1]-thumb.jpg + Project Zombie - Jesse's Minecraft Server [Trailer - Feb.1].info.json + Project Zombie - Jesse's Minecraft Server [Trailer - Feb.1].mp4 + Project Zombie - Jesse's Minecraft Server [Trailer - Feb.1].nfo + NFO tags: + musicvideo: + album: Music Videos + artist: Project Zombie + desc_cap: www.jesseminecraft.webs + override_with_capture_variable: contains Trailer + override_with_capture_variable_sanitized: contains Trailer + title: Jesse's Minecraft Server [Trailer - Feb.1] + title_cap_1: Trailer + title_cap_1_sanitized: Trailer + title_cap_2: Feb.1 + upload_date_both_caps: First and Second containing in regex default + year: 2011 \ No newline at end of file