diff --git a/src/ytdl_sub/plugins/regex.py b/src/ytdl_sub/plugins/regex.py index ec0b666f..1e7a5b03 100644 --- a/src/ytdl_sub/plugins/regex.py +++ b/src/ytdl_sub/plugins/regex.py @@ -40,7 +40,7 @@ class SourceVariableRegex(StrictDictValidator): if self._match is None and self._exclude is None: raise self._validation_exception("must specify either `match` or `exclude`") - if self._match is None and (self._capture_group_defaults or self._capture_group_names): + if self._match is None and (self._capture_group_defaults or self._capture_group_names.list): raise self._validation_exception( "capture group parameters requires at least one `match` to be specified" ) @@ -56,7 +56,7 @@ class SourceVariableRegex(StrictDictValidator): ) # If there are capture groups, ensure there are capture group names - if len(self._capture_group_names.list) != self._match.num_capture_groups: + if self._match and (len(self._capture_group_names.list) != self._match.num_capture_groups): raise self._validation_exception( f"number of capture group names must match number of capture groups, " f"{len(self._capture_group_names.list)} != {self._match.num_capture_groups}" @@ -66,7 +66,7 @@ class SourceVariableRegex(StrictDictValidator): def match(self) -> Optional[RegexListValidator]: """ List of regex strings to try to match against a source variable. Each regex - string must have the same number of capture groups. Can not be used with ``match`` + string must have the same number of capture groups. """ return self._match @@ -74,7 +74,8 @@ class SourceVariableRegex(StrictDictValidator): def exclude(self) -> Optional[RegexListValidator]: """ List of regex strings to try to match against a source variable. If one of the regex strings - match, then the entry will be skipped. + match, then the entry will be skipped. If both ``exclude`` and ``match`` are specified, + entries will get skipped if the regex matches against both ``exclude`` and ``match``. """ return self._exclude @@ -332,62 +333,62 @@ class RegexPlugin(Plugin[RegexOptions]): ): return self._try_skip_entry(entry=entry, source_var=source_var) - maybe_capture: Optional[List[str]] = None + # If match is present if regex_options.match is not None: maybe_capture = regex_options.match.match_any( input_str=entry_variable_dict[source_var] ) - # If no capture - if maybe_capture is None: - # and no defaults - if not regex_options.has_defaults: - return self._try_skip_entry(entry=entry, source_var=source_var) + # And nothing matched + if maybe_capture is None: + # and no defaults + if not regex_options.has_defaults: + return self._try_skip_entry(entry=entry, source_var=source_var) - # otherwise, use defaults (apply them using the original entry source dict) - source_variables_and_overrides_dict = dict( - entry_variable_dict, **self.overrides.dict_with_format_strings - ) + # otherwise, use defaults (apply them using the original entry source dict) + source_variables_and_overrides_dict = dict( + entry_variable_dict, **self.overrides.dict_with_format_strings + ) - # add both the default... - entry.add_variables( - variables_to_add={ - regex_options.capture_group_names[i]: default.apply_formatter( - variable_dict=source_variables_and_overrides_dict - ) - for i, default in enumerate(regex_options.capture_group_defaults) - }, - ) - # and sanitized default - entry.add_variables( - variables_to_add={ - f"{regex_options.capture_group_names[i]}_sanitized": sanitize_filename( - default.apply_formatter( + # add both the default... + entry.add_variables( + variables_to_add={ + regex_options.capture_group_names[i]: default.apply_formatter( variable_dict=source_variables_and_overrides_dict ) - ) - for i, default in enumerate(regex_options.capture_group_defaults) - }, - ) - # There is a capture, add the source variables to the entry as - # {source_var}_capture_1, {source_var}_capture_2, ... - else: - # Add the value... - entry.add_variables( - variables_to_add={ - regex_options.capture_group_names[i]: capture - for i, capture in enumerate(maybe_capture) - }, - ) - # And the sanitized value - entry.add_variables( - variables_to_add={ - f"{regex_options.capture_group_names[i]}_sanitized": sanitize_filename( - capture - ) - for i, capture in enumerate(maybe_capture) - }, - ) + for i, default in enumerate(regex_options.capture_group_defaults) + }, + ) + # and sanitized default + entry.add_variables( + variables_to_add={ + f"{regex_options.capture_group_names[i]}_sanitized": sanitize_filename( + default.apply_formatter( + variable_dict=source_variables_and_overrides_dict + ) + ) + for i, default in enumerate(regex_options.capture_group_defaults) + }, + ) + # There is a capture, add the source variables to the entry as + # {source_var}_capture_1, {source_var}_capture_2, ... + else: + # Add the value... + entry.add_variables( + variables_to_add={ + regex_options.capture_group_names[i]: capture + for i, capture in enumerate(maybe_capture) + }, + ) + # And the sanitized value + entry.add_variables( + variables_to_add={ + f"{regex_options.capture_group_names[i]}_sanitized": sanitize_filename( + capture + ) + for i, capture in enumerate(maybe_capture) + }, + ) return entry diff --git a/tests/e2e/plugins/test_regex.py b/tests/e2e/plugins/test_regex.py index fa32d5ab..fc4bd6b4 100644 --- a/tests/e2e/plugins/test_regex.py +++ b/tests/e2e/plugins/test_regex.py @@ -1,17 +1,20 @@ import copy import re +from typing import Any +from typing import Dict +import mergedeep import pytest from expected_transaction_log import assert_transaction_log_matches -from ytdl_sub.config.preset import Preset +from ytdl_sub.config.config_file import ConfigFile from ytdl_sub.subscriptions.subscription import Subscription from ytdl_sub.utils.exceptions import RegexNoMatchException from ytdl_sub.utils.exceptions import ValidationException @pytest.fixture -def regex_subscription_dict(output_directory): +def regex_subscription_dict_base(output_directory): return { "preset": "music_video", "download": {"url": "https://youtube.com/playlist?list=PL5BC0FC26BECA5A35"}, @@ -24,13 +27,6 @@ def regex_subscription_dict(output_directory): "regex": { # tests that skip_if_match_fails defaults to True "from": { - "title": { - "match": [ - "should not cap (.+) - (.+)", - ".*\\[(.+) - (Feb.+)]", # should filter out march video - ], - "capture_group_names": ["title_type", "title_date"], - }, "description": { "match": [".*http:\\/\\/(.+).com.*"], "capture_group_names": ["description_website"], @@ -48,58 +44,149 @@ def regex_subscription_dict(output_directory): }, }, }, - "nfo_tags": { - "tags": { - "title_cap_1": "{title_type}", - "title_cap_1_sanitized": "{title_type_sanitized}", - "title_cap_2": "{title_date}", - "desc_cap": "{description_website}", - "upload_date_both_caps": "{upload_captured_year} and {upload_captured_month}", - "override_with_capture_variable": "{contains_regex_default}", - "override_with_capture_variable_sanitized": "{contains_regex_sanitized_default}", - } - }, "overrides": { "in_regex_default": "in regex default", - "contains_regex_default": "contains {title_type}", - "contains_regex_sanitized_default": "contains {title_type_sanitized}", }, } @pytest.fixture -def regex_subscription_dict_no_match_fails(regex_subscription_dict): - regex_subscription_dict["regex"]["skip_if_match_fails"] = False - return regex_subscription_dict +def regex_subscription_dict(regex_subscription_dict_base, output_directory): + return mergedeep.merge( + regex_subscription_dict_base, + { + "regex": { + # tests that skip_if_match_fails defaults to True + "from": { + "title": { + "match": [ + "should not cap (.+) - (.+)", + ".*\\[(.+) - (Feb.+)]", # should filter out march video + ], + "capture_group_names": ["title_type", "title_date"], + }, + }, + }, + "nfo_tags": { + "tags": { + "title_cap_1": "{title_type}", + "title_cap_1_sanitized": "{title_type_sanitized}", + "title_cap_2": "{title_date}", + "desc_cap": "{description_website}", + "upload_date_both_caps": "{upload_captured_year} and {upload_captured_month}", + "override_with_capture_variable": "{contains_regex_default}", + "override_with_capture_variable_sanitized": "{contains_regex_sanitized_default}", + } + }, + "overrides": { + "contains_regex_default": "contains {title_type}", + "contains_regex_sanitized_default": "contains {title_type_sanitized}", + }, + }, + ) + + +@pytest.fixture +def regex_subscription_dict_exclude(regex_subscription_dict_base, output_directory): + return mergedeep.merge( + regex_subscription_dict_base, + { + "regex": { + # tests that skip_if_match_fails defaults to True + "from": { + "title": { + "exclude": [ + "should not cap", + ".*Feb.*", # should filter out march video + ], + }, + }, + }, + }, + ) + + +@pytest.fixture +def regex_subscription_dict_match_and_exclude(regex_subscription_dict_base, output_directory): + return mergedeep.merge( + regex_subscription_dict_base, + { + "regex": { + # tests that skip_if_match_fails defaults to True + "from": { + "title": { + "match": [ + "should not cap (.+) - (.+)", + ".*\\[(.+) - (Feb.+)]", # should filter out march video + ], + "capture_group_names": ["title_type", "title_date"], + "exclude": [ + "should not cap", + ".*27.*", # should filter out Feb 27th video + ], + }, + }, + }, + "nfo_tags": { + "tags": { + "title_cap_1": "{title_type}", + "title_cap_1_sanitized": "{title_type_sanitized}", + "title_cap_2": "{title_date}", + "desc_cap": "{description_website}", + "upload_date_both_caps": "{upload_captured_year} and {upload_captured_month}", + "override_with_capture_variable": "{contains_regex_default}", + "override_with_capture_variable_sanitized": "{contains_regex_sanitized_default}", + } + }, + "overrides": { + "contains_regex_default": "contains {title_type}", + "contains_regex_sanitized_default": "contains {title_type_sanitized}", + }, + }, + ) @pytest.fixture def playlist_subscription(music_video_config, regex_subscription_dict): - playlist_preset = Preset.from_dict( + return Subscription.from_dict( config=music_video_config, preset_name="regex_capture_playlist_test", preset_dict=regex_subscription_dict, ) - return Subscription.from_preset( - preset=playlist_preset, + +@pytest.fixture +def playlist_subscription_no_match_fails( + music_video_config: ConfigFile, regex_subscription_dict: Dict[str, Any] +): + regex_subscription_dict["regex"]["skip_if_match_fails"] = False + + return Subscription.from_dict( config=music_video_config, + preset_name="regex_capture_playlist_test", + preset_dict=regex_subscription_dict, ) @pytest.fixture -def playlist_subscription_no_match_fails( - music_video_config, regex_subscription_dict_no_match_fails -): - playlist_preset = Preset.from_dict( +def playlist_subscription_exclude( + music_video_config: ConfigFile, regex_subscription_dict_exclude: Dict[str, Any] +) -> Subscription: + return Subscription.from_dict( config=music_video_config, - preset_name="regex_capture_playlist_test", - preset_dict=regex_subscription_dict_no_match_fails, + preset_name="regex_exclude_playlist_test", + preset_dict=regex_subscription_dict_exclude, ) - return Subscription.from_preset( - preset=playlist_preset, + +@pytest.fixture +def playlist_subscription_match_and_exclude( + music_video_config: ConfigFile, regex_subscription_dict_match_and_exclude: Dict[str, Any] +) -> Subscription: + return Subscription.from_dict( config=music_video_config, + preset_name="regex_match_and_exclude_playlist_test", + preset_dict=regex_subscription_dict_match_and_exclude, ) @@ -113,6 +200,27 @@ class TestRegex: transaction_log_summary_file_name="plugins/test_regex.txt", ) + def test_regex_excludes_success(self, playlist_subscription_exclude, output_directory): + # Should only contain the march video + transaction_log = playlist_subscription_exclude.download(dry_run=True) + assert_transaction_log_matches( + output_directory=output_directory, + transaction_log=transaction_log, + transaction_log_summary_file_name="plugins/test_regex_exclude.txt", + ) + + def test_regex_match_and_excludes_success( + self, playlist_subscription_match_and_exclude, output_directory + ): + # Should only contain the Feb 1st video + transaction_log = playlist_subscription_match_and_exclude.download(dry_run=True) + assert_transaction_log_matches( + output_directory=output_directory, + transaction_log=transaction_log, + transaction_log_summary_file_name="plugins/test_regex_match_and_exclude.txt", + regenerate_transaction_log=True, + ) + def test_regex_fails_no_match(self, playlist_subscription_no_match_fails, output_directory): with pytest.raises( RegexNoMatchException, @@ -122,6 +230,34 @@ class TestRegex: ): _ = playlist_subscription_no_match_fails.download(dry_run=True) + def test_regex_fails_capture_group_with_only_excludes( + self, regex_subscription_dict_exclude, music_video_config + ): + regex_subscription_dict_exclude["regex"]["from"]["title"]["capture_group_names"] = ["uid"] + with pytest.raises( + ValidationException, + match=re.escape( + "capture group parameters requires at least one `match` to be specified" + ), + ): + _ = Subscription.from_dict( + config=music_video_config, + preset_name="test_regex_fails_capture_group_is_source_variable", + preset_dict=regex_subscription_dict_exclude, + ) + + def test_regex_fails_no_match_or_exclude(self, regex_subscription_dict, music_video_config): + del regex_subscription_dict["regex"]["from"]["title"]["match"] + with pytest.raises( + ValidationException, + match=re.escape("must specify either `match` or `exclude`"), + ): + _ = Subscription.from_dict( + config=music_video_config, + preset_name="test_regex_fails_capture_group_is_source_variable", + preset_dict=regex_subscription_dict, + ) + def test_regex_fails_capture_group_is_source_variable( self, regex_subscription_dict, music_video_config ): diff --git a/tests/resources/transaction_log_summaries/plugins/test_regex_exclude.txt b/tests/resources/transaction_log_summaries/plugins/test_regex_exclude.txt new file mode 100644 index 00000000..f82556b6 --- /dev/null +++ b/tests/resources/transaction_log_summaries/plugins/test_regex_exclude.txt @@ -0,0 +1,15 @@ +Files created: +---------------------------------------- +{output_directory} + .ytdl-sub-regex_exclude_playlist_test-download-archive.json +{output_directory}/Project Zombie + Project Zombie - Jesse's Minecraft Server [Trailer - Mar.21]-thumb.jpg + Project Zombie - Jesse's Minecraft Server [Trailer - Mar.21].info.json + Project Zombie - Jesse's Minecraft Server [Trailer - Mar.21].mp4 + Project Zombie - Jesse's Minecraft Server [Trailer - Mar.21].nfo + NFO tags: + musicvideo: + album: Music Videos + artist: Project Zombie + title: Jesse's Minecraft Server [Trailer - Mar.21] + year: 2011 \ No newline at end of file diff --git a/tests/resources/transaction_log_summaries/plugins/test_regex_match_and_exclude.txt b/tests/resources/transaction_log_summaries/plugins/test_regex_match_and_exclude.txt new file mode 100644 index 00000000..6a23ffab --- /dev/null +++ b/tests/resources/transaction_log_summaries/plugins/test_regex_match_and_exclude.txt @@ -0,0 +1,22 @@ +Files created: +---------------------------------------- +{output_directory} + .ytdl-sub-regex_match_and_exclude_playlist_test-download-archive.json +{output_directory}/Project Zombie + Project Zombie - Jesse's Minecraft Server [Trailer - Feb.1]-thumb.jpg + Project Zombie - Jesse's Minecraft Server [Trailer - Feb.1].info.json + Project Zombie - Jesse's Minecraft Server [Trailer - Feb.1].mp4 + Project Zombie - Jesse's Minecraft Server [Trailer - Feb.1].nfo + NFO tags: + musicvideo: + album: Music Videos + artist: Project Zombie + desc_cap: www.jesseminecraft.webs + override_with_capture_variable: contains Trailer + override_with_capture_variable_sanitized: contains Trailer + title: Jesse's Minecraft Server [Trailer - Feb.1] + title_cap_1: Trailer + title_cap_1_sanitized: Trailer + title_cap_2: Feb.1 + upload_date_both_caps: First and Second containing in regex default + year: 2011 \ No newline at end of file