From 3ca6af38dc5964adc960e45ccd18d48d30581ef6 Mon Sep 17 00:00:00 2001 From: jbannon Date: Thu, 14 Jul 2022 05:38:15 +0000 Subject: [PATCH] refactor plugin name, matching now working --- src/ytdl_sub/config/preset_class_mappings.py | 4 +- .../plugins/{regex_capture.py => regex.py} | 32 ++++++------ src/ytdl_sub/subscriptions/subscription.py | 8 ++- src/ytdl_sub/utils/exceptions.py | 4 ++ src/ytdl_sub/validators/regex_validator.py | 9 ++-- .../{test_regex_capture.py => test_regex.py} | 52 +++++++++++++++---- ...{test_regex_mapping.txt => test_regex.txt} | 14 ----- tests/unit/validators/test_regex_validator.py | 25 ++++----- 8 files changed, 88 insertions(+), 60 deletions(-) rename src/ytdl_sub/plugins/{regex_capture.py => regex.py} (88%) rename tests/e2e/plugins/{test_regex_capture.py => test_regex.py} (60%) rename tests/e2e/resources/transaction_log_summaries/plugins/{test_regex_mapping.txt => test_regex.txt} (69%) diff --git a/src/ytdl_sub/config/preset_class_mappings.py b/src/ytdl_sub/config/preset_class_mappings.py index 878faa46..e6d7bf2d 100644 --- a/src/ytdl_sub/config/preset_class_mappings.py +++ b/src/ytdl_sub/config/preset_class_mappings.py @@ -13,7 +13,7 @@ from ytdl_sub.plugins.music_tags import MusicTagsPlugin from ytdl_sub.plugins.nfo_tags import NfoTagsPlugin from ytdl_sub.plugins.output_directory_nfo_tags import OutputDirectoryNfoTagsPlugin from ytdl_sub.plugins.plugin import Plugin -from ytdl_sub.plugins.regex_capture import RegexCapturePlugin +from ytdl_sub.plugins.regex import RegexPlugin class DownloadStrategyMapping: @@ -108,7 +108,7 @@ class PluginMapping: "music_tags": MusicTagsPlugin, "nfo_tags": NfoTagsPlugin, "output_directory_nfo_tags": OutputDirectoryNfoTagsPlugin, - "regex": RegexCapturePlugin, + "regex": RegexPlugin, } @classmethod diff --git a/src/ytdl_sub/plugins/regex_capture.py b/src/ytdl_sub/plugins/regex.py similarity index 88% rename from src/ytdl_sub/plugins/regex_capture.py rename to src/ytdl_sub/plugins/regex.py index aba5e8fc..a3a0d03b 100644 --- a/src/ytdl_sub/plugins/regex_capture.py +++ b/src/ytdl_sub/plugins/regex.py @@ -5,7 +5,7 @@ from typing import Optional from ytdl_sub.entries.entry import Entry from ytdl_sub.plugins.plugin import Plugin from ytdl_sub.plugins.plugin import PluginOptions -from ytdl_sub.utils.exceptions import ValidationException +from ytdl_sub.utils.exceptions import RegexNoMatchException from ytdl_sub.utils.logger import Logger from ytdl_sub.validators.regex_validator import RegexListValidator from ytdl_sub.validators.strict_dict_validator import StrictDictValidator @@ -20,7 +20,7 @@ def _source_var_name(source_variable: str, capture_group_idx: int) -> str: return f"{source_variable}_capture_{capture_group_idx+1}" -class SourceVariableRegexCapture(StrictDictValidator): +class SourceVariableRegex(StrictDictValidator): _required_keys = {"match"} _optional_keys = {"defaults"} @@ -70,27 +70,26 @@ class SourceVariableRegexCapture(StrictDictValidator): return self._defaults.list if self.has_defaults else None -class FromSourceVariablesRegexCapture(StrictDictValidator): +class FromSourceVariablesRegex(StrictDictValidator): _optional_keys = Entry.source_variables() _allow_extra_keys = True def __init__(self, name, value): super().__init__(name, value) - self.source_variable_capture_dict: Dict[str, SourceVariableRegexCapture] = { - key: self._validate_key(key=key, validator=SourceVariableRegexCapture) - for key in self._keys + self.source_variable_capture_dict: Dict[str, SourceVariableRegex] = { + key: self._validate_key(key=key, validator=SourceVariableRegex) for key in self._keys } -class RegexCaptureOptions(PluginOptions): +class RegexOptions(PluginOptions): _required_keys = {"from"} _optional_keys = {"skip_if_match_fails"} def __init__(self, name, value): super().__init__(name, value) - self._from = self._validate_key(key="from", validator=FromSourceVariablesRegexCapture) + self._from = self._validate_key(key="from", validator=FromSourceVariablesRegex) self.skip_if_match_fails: bool = self._validate_key_if_present( key="skip_if_match_fails", validator=BoolValidator, default=False ).value @@ -111,7 +110,7 @@ class RegexCaptureOptions(PluginOptions): ) @property - def source_variable_capture_dict(self) -> Dict[str, SourceVariableRegexCapture]: + def source_variable_capture_dict(self) -> Dict[str, SourceVariableRegex]: """ Returns ------- @@ -135,8 +134,8 @@ class RegexCaptureOptions(PluginOptions): return added_source_vars -class RegexCapturePlugin(Plugin[RegexCaptureOptions]): - plugin_options_type = RegexCaptureOptions +class RegexPlugin(Plugin[RegexOptions]): + plugin_options_type = RegexOptions def modify_entry(self, entry: Entry) -> Optional[Entry]: """ @@ -158,7 +157,7 @@ class RegexCapturePlugin(Plugin[RegexCaptureOptions]): # Iterate each source var to capture and add to the entry for source_var, regex_options in self.plugin_options.source_variable_capture_dict.items(): - maybe_capture = regex_options.capture_list.capture_any( + maybe_capture = regex_options.capture_list.match_any( input_str=entry_variable_dict[source_var] ) @@ -169,14 +168,15 @@ class RegexCapturePlugin(Plugin[RegexCaptureOptions]): # Skip the entry if toggled if self.plugin_options.skip_if_match_fails: logger.info( - "Entry with title '%s' failed to match regex, skipping.", entry.title + "Regex failed to match '%s' from '%s', skipping.", + source_var, + entry.title, ) return None # Otherwise, error - raise ValidationException( - f"Failed to capture {source_var} from an entry with the value:\n" - f"{entry_variable_dict[source_var]}" + raise RegexNoMatchException( + f"Regex failed to match '{source_var}' from '{entry.title}'" ) # otherwise, use defaults (apply them using the original entry source dict) diff --git a/src/ytdl_sub/subscriptions/subscription.py b/src/ytdl_sub/subscriptions/subscription.py index f128c4e9..e15e26d8 100644 --- a/src/ytdl_sub/subscriptions/subscription.py +++ b/src/ytdl_sub/subscriptions/subscription.py @@ -273,7 +273,13 @@ class Subscription: # First, modify the entry with all plugins for plugin in plugins: - entry = plugin.modify_entry(entry) + # Break out of this plugin loop if entry is None, it is indicated to not DL it + if (entry := plugin.modify_entry(entry)) is None: + break + + # If entry is None from the broken out loop, continue over the other entries + if entry is None: + continue # Then, post-process the entry with all plugins for plugin in plugins: diff --git a/src/ytdl_sub/utils/exceptions.py b/src/ytdl_sub/utils/exceptions.py index 9574c430..1a8ac4ff 100644 --- a/src/ytdl_sub/utils/exceptions.py +++ b/src/ytdl_sub/utils/exceptions.py @@ -20,3 +20,7 @@ class FileNotFoundException(ValidationException): class InvalidYamlException(ValidationException): """User yaml that is invalid""" + + +class RegexNoMatchException(ValidationException): + """Regex failed to match during download""" diff --git a/src/ytdl_sub/validators/regex_validator.py b/src/ytdl_sub/validators/regex_validator.py index 8a6fd79c..c9087b20 100644 --- a/src/ytdl_sub/validators/regex_validator.py +++ b/src/ytdl_sub/validators/regex_validator.py @@ -28,7 +28,7 @@ class RegexValidator(StringValidator): """ return self._compiled_regex.groups - def capture(self, input_str: str) -> Optional[List[str]]: + def match(self, input_str: str) -> Optional[List[str]]: """ Parameters ---------- @@ -68,7 +68,7 @@ class RegexListValidator(ListValidator[RegexValidator]): """ return self._num_capture_groups - def capture_any(self, input_str: str) -> Optional[List[str]]: + def match_any(self, input_str: str) -> Optional[List[str]]: """ Parameters ---------- @@ -77,9 +77,10 @@ class RegexListValidator(ListValidator[RegexValidator]): Returns ------- - List of captures on the first regex that matches. None if no regexes match. + List of captures on the first regex that matches. If the regex has no capture groups, then + the list will be emtpy. None is returned if the input_str failed to match """ for reg in self._list: - if (maybe_capture := reg.capture(input_str)) is not None: + if (maybe_capture := reg.match(input_str)) is not None: return maybe_capture return None diff --git a/tests/e2e/plugins/test_regex_capture.py b/tests/e2e/plugins/test_regex.py similarity index 60% rename from tests/e2e/plugins/test_regex_capture.py rename to tests/e2e/plugins/test_regex.py index 65de1964..f8f059a7 100644 --- a/tests/e2e/plugins/test_regex_capture.py +++ b/tests/e2e/plugins/test_regex.py @@ -1,12 +1,15 @@ +import re + import pytest from e2e.expected_transaction_log import assert_transaction_log_matches from ytdl_sub.config.preset import Preset from ytdl_sub.subscriptions.subscription import Subscription +from ytdl_sub.utils.exceptions import RegexNoMatchException @pytest.fixture -def regex_capture_subscription_dict(output_directory): +def regex_subscription_dict(output_directory): return { "preset": "yt_music_video_playlist", "youtube": {"playlist_url": "https://youtube.com/playlist?list=PL5BC0FC26BECA5A35"}, @@ -15,15 +18,14 @@ def regex_capture_subscription_dict(output_directory): # download the worst format so it is fast "ytdl_options": { "format": "best[height<=480]", - "postprocessor_args": {"ffmpeg": ["-bitexact"]}, # Must add this for reproducibility }, "regex": { - "skip_if_match_fails": False, + "skip_if_match_fails": True, "from": { "title": { "match": [ "should not cap (.+) - (.+)", - ".*\\[(.+) - (.+)]", + ".*\\[(.+) - (Feb.+)]", # should filter out march video ], }, "description": {"match": [".*http:\\/\\/(.+).com.*"]}, @@ -50,11 +52,18 @@ def regex_capture_subscription_dict(output_directory): @pytest.fixture -def playlist_subscription(music_video_config, regex_capture_subscription_dict): +def regex_subscription_dict_no_match_fails(regex_subscription_dict): + # tests that skip_if_match_fails defaults to False + del regex_subscription_dict["regex"]["skip_if_match_fails"] + return regex_subscription_dict + + +@pytest.fixture +def playlist_subscription(music_video_config, regex_subscription_dict): playlist_preset = Preset.from_dict( config=music_video_config, preset_name="regex_capture_playlist_test", - preset_dict=regex_capture_subscription_dict, + preset_dict=regex_subscription_dict, ) return Subscription.from_preset( @@ -63,12 +72,37 @@ def playlist_subscription(music_video_config, regex_capture_subscription_dict): ) -class TestRegexCapture: - def test_regex_capture_success(self, playlist_subscription, output_directory): +@pytest.fixture +def playlist_subscription_no_match_fails( + music_video_config, regex_subscription_dict_no_match_fails +): + playlist_preset = Preset.from_dict( + config=music_video_config, + preset_name="regex_capture_playlist_test", + preset_dict=regex_subscription_dict_no_match_fails, + ) + + return Subscription.from_preset( + preset=playlist_preset, + config=music_video_config, + ) + + +class TestRegex: + def test_regex_success(self, playlist_subscription, output_directory): # Only dry run is needed to see if capture variables are created transaction_log = playlist_subscription.download(dry_run=True) assert_transaction_log_matches( output_directory=output_directory, transaction_log=transaction_log, - transaction_log_summary_file_name="plugins/test_regex_mapping.txt", + transaction_log_summary_file_name="plugins/test_regex.txt", ) + + def test_regex_fails_no_match(self, playlist_subscription_no_match_fails, output_directory): + with pytest.raises( + RegexNoMatchException, + match=re.escape( + "Regex failed to match 'title' from 'Jesse's Minecraft Server [Trailer - Mar.21]'" + ), + ): + _ = playlist_subscription_no_match_fails.download(dry_run=True) diff --git a/tests/e2e/resources/transaction_log_summaries/plugins/test_regex_mapping.txt b/tests/e2e/resources/transaction_log_summaries/plugins/test_regex.txt similarity index 69% rename from tests/e2e/resources/transaction_log_summaries/plugins/test_regex_mapping.txt rename to tests/e2e/resources/transaction_log_summaries/plugins/test_regex.txt index 6df1a81e..2565c3a6 100644 --- a/tests/e2e/resources/transaction_log_summaries/plugins/test_regex_mapping.txt +++ b/tests/e2e/resources/transaction_log_summaries/plugins/test_regex.txt @@ -28,18 +28,4 @@ Project Zombie - Jesse's Minecraft Server [Trailer - Feb.27].nfo title_cap_1: Trailer title_cap_2: Feb.27 upload_date_both_caps: 2011 and 02 - year: 2011 -Project Zombie - Jesse's Minecraft Server [Trailer - Mar.21]-thumb.jpg -Project Zombie - Jesse's Minecraft Server [Trailer - Mar.21].mp4 -Project Zombie - Jesse's Minecraft Server [Trailer - Mar.21].nfo - NFO tags: - musicvideo: - album: Music Videos - artist: Project Zombie - artist_cap_always_default: Always default - desc_cap: jesseminecraft.webs - title: Jesse's Minecraft Server [Trailer - Mar.21] - title_cap_1: Trailer - title_cap_2: Mar.21 - upload_date_both_caps: First and Second year: 2011 \ No newline at end of file diff --git a/tests/unit/validators/test_regex_validator.py b/tests/unit/validators/test_regex_validator.py index ab49af6c..87ecfb39 100644 --- a/tests/unit/validators/test_regex_validator.py +++ b/tests/unit/validators/test_regex_validator.py @@ -6,21 +6,20 @@ from ytdl_sub.validators.regex_validator import RegexValidator @pytest.mark.parametrize( - "regex_value, input_str, matches, captures", + "regex_value, input_str, expected_output", [ - ("^match this$", "match this", True, None), - ("^my (.+) cap", "my first cap", True, ["first"]), - (".* (.+) - (.+) two", "my other - capped two", True, ["other", "capped"]), - ("failed match", "nope", False, None), + ("^match this$", "match this", []), + ("^my (.+) cap", "my first cap", ["first"]), + (".* (.+) - (.+) two", "my other - capped two", ["other", "capped"]), + ("failed match", "nope", None), ], ) -def test_regex_validator(regex_value, input_str, matches, captures): +def test_regex_validator(regex_value, input_str, expected_output): regex_validator = RegexValidator(name="good_regex_validator", value=regex_value) assert regex_validator._name == "good_regex_validator" assert regex_validator._value == regex_value - assert regex_validator.is_match(input_str=input_str) is matches - assert regex_validator.capture(input_str=input_str) == captures + assert regex_validator.match(input_str=input_str) == expected_output @pytest.mark.parametrize("regex_value", ["(", "??"]) @@ -29,17 +28,16 @@ def test_regex_validator_fails_bad_value(regex_value): _ = RegexValidator(name="fail", value=regex_value) -def test_regex_list_validator(): +def test_regex_list_validator_matches(): regex_list_validator_raw_value = ["try matching this", "try matching that", "how about this"] regex_list = RegexListValidator(name="list val", value=regex_list_validator_raw_value) for raw_val in regex_list_validator_raw_value: - assert regex_list.matches_any(raw_val) - assert regex_list.capture_any(raw_val) is None + assert regex_list.match_any(raw_val) == [] -def test_regex_list_validator_capture(): +def test_regex_list_validator_captures(): regex_list_validator_raw_value = ["try (.+) this", "try (.+) that", "how (.+) this"] regex_list = RegexListValidator(name="list val", value=regex_list_validator_raw_value) @@ -50,8 +48,7 @@ def test_regex_list_validator_capture(): ] for capture_str, expected_capture in captures_test: - assert regex_list.matches_any(capture_str) - assert regex_list.capture_any(capture_str) == [expected_capture] + assert regex_list.match_any(capture_str) == [expected_capture] def test_regex_list_validator_invalid_regex():