From 142d87d14e919f5791f84b36642ac69a4f2b6271 Mon Sep 17 00:00:00 2001 From: jbannon Date: Sun, 17 Jul 2022 06:14:20 +0000 Subject: [PATCH] updated with name, default, better docs --- docs/config.rst | 11 +++ src/ytdl_sub/plugins/regex.py | 84 ++++++++++--------- tests/e2e/plugins/test_regex.py | 11 ++- .../test_playlist_as_kodi_music_videos.py | 8 +- 4 files changed, 66 insertions(+), 48 deletions(-) diff --git a/docs/config.rst b/docs/config.rst index 08db9eb0..521628e6 100644 --- a/docs/config.rst +++ b/docs/config.rst @@ -192,6 +192,17 @@ nfo_output_directory ------------------------------------------------------------------------------- +regex +''''' +.. autoclass:: ytdl_sub.plugins.regex.RegexOptions() + :members: skip_if_match_fails + +.. autoclass:: ytdl_sub.plugins.regex.SourceVariableRegex() + :members: match, capture_group_names, capture_group_defaults + :member-order: bysource + +------------------------------------------------------------------------------- + .. _subscription_yaml: subscription.yaml diff --git a/src/ytdl_sub/plugins/regex.py b/src/ytdl_sub/plugins/regex.py index 42dd1f6b..dcb2b5b2 100644 --- a/src/ytdl_sub/plugins/regex.py +++ b/src/ytdl_sub/plugins/regex.py @@ -20,13 +20,13 @@ logger = Logger.get(name="regex") class SourceVariableRegex(StrictDictValidator): _required_keys = {"match"} - _optional_keys = {"defaults", "capture_group_names"} + _optional_keys = {"capture_group_defaults", "capture_group_names"} def __init__(self, name, value): super().__init__(name, value) self._match = self._validate_key(key="match", validator=RegexListValidator) - self._defaults = self._validate_key_if_present( - key="defaults", validator=ListFormatterValidator + self._capture_group_defaults = self._validate_key_if_present( + key="capture_group_defaults", validator=ListFormatterValidator ) self._capture_group_names = self._validate_key_if_present( key="capture_group_names", validator=SourceVariableNameListValidator, default=[] @@ -34,12 +34,12 @@ class SourceVariableRegex(StrictDictValidator): # If defaults are to be used, ensure there are the same number of defaults as there are # capture groups - if self._defaults is not None and self._match.num_capture_groups != len( - self._defaults.list + if self._capture_group_defaults is not None and self._match.num_capture_groups != len( + self._capture_group_defaults.list ): raise self._validation_exception( f"number of defaults must match number of capture groups, " - f"{len(self._defaults.list)} != {self._match.num_capture_groups}" + f"{len(self._capture_group_defaults.list)} != {self._match.num_capture_groups}" ) # If there are capture groups, ensure there are capture group names @@ -50,14 +50,31 @@ class SourceVariableRegex(StrictDictValidator): ) @property - def capture_list(self) -> RegexListValidator: + def match(self) -> RegexListValidator: """ - Returns - ------- - List of regex captures + Required. List of regex strings to try to match against a source variable. Each regex + string must have the same number of capture groups. """ return self._match + @property + def capture_group_names(self) -> Optional[List[str]]: + """ + Optional (only when no capture groups are in the regex string). List of names to store the + capture group values to. These and ``_sanitized`` versions will be available to use as + source variables. The list's length must be equal to the number of match capture groups. + """ + return [validator.value for validator in self._capture_group_names.list] + + @property + def capture_group_defaults(self) -> Optional[List[StringFormatterValidator]]: + """ + Optional. List of string format validators to use for capture group defaults if a + source variable cannot be matched. The list's length must be equal to the number of match + capture groups. + """ + return self._capture_group_defaults.list if self.has_defaults else None + @property def has_defaults(self) -> bool: """ @@ -65,25 +82,7 @@ class SourceVariableRegex(StrictDictValidator): ------- True if a validation exception should be raised if not captured. False otherwise. """ - return self._defaults is not None - - @property - def defaults(self) -> Optional[List[StringFormatterValidator]]: - """ - Returns - ------- - List of string format validators to use for the defaults - """ - return self._defaults.list if self.has_defaults else None - - @property - def capture_group_names(self) -> List[str]: - """ - Returns - ------- - List of new capture group names - """ - return [validator.value for validator in self._capture_group_names.list] + return self._capture_group_defaults is not None class FromSourceVariablesRegex(StrictDictValidator): @@ -115,16 +114,19 @@ class RegexOptions(PluginOptions): skip_if_match_fails: True from: + # For each entry's `title` value... title: - # Match with capture groups act as a filter. + # Perform this regex match on it to act as a filter. # This will only download videos with "Official Video" in it. - match: "\[Official Video\]" + match: '\[Official Video\]' + # For each entry's `description` value... description: # Match with capture groups and defaults. # This tries to scrape a date from the description and produce new source variables match: "([0-9]{4})-([0-9]{2})-([0-9]{2})" - # Each capture group creates these new source variables, respectively + # Each capture group creates these new source variables, respectively, as well + # a sanitized version, i.e. `sanitized_captured_upload_year` capture_group_names: - "captured_upload_year" - "captured_upload_month" @@ -144,10 +146,18 @@ class RegexOptions(PluginOptions): def __init__(self, name, value): super().__init__(name, value) self._from = self._validate_key(key="from", validator=FromSourceVariablesRegex) - self.skip_if_match_fails: bool = self._validate_key_if_present( - key="skip_if_match_fails", validator=BoolValidator, default=False + self._skip_if_match_fails: bool = self._validate_key_if_present( + key="skip_if_match_fails", validator=BoolValidator, default=True ).value + @property + def skip_if_match_fails(self) -> Optional[bool]: + """ + Defaults to True. If True, when any match fails and has no defaults, the entry will be + skipped. If False, ytdl-sub will error and all downloads will not proceed. + """ + return self._skip_if_match_fails + def validate_with_variables( self, source_variables: List[str], override_variables: List[str] ) -> None: @@ -226,9 +236,7 @@ class RegexPlugin(Plugin[RegexOptions]): # Iterate each source var to capture and add to the entry for source_var, regex_options in self.plugin_options.source_variable_capture_dict.items(): - maybe_capture = regex_options.capture_list.match_any( - input_str=entry_variable_dict[source_var] - ) + maybe_capture = regex_options.match.match_any(input_str=entry_variable_dict[source_var]) # If no capture if maybe_capture is None: @@ -257,7 +265,7 @@ class RegexPlugin(Plugin[RegexOptions]): regex_options.capture_group_names[i]: default.apply_formatter( variable_dict=source_variables_and_overrides_dict ) - for i, default in enumerate(regex_options.defaults) + for i, default in enumerate(regex_options.capture_group_defaults) }, ) # There is a capture, add the source variables to the entry as diff --git a/tests/e2e/plugins/test_regex.py b/tests/e2e/plugins/test_regex.py index d66d0c11..404ced41 100644 --- a/tests/e2e/plugins/test_regex.py +++ b/tests/e2e/plugins/test_regex.py @@ -22,7 +22,7 @@ def regex_subscription_dict(output_directory): "format": "best[height<=480]", }, "regex": { - "skip_if_match_fails": True, + # tests that skip_if_match_fails defaults to True "from": { "title": { "match": [ @@ -41,7 +41,7 @@ def regex_subscription_dict(output_directory): "upload_captured_year", "upload_captured_month", ], - "defaults": [ + "capture_group_defaults": [ "First", "Second containing {in_regex_default}", ], @@ -49,7 +49,7 @@ def regex_subscription_dict(output_directory): "artist": { "match": ["Never (.*) capture"], "capture_group_names": ["always_default"], - "defaults": ["Always default"], + "capture_group_defaults": ["Always default"], }, }, }, @@ -72,8 +72,7 @@ def regex_subscription_dict(output_directory): @pytest.fixture def regex_subscription_dict_no_match_fails(regex_subscription_dict): - # tests that skip_if_match_fails defaults to False - del regex_subscription_dict["regex"]["skip_if_match_fails"] + regex_subscription_dict["regex"]["skip_if_match_fails"] = False return regex_subscription_dict @@ -177,7 +176,7 @@ class TestRegex: ) def test_regex_fails_unequal_defaults(self, regex_subscription_dict, music_video_config): - regex_subscription_dict["regex"]["from"]["title"]["defaults"] = ["1 != 2"] + regex_subscription_dict["regex"]["from"]["title"]["capture_group_defaults"] = ["1 != 2"] with pytest.raises( ValidationException, match=re.escape("number of defaults must match number of capture groups, 1 != 2"), diff --git a/tests/e2e/youtube/test_playlist_as_kodi_music_videos.py b/tests/e2e/youtube/test_playlist_as_kodi_music_videos.py index 5308544f..a8e0e797 100644 --- a/tests/e2e/youtube/test_playlist_as_kodi_music_videos.py +++ b/tests/e2e/youtube/test_playlist_as_kodi_music_videos.py @@ -52,7 +52,7 @@ def expected_playlist_download(): return ExpectedDownloads( expected_downloads=[ # Download mapping - ExpectedDownloadFile(path=Path(".ytdl-sub-jmc-download-archive.json"), md5="9f785c29194a6ecfba6a6b4018763ddc"), + ExpectedDownloadFile(path=Path(".ytdl-sub-music_video_playlist_test-download-archive.json"), md5="9f785c29194a6ecfba6a6b4018763ddc"), # Entry files ExpectedDownloadFile(path=Path("JMC - Jesse's Minecraft Server [Trailer - Feb.1]-thumb.jpg"), md5="b232d253df621aa770b780c1301d364d"), @@ -87,16 +87,16 @@ def single_video_subscription_dict(subscription_dict): @pytest.fixture -def single_video_subscription(config, single_video_subscription_dict): +def single_video_subscription(music_video_config, single_video_subscription_dict): single_video_preset = Preset.from_dict( - config=config, + config=music_video_config, preset_name="music_video_single_video_test", preset_dict=single_video_subscription_dict, ) return Subscription.from_preset( preset=single_video_preset, - config=config, + config=music_video_config, )