diff --git a/docs/config.rst b/docs/config.rst index 56438f4d..225034a3 100644 --- a/docs/config.rst +++ b/docs/config.rst @@ -213,7 +213,7 @@ regex .. autoclass:: ytdl_sub.plugins.regex.RegexOptions() :members: skip_if_match_fails -.. autoclass:: ytdl_sub.plugins.regex.SourceVariableRegex() +.. autoclass:: ytdl_sub.plugins.regex.VariableRegex() :members: match, capture_group_names, capture_group_defaults, exclude :member-order: bysource :exclude-members: partial_validate diff --git a/src/ytdl_sub/plugins/regex.py b/src/ytdl_sub/plugins/regex.py index 1e7a5b03..39485c10 100644 --- a/src/ytdl_sub/plugins/regex.py +++ b/src/ytdl_sub/plugins/regex.py @@ -11,6 +11,7 @@ from ytdl_sub.plugins.plugin import Plugin from ytdl_sub.plugins.plugin import PluginOptions from ytdl_sub.plugins.plugin import PluginPriority from ytdl_sub.utils.exceptions import RegexNoMatchException +from ytdl_sub.utils.exceptions import StringFormattingVariableNotFoundException from ytdl_sub.utils.logger import Logger from ytdl_sub.validators.regex_validator import RegexListValidator from ytdl_sub.validators.source_variable_validator import SourceVariableNameListValidator @@ -22,7 +23,7 @@ from ytdl_sub.validators.validators import BoolValidator logger = Logger.get(name="regex") -class SourceVariableRegex(StrictDictValidator): +class VariableRegex(StrictDictValidator): _optional_keys = {"match", "exclude", "capture_group_defaults", "capture_group_names"} @@ -65,7 +66,7 @@ class SourceVariableRegex(StrictDictValidator): @property def match(self) -> Optional[RegexListValidator]: """ - List of regex strings to try to match against a source variable. Each regex + List of regex strings to try to match against a source or override variable. Each regex string must have the same number of capture groups. """ return self._match @@ -73,9 +74,10 @@ class SourceVariableRegex(StrictDictValidator): @property def exclude(self) -> Optional[RegexListValidator]: """ - List of regex strings to try to match against a source variable. If one of the regex strings - match, then the entry will be skipped. If both ``exclude`` and ``match`` are specified, - entries will get skipped if the regex matches against both ``exclude`` and ``match``. + List of regex strings to try to match against a source or override variable. If one of the + regex strings match, then the entry will be skipped. If both ``exclude`` and ``match`` are + specified, entries will get skipped if the regex matches against both ``exclude`` and + ``match``. """ return self._exclude @@ -114,19 +116,25 @@ class FromSourceVariablesRegex(StrictDictValidator): def __init__(self, name, value): super().__init__(name, value) - self.source_variable_capture_dict: Dict[str, SourceVariableRegex] = { - key: self._validate_key(key=key, validator=SourceVariableRegex) for key in self._keys + self.variable_capture_dict: Dict[str, VariableRegex] = { + key: self._validate_key(key=key, validator=VariableRegex) for key in self._keys } class RegexOptions(PluginOptions): r""" - Performs regex matching on an entry's source variables. Regex can be used to filter entries - from proceeding with download or capture groups to create new source variables. NOTE to + Performs regex matching on an entry's source or override variables. Regex can be used to filter + entries from proceeding with download or capture groups to create new source variables. NOTE to use backslashes anywhere in your regex, i.e. ``\d``, you must add another backslash escape. This means ``\d`` should be written as ``\\d``. This is because YAML requires an escape for any backslash usage. + If you want to regex-search multiple source variables to create a logical OR effect, you can + create an override variable that contains the concatenation of them, and search that with regex. + For example, creating the override variable ``"title_and_description": "{title} {description}"`` + and using ``title_and_description`` can regex match/exclude from either ``title`` or + ``description``. + Usage: .. code-block:: yaml @@ -216,9 +224,9 @@ class RegexOptions(PluginOptions): """ for key, regex_options in self.source_variable_capture_dict.items(): # Ensure each variable getting captured is a source variable - if key not in source_variables: + if key not in source_variables and key not in override_variables: raise self._validation_exception( - f"cannot regex capture '{key}' because it is not a source variable" + f"cannot regex capture '{key}' because it is not a source or override variable" ) # Ensure the capture group names are not existing source/override variables @@ -235,13 +243,13 @@ class RegexOptions(PluginOptions): ) @property - def source_variable_capture_dict(self) -> Dict[str, SourceVariableRegex]: + def source_variable_capture_dict(self) -> Dict[str, VariableRegex]: """ Returns ------- Dict of { source variable: capture options } """ - return self._from.source_variable_capture_dict + return self._from.variable_capture_dict def added_source_variables(self) -> List[str]: """ @@ -267,28 +275,56 @@ class RegexPlugin(Plugin[RegexOptions]): ) @classmethod - def _add_processed_regex_source_var(cls, entry: Entry, source_var: str) -> None: + def _add_processed_regex_variable_name(cls, entry: Entry, source_var: str) -> None: if not entry.kwargs_contains(YTDL_SUB_REGEX_SOURCE_VARS): entry.add_kwargs({YTDL_SUB_REGEX_SOURCE_VARS: []}) entry.kwargs(YTDL_SUB_REGEX_SOURCE_VARS).append(source_var) @classmethod - def _contains_processed_regex_source_var(cls, entry: Entry, source_var: str) -> bool: - return source_var in entry.kwargs_get(YTDL_SUB_REGEX_SOURCE_VARS, []) + def _contains_processed_regex_variable(cls, entry: Entry, variable_name: str) -> bool: + return variable_name in entry.kwargs_get(YTDL_SUB_REGEX_SOURCE_VARS, []) - def _try_skip_entry(self, entry: Entry, source_var: str) -> None: + def _try_skip_entry(self, entry: Entry, variable_name: str) -> None: # Skip the entry if toggled if self.plugin_options.skip_if_match_fails: logger.info( "Regex failed to match '%s' from '%s', skipping.", - source_var, + variable_name, entry.title, ) return None # Otherwise, error - raise RegexNoMatchException(f"Regex failed to match '{source_var}' from '{entry.title}'") + raise RegexNoMatchException(f"Regex failed to match '{variable_name}' from '{entry.title}'") + + def _can_process_at_metadata_stage(self, entry: Entry, variable_name: str) -> bool: + # If the variable is an override... + if variable_name in self.overrides.dict: + # Try to see if it can resolve + try: + self.overrides.apply_formatter( + formatter=self.overrides.dict[variable_name], + entry=entry, + ) + # If it can not from missing variables (from post-metadata stage), return False + except StringFormattingVariableNotFoundException: + return False + # If it is a source variable and not present, return false + elif variable_name not in entry.to_dict(): + return False + + return True + + def _get_regex_input_string(self, entry: Entry, variable_name: str) -> str: + # Apply override formatter if it's an override + if variable_name in self.overrides.dict: + return self.overrides.apply_formatter( + formatter=self.overrides.dict[variable_name], + entry=entry, + ) + # Otherwise pluck from the entry's source variable + return entry.to_dict()[variable_name] def _modify_entry_metadata(self, entry: Entry, is_metadata_stage: bool) -> Optional[Entry]: """ @@ -308,46 +344,50 @@ class RegexPlugin(Plugin[RegexOptions]): ValidationException If no capture and no defaults """ - entry_variable_dict = entry.to_dict() - # Iterate each source var to capture and add to the entry - for source_var, regex_options in self.plugin_options.source_variable_capture_dict.items(): + for ( + variable_name, + regex_options, + ) in self.plugin_options.source_variable_capture_dict.items(): # Record which regex source variables are processed, to # process as many variables as possible in the metadata stage, then the rest # after the media file has been downloaded. - if self._contains_processed_regex_source_var(entry, source_var): + if self._contains_processed_regex_variable(entry, variable_name): continue - # Continue if the source var isn't in the dict since entry variables could be added - # after the metadata stage - if is_metadata_stage and source_var not in entry_variable_dict: + # If it's the metadata stage, and it can't be processed, skip until post-metadata + if is_metadata_stage and not self._can_process_at_metadata_stage( + entry=entry, variable_name=variable_name + ): continue - self._add_processed_regex_source_var(entry, source_var) + self._add_processed_regex_variable_name(entry, variable_name) + + regex_input_str = self._get_regex_input_string( + entry=entry, + variable_name=variable_name, + ) if ( regex_options.exclude is not None - and regex_options.exclude.match_any(input_str=entry_variable_dict[source_var]) - is not None + and regex_options.exclude.match_any(input_str=regex_input_str) is not None ): - return self._try_skip_entry(entry=entry, source_var=source_var) + return self._try_skip_entry(entry=entry, variable_name=variable_name) # If match is present if regex_options.match is not None: - maybe_capture = regex_options.match.match_any( - input_str=entry_variable_dict[source_var] - ) + maybe_capture = regex_options.match.match_any(input_str=regex_input_str) # And nothing matched if maybe_capture is None: # and no defaults if not regex_options.has_defaults: - return self._try_skip_entry(entry=entry, source_var=source_var) + return self._try_skip_entry(entry=entry, variable_name=variable_name) # otherwise, use defaults (apply them using the original entry source dict) source_variables_and_overrides_dict = dict( - entry_variable_dict, **self.overrides.dict_with_format_strings + entry.to_dict(), **self.overrides.dict_with_format_strings ) # add both the default... diff --git a/tests/e2e/plugins/test_music_tags.py b/tests/e2e/plugins/test_music_tags.py index c6cecf21..5538357b 100644 --- a/tests/e2e/plugins/test_music_tags.py +++ b/tests/e2e/plugins/test_music_tags.py @@ -1,8 +1,6 @@ import re import pytest -from expected_download import assert_expected_downloads -from expected_transaction_log import assert_transaction_log_matches from ytdl_sub.subscriptions.subscription import Subscription from ytdl_sub.utils.exceptions import ValidationException diff --git a/tests/e2e/plugins/test_regex.py b/tests/e2e/plugins/test_regex.py index fc4bd6b4..32e899eb 100644 --- a/tests/e2e/plugins/test_regex.py +++ b/tests/e2e/plugins/test_regex.py @@ -146,6 +146,33 @@ def regex_subscription_dict_match_and_exclude(regex_subscription_dict_base, outp ) +@pytest.fixture +def regex_subscription_dict_match_and_exclude_override_variable( + regex_subscription_dict_base, output_directory +): + return mergedeep.merge( + regex_subscription_dict_base, + { + "regex": { + # tests that skip_if_match_fails defaults to True + "from": { + "override_title": { + "exclude": [ + "should not cap", + ".*Feb.*", # should filter out march video + ], + }, + "override_description": { + "match": [".*http:\\/\\/(.+).com.*"], + "capture_group_names": ["override_description_website"], + }, + }, + }, + "overrides": {"override_title": "{title}", "override_description": "{description}"}, + }, + ) + + @pytest.fixture def playlist_subscription(music_video_config, regex_subscription_dict): return Subscription.from_dict( @@ -179,6 +206,18 @@ def playlist_subscription_exclude( ) +@pytest.fixture +def playlist_subscription_overrides( + music_video_config: ConfigFile, + regex_subscription_dict_match_and_exclude_override_variable: Dict[str, Any], +) -> Subscription: + return Subscription.from_dict( + config=music_video_config, + preset_name="regex_using_overrides_test", + preset_dict=regex_subscription_dict_match_and_exclude_override_variable, + ) + + @pytest.fixture def playlist_subscription_match_and_exclude( music_video_config: ConfigFile, regex_subscription_dict_match_and_exclude: Dict[str, Any] @@ -221,6 +260,16 @@ class TestRegex: regenerate_transaction_log=True, ) + def test_regex_using_overrides_success(self, playlist_subscription_overrides, output_directory): + # Should only contain the march video + transaction_log = playlist_subscription_overrides.download(dry_run=True) + assert_transaction_log_matches( + output_directory=output_directory, + transaction_log=transaction_log, + transaction_log_summary_file_name="plugins/test_regex_overrides.txt", + regenerate_transaction_log=True, + ) + def test_regex_fails_no_match(self, playlist_subscription_no_match_fails, output_directory): with pytest.raises( RegexNoMatchException, @@ -300,7 +349,9 @@ class TestRegex: ) with pytest.raises( ValidationException, - match=re.escape("cannot regex capture 'dne' because it is not a source variable"), + match=re.escape( + "cannot regex capture 'dne' because it is not a source or override variable" + ), ): _ = Subscription.from_dict( config=music_video_config, diff --git a/tests/e2e/plugins/test_split_by_chapters.py b/tests/e2e/plugins/test_split_by_chapters.py index 4151ab71..f3e02e79 100644 --- a/tests/e2e/plugins/test_split_by_chapters.py +++ b/tests/e2e/plugins/test_split_by_chapters.py @@ -32,7 +32,9 @@ def yt_album_as_chapters_with_regex_preset_dict(yt_album_as_chapters_preset_dict { "regex": { "from": { - "chapter_title": { + # Ensure regex can handle override variables that come from the + # post-metadata stage + "override_chapter_title": { "match": r"\d+\. (.+)", "capture_group_names": "captured_track_title", "capture_group_defaults": "{chapter_title}", @@ -54,6 +56,7 @@ def yt_album_as_chapters_with_regex_preset_dict(yt_album_as_chapters_preset_dict } }, "overrides": { + "override_chapter_title": "{chapter_title}", "track_title": "{captured_track_title}", "track_album": "{captured_track_album}", "track_artist": "{captured_track_artist}", diff --git a/tests/resources/transaction_log_summaries/plugins/test_regex_overrides.txt b/tests/resources/transaction_log_summaries/plugins/test_regex_overrides.txt new file mode 100644 index 00000000..663087d2 --- /dev/null +++ b/tests/resources/transaction_log_summaries/plugins/test_regex_overrides.txt @@ -0,0 +1,15 @@ +Files created: +---------------------------------------- +{output_directory} + .ytdl-sub-regex_using_overrides_test-download-archive.json +{output_directory}/Project Zombie + Project Zombie - Jesse's Minecraft Server [Trailer - Mar.21]-thumb.jpg + Project Zombie - Jesse's Minecraft Server [Trailer - Mar.21].info.json + Project Zombie - Jesse's Minecraft Server [Trailer - Mar.21].mp4 + Project Zombie - Jesse's Minecraft Server [Trailer - Mar.21].nfo + NFO tags: + musicvideo: + album: Music Videos + artist: Project Zombie + title: Jesse's Minecraft Server [Trailer - Mar.21] + year: 2011 \ No newline at end of file