[FEATURE] exclude field in regex plugin

This commit is contained in:
Jesse Bannon 2023-05-21 22:00:03 -07:00
parent 2f1756dea7
commit 5b0750be4f

View file

@ -24,12 +24,12 @@ logger = Logger.get(name="regex")
class SourceVariableRegex(StrictDictValidator): class SourceVariableRegex(StrictDictValidator):
_required_keys = {"match"} _optional_keys = {"match", "exclude", "capture_group_defaults", "capture_group_names"}
_optional_keys = {"capture_group_defaults", "capture_group_names"}
def __init__(self, name, value): def __init__(self, name, value):
super().__init__(name, value) super().__init__(name, value)
self._match = self._validate_key(key="match", validator=RegexListValidator) self._match = self._validate_key_if_present(key="match", validator=RegexListValidator)
self._exclude = self._validate_key_if_present(key="exclude", validator=RegexListValidator)
self._capture_group_defaults = self._validate_key_if_present( self._capture_group_defaults = self._validate_key_if_present(
key="capture_group_defaults", validator=ListFormatterValidator key="capture_group_defaults", validator=ListFormatterValidator
) )
@ -37,6 +37,14 @@ class SourceVariableRegex(StrictDictValidator):
key="capture_group_names", validator=SourceVariableNameListValidator, default=[] key="capture_group_names", validator=SourceVariableNameListValidator, default=[]
) )
if self._match is None and self._exclude is None:
raise self._validation_exception("must specify either `match` or `exclude`")
if self._match is None and (self._capture_group_defaults or self._capture_group_names):
raise self._validation_exception(
"capture group parameters requires at least one `match` to be specified"
)
# If defaults are to be used, ensure there are the same number of defaults as there are # If defaults are to be used, ensure there are the same number of defaults as there are
# capture groups # capture groups
if self._capture_group_defaults is not None and self._match.num_capture_groups != len( if self._capture_group_defaults is not None and self._match.num_capture_groups != len(
@ -55,13 +63,21 @@ class SourceVariableRegex(StrictDictValidator):
) )
@property @property
def match(self) -> RegexListValidator: def match(self) -> Optional[RegexListValidator]:
""" """
Required. List of regex strings to try to match against a source variable. Each regex List of regex strings to try to match against a source variable. Each regex
string must have the same number of capture groups. string must have the same number of capture groups. Can not be used with ``match``
""" """
return self._match return self._match
@property
def exclude(self) -> Optional[RegexListValidator]:
"""
List of regex strings to try to match against a source variable. If one of the regex strings
match, then the entry will be skipped.
"""
return self._exclude
@property @property
def capture_group_names(self) -> Optional[List[str]]: def capture_group_names(self) -> Optional[List[str]]:
""" """
@ -129,7 +145,7 @@ class RegexOptions(PluginOptions):
# This will only download videos with "[Official Video]" in it. Note that we # This will only download videos with "[Official Video]" in it. Note that we
# double backslash to make YAML happy # double backslash to make YAML happy
match: match:
- '\\[Official Video\\]' - '\\[Official Video\\]'
# For each entry's `description` value... # For each entry's `description` value...
description: description:
@ -137,14 +153,17 @@ class RegexOptions(PluginOptions):
# This tries to scrape a date from the description and produce new # This tries to scrape a date from the description and produce new
# source variables # source variables
match: match:
- "([0-9]{4})-([0-9]{2})-([0-9]{2})" - "([0-9]{4})-([0-9]{2})-([0-9]{2})"
# Exclude any entry where the description contains #short
exclude:
- "#short"
# Each capture group creates these new source variables, respectively, # Each capture group creates these new source variables, respectively,
# as well a sanitized version, i.e. `captured_upload_year_sanitized` # as well a sanitized version, i.e. `captured_upload_year_sanitized`
capture_group_names: capture_group_names:
- "captured_upload_year" - "captured_upload_year"
- "captured_upload_month" - "captured_upload_month"
- "captured_upload_day" - "captured_upload_day"
# And if the string does not match, use these as respective default # And if the string does not match, use these as respective default
# values for the new source variables. # values for the new source variables.
@ -257,6 +276,19 @@ class RegexPlugin(Plugin[RegexOptions]):
def _contains_processed_regex_source_var(cls, entry: Entry, source_var: str) -> bool: def _contains_processed_regex_source_var(cls, entry: Entry, source_var: str) -> bool:
return source_var in entry.kwargs_get(YTDL_SUB_REGEX_SOURCE_VARS, []) return source_var in entry.kwargs_get(YTDL_SUB_REGEX_SOURCE_VARS, [])
def _try_skip_entry(self, entry: Entry, source_var: str) -> None:
# Skip the entry if toggled
if self.plugin_options.skip_if_match_fails:
logger.info(
"Regex failed to match '%s' from '%s', skipping.",
source_var,
entry.title,
)
return None
# Otherwise, error
raise RegexNoMatchException(f"Regex failed to match '{source_var}' from '{entry.title}'")
def _modify_entry_metadata(self, entry: Entry, is_metadata_stage: bool) -> Optional[Entry]: def _modify_entry_metadata(self, entry: Entry, is_metadata_stage: bool) -> Optional[Entry]:
""" """
Parameters Parameters
@ -292,25 +324,25 @@ class RegexPlugin(Plugin[RegexOptions]):
continue continue
self._add_processed_regex_source_var(entry, source_var) self._add_processed_regex_source_var(entry, source_var)
maybe_capture = regex_options.match.match_any(input_str=entry_variable_dict[source_var])
if (
regex_options.exclude is not None
and regex_options.exclude.match_any(input_str=entry_variable_dict[source_var])
is not None
):
return self._try_skip_entry(entry=entry, source_var=source_var)
maybe_capture: Optional[List[str]] = None
if regex_options.match is not None:
maybe_capture = regex_options.match.match_any(
input_str=entry_variable_dict[source_var]
)
# If no capture # If no capture
if maybe_capture is None: if maybe_capture is None:
# and no defaults # and no defaults
if not regex_options.has_defaults: if not regex_options.has_defaults:
# Skip the entry if toggled return self._try_skip_entry(entry=entry, source_var=source_var)
if self.plugin_options.skip_if_match_fails:
logger.info(
"Regex failed to match '%s' from '%s', skipping.",
source_var,
entry.title,
)
return None
# Otherwise, error
raise RegexNoMatchException(
f"Regex failed to match '{source_var}' from '{entry.title}'"
)
# otherwise, use defaults (apply them using the original entry source dict) # otherwise, use defaults (apply them using the original entry source dict)
source_variables_and_overrides_dict = dict( source_variables_and_overrides_dict = dict(