[FEATURE] exclude field in regex plugin
This commit is contained in:
parent
2f1756dea7
commit
5b0750be4f
1 changed files with 57 additions and 25 deletions
|
|
@ -24,12 +24,12 @@ logger = Logger.get(name="regex")
|
||||||
|
|
||||||
class SourceVariableRegex(StrictDictValidator):
|
class SourceVariableRegex(StrictDictValidator):
|
||||||
|
|
||||||
_required_keys = {"match"}
|
_optional_keys = {"match", "exclude", "capture_group_defaults", "capture_group_names"}
|
||||||
_optional_keys = {"capture_group_defaults", "capture_group_names"}
|
|
||||||
|
|
||||||
def __init__(self, name, value):
|
def __init__(self, name, value):
|
||||||
super().__init__(name, value)
|
super().__init__(name, value)
|
||||||
self._match = self._validate_key(key="match", validator=RegexListValidator)
|
self._match = self._validate_key_if_present(key="match", validator=RegexListValidator)
|
||||||
|
self._exclude = self._validate_key_if_present(key="exclude", validator=RegexListValidator)
|
||||||
self._capture_group_defaults = self._validate_key_if_present(
|
self._capture_group_defaults = self._validate_key_if_present(
|
||||||
key="capture_group_defaults", validator=ListFormatterValidator
|
key="capture_group_defaults", validator=ListFormatterValidator
|
||||||
)
|
)
|
||||||
|
|
@ -37,6 +37,14 @@ class SourceVariableRegex(StrictDictValidator):
|
||||||
key="capture_group_names", validator=SourceVariableNameListValidator, default=[]
|
key="capture_group_names", validator=SourceVariableNameListValidator, default=[]
|
||||||
)
|
)
|
||||||
|
|
||||||
|
if self._match is None and self._exclude is None:
|
||||||
|
raise self._validation_exception("must specify either `match` or `exclude`")
|
||||||
|
|
||||||
|
if self._match is None and (self._capture_group_defaults or self._capture_group_names):
|
||||||
|
raise self._validation_exception(
|
||||||
|
"capture group parameters requires at least one `match` to be specified"
|
||||||
|
)
|
||||||
|
|
||||||
# If defaults are to be used, ensure there are the same number of defaults as there are
|
# If defaults are to be used, ensure there are the same number of defaults as there are
|
||||||
# capture groups
|
# capture groups
|
||||||
if self._capture_group_defaults is not None and self._match.num_capture_groups != len(
|
if self._capture_group_defaults is not None and self._match.num_capture_groups != len(
|
||||||
|
|
@ -55,13 +63,21 @@ class SourceVariableRegex(StrictDictValidator):
|
||||||
)
|
)
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def match(self) -> RegexListValidator:
|
def match(self) -> Optional[RegexListValidator]:
|
||||||
"""
|
"""
|
||||||
Required. List of regex strings to try to match against a source variable. Each regex
|
List of regex strings to try to match against a source variable. Each regex
|
||||||
string must have the same number of capture groups.
|
string must have the same number of capture groups. Can not be used with ``match``
|
||||||
"""
|
"""
|
||||||
return self._match
|
return self._match
|
||||||
|
|
||||||
|
@property
|
||||||
|
def exclude(self) -> Optional[RegexListValidator]:
|
||||||
|
"""
|
||||||
|
List of regex strings to try to match against a source variable. If one of the regex strings
|
||||||
|
match, then the entry will be skipped.
|
||||||
|
"""
|
||||||
|
return self._exclude
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def capture_group_names(self) -> Optional[List[str]]:
|
def capture_group_names(self) -> Optional[List[str]]:
|
||||||
"""
|
"""
|
||||||
|
|
@ -138,6 +154,9 @@ class RegexOptions(PluginOptions):
|
||||||
# source variables
|
# source variables
|
||||||
match:
|
match:
|
||||||
- "([0-9]{4})-([0-9]{2})-([0-9]{2})"
|
- "([0-9]{4})-([0-9]{2})-([0-9]{2})"
|
||||||
|
# Exclude any entry where the description contains #short
|
||||||
|
exclude:
|
||||||
|
- "#short"
|
||||||
|
|
||||||
# Each capture group creates these new source variables, respectively,
|
# Each capture group creates these new source variables, respectively,
|
||||||
# as well a sanitized version, i.e. `captured_upload_year_sanitized`
|
# as well a sanitized version, i.e. `captured_upload_year_sanitized`
|
||||||
|
|
@ -257,6 +276,19 @@ class RegexPlugin(Plugin[RegexOptions]):
|
||||||
def _contains_processed_regex_source_var(cls, entry: Entry, source_var: str) -> bool:
|
def _contains_processed_regex_source_var(cls, entry: Entry, source_var: str) -> bool:
|
||||||
return source_var in entry.kwargs_get(YTDL_SUB_REGEX_SOURCE_VARS, [])
|
return source_var in entry.kwargs_get(YTDL_SUB_REGEX_SOURCE_VARS, [])
|
||||||
|
|
||||||
|
def _try_skip_entry(self, entry: Entry, source_var: str) -> None:
|
||||||
|
# Skip the entry if toggled
|
||||||
|
if self.plugin_options.skip_if_match_fails:
|
||||||
|
logger.info(
|
||||||
|
"Regex failed to match '%s' from '%s', skipping.",
|
||||||
|
source_var,
|
||||||
|
entry.title,
|
||||||
|
)
|
||||||
|
return None
|
||||||
|
|
||||||
|
# Otherwise, error
|
||||||
|
raise RegexNoMatchException(f"Regex failed to match '{source_var}' from '{entry.title}'")
|
||||||
|
|
||||||
def _modify_entry_metadata(self, entry: Entry, is_metadata_stage: bool) -> Optional[Entry]:
|
def _modify_entry_metadata(self, entry: Entry, is_metadata_stage: bool) -> Optional[Entry]:
|
||||||
"""
|
"""
|
||||||
Parameters
|
Parameters
|
||||||
|
|
@ -292,25 +324,25 @@ class RegexPlugin(Plugin[RegexOptions]):
|
||||||
continue
|
continue
|
||||||
|
|
||||||
self._add_processed_regex_source_var(entry, source_var)
|
self._add_processed_regex_source_var(entry, source_var)
|
||||||
maybe_capture = regex_options.match.match_any(input_str=entry_variable_dict[source_var])
|
|
||||||
|
if (
|
||||||
|
regex_options.exclude is not None
|
||||||
|
and regex_options.exclude.match_any(input_str=entry_variable_dict[source_var])
|
||||||
|
is not None
|
||||||
|
):
|
||||||
|
return self._try_skip_entry(entry=entry, source_var=source_var)
|
||||||
|
|
||||||
|
maybe_capture: Optional[List[str]] = None
|
||||||
|
if regex_options.match is not None:
|
||||||
|
maybe_capture = regex_options.match.match_any(
|
||||||
|
input_str=entry_variable_dict[source_var]
|
||||||
|
)
|
||||||
|
|
||||||
# If no capture
|
# If no capture
|
||||||
if maybe_capture is None:
|
if maybe_capture is None:
|
||||||
# and no defaults
|
# and no defaults
|
||||||
if not regex_options.has_defaults:
|
if not regex_options.has_defaults:
|
||||||
# Skip the entry if toggled
|
return self._try_skip_entry(entry=entry, source_var=source_var)
|
||||||
if self.plugin_options.skip_if_match_fails:
|
|
||||||
logger.info(
|
|
||||||
"Regex failed to match '%s' from '%s', skipping.",
|
|
||||||
source_var,
|
|
||||||
entry.title,
|
|
||||||
)
|
|
||||||
return None
|
|
||||||
|
|
||||||
# Otherwise, error
|
|
||||||
raise RegexNoMatchException(
|
|
||||||
f"Regex failed to match '{source_var}' from '{entry.title}'"
|
|
||||||
)
|
|
||||||
|
|
||||||
# otherwise, use defaults (apply them using the original entry source dict)
|
# otherwise, use defaults (apply them using the original entry source dict)
|
||||||
source_variables_and_overrides_dict = dict(
|
source_variables_and_overrides_dict = dict(
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue