[FEATURE] exclude field in regex plugin (#620)

* [FEATURE] `exclude` field in regex plugin

* tests

* excludes doc
This commit is contained in:
Jesse Bannon 2023-05-22 10:23:00 -07:00 committed by GitHub
parent 2f1756dea7
commit ec447cfbd7
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
5 changed files with 312 additions and 106 deletions

View file

@ -214,7 +214,7 @@ regex
:members: skip_if_match_fails
.. autoclass:: ytdl_sub.plugins.regex.SourceVariableRegex()
:members: match, capture_group_names, capture_group_defaults
:members: match, capture_group_names, capture_group_defaults, exclude
:member-order: bysource
:exclude-members: partial_validate

View file

@ -24,12 +24,12 @@ logger = Logger.get(name="regex")
class SourceVariableRegex(StrictDictValidator):
_required_keys = {"match"}
_optional_keys = {"capture_group_defaults", "capture_group_names"}
_optional_keys = {"match", "exclude", "capture_group_defaults", "capture_group_names"}
def __init__(self, name, value):
super().__init__(name, value)
self._match = self._validate_key(key="match", validator=RegexListValidator)
self._match = self._validate_key_if_present(key="match", validator=RegexListValidator)
self._exclude = self._validate_key_if_present(key="exclude", validator=RegexListValidator)
self._capture_group_defaults = self._validate_key_if_present(
key="capture_group_defaults", validator=ListFormatterValidator
)
@ -37,6 +37,14 @@ class SourceVariableRegex(StrictDictValidator):
key="capture_group_names", validator=SourceVariableNameListValidator, default=[]
)
if self._match is None and self._exclude is None:
raise self._validation_exception("must specify either `match` or `exclude`")
if self._match is None and (self._capture_group_defaults or self._capture_group_names.list):
raise self._validation_exception(
"capture group parameters requires at least one `match` to be specified"
)
# If defaults are to be used, ensure there are the same number of defaults as there are
# capture groups
if self._capture_group_defaults is not None and self._match.num_capture_groups != len(
@ -48,20 +56,29 @@ class SourceVariableRegex(StrictDictValidator):
)
# If there are capture groups, ensure there are capture group names
if len(self._capture_group_names.list) != self._match.num_capture_groups:
if self._match and (len(self._capture_group_names.list) != self._match.num_capture_groups):
raise self._validation_exception(
f"number of capture group names must match number of capture groups, "
f"{len(self._capture_group_names.list)} != {self._match.num_capture_groups}"
)
@property
def match(self) -> RegexListValidator:
def match(self) -> Optional[RegexListValidator]:
"""
Required. List of regex strings to try to match against a source variable. Each regex
List of regex strings to try to match against a source variable. Each regex
string must have the same number of capture groups.
"""
return self._match
@property
def exclude(self) -> Optional[RegexListValidator]:
"""
List of regex strings to try to match against a source variable. If one of the regex strings
match, then the entry will be skipped. If both ``exclude`` and ``match`` are specified,
entries will get skipped if the regex matches against both ``exclude`` and ``match``.
"""
return self._exclude
@property
def capture_group_names(self) -> Optional[List[str]]:
"""
@ -129,7 +146,7 @@ class RegexOptions(PluginOptions):
# This will only download videos with "[Official Video]" in it. Note that we
# double backslash to make YAML happy
match:
- '\\[Official Video\\]'
- '\\[Official Video\\]'
# For each entry's `description` value...
description:
@ -137,14 +154,17 @@ class RegexOptions(PluginOptions):
# This tries to scrape a date from the description and produce new
# source variables
match:
- "([0-9]{4})-([0-9]{2})-([0-9]{2})"
- "([0-9]{4})-([0-9]{2})-([0-9]{2})"
# Exclude any entry where the description contains #short
exclude:
- "#short"
# Each capture group creates these new source variables, respectively,
# as well a sanitized version, i.e. `captured_upload_year_sanitized`
capture_group_names:
- "captured_upload_year"
- "captured_upload_month"
- "captured_upload_day"
- "captured_upload_year"
- "captured_upload_month"
- "captured_upload_day"
# And if the string does not match, use these as respective default
# values for the new source variables.
@ -257,6 +277,19 @@ class RegexPlugin(Plugin[RegexOptions]):
def _contains_processed_regex_source_var(cls, entry: Entry, source_var: str) -> bool:
return source_var in entry.kwargs_get(YTDL_SUB_REGEX_SOURCE_VARS, [])
def _try_skip_entry(self, entry: Entry, source_var: str) -> None:
# Skip the entry if toggled
if self.plugin_options.skip_if_match_fails:
logger.info(
"Regex failed to match '%s' from '%s', skipping.",
source_var,
entry.title,
)
return None
# Otherwise, error
raise RegexNoMatchException(f"Regex failed to match '{source_var}' from '{entry.title}'")
def _modify_entry_metadata(self, entry: Entry, is_metadata_stage: bool) -> Optional[Entry]:
"""
Parameters
@ -292,70 +325,70 @@ class RegexPlugin(Plugin[RegexOptions]):
continue
self._add_processed_regex_source_var(entry, source_var)
maybe_capture = regex_options.match.match_any(input_str=entry_variable_dict[source_var])
# If no capture
if maybe_capture is None:
# and no defaults
if not regex_options.has_defaults:
# Skip the entry if toggled
if self.plugin_options.skip_if_match_fails:
logger.info(
"Regex failed to match '%s' from '%s', skipping.",
source_var,
entry.title,
)
return None
if (
regex_options.exclude is not None
and regex_options.exclude.match_any(input_str=entry_variable_dict[source_var])
is not None
):
return self._try_skip_entry(entry=entry, source_var=source_var)
# Otherwise, error
raise RegexNoMatchException(
f"Regex failed to match '{source_var}' from '{entry.title}'"
# If match is present
if regex_options.match is not None:
maybe_capture = regex_options.match.match_any(
input_str=entry_variable_dict[source_var]
)
# And nothing matched
if maybe_capture is None:
# and no defaults
if not regex_options.has_defaults:
return self._try_skip_entry(entry=entry, source_var=source_var)
# otherwise, use defaults (apply them using the original entry source dict)
source_variables_and_overrides_dict = dict(
entry_variable_dict, **self.overrides.dict_with_format_strings
)
# otherwise, use defaults (apply them using the original entry source dict)
source_variables_and_overrides_dict = dict(
entry_variable_dict, **self.overrides.dict_with_format_strings
)
# add both the default...
entry.add_variables(
variables_to_add={
regex_options.capture_group_names[i]: default.apply_formatter(
variable_dict=source_variables_and_overrides_dict
)
for i, default in enumerate(regex_options.capture_group_defaults)
},
)
# and sanitized default
entry.add_variables(
variables_to_add={
f"{regex_options.capture_group_names[i]}_sanitized": sanitize_filename(
default.apply_formatter(
# add both the default...
entry.add_variables(
variables_to_add={
regex_options.capture_group_names[i]: default.apply_formatter(
variable_dict=source_variables_and_overrides_dict
)
)
for i, default in enumerate(regex_options.capture_group_defaults)
},
)
# There is a capture, add the source variables to the entry as
# {source_var}_capture_1, {source_var}_capture_2, ...
else:
# Add the value...
entry.add_variables(
variables_to_add={
regex_options.capture_group_names[i]: capture
for i, capture in enumerate(maybe_capture)
},
)
# And the sanitized value
entry.add_variables(
variables_to_add={
f"{regex_options.capture_group_names[i]}_sanitized": sanitize_filename(
capture
)
for i, capture in enumerate(maybe_capture)
},
)
for i, default in enumerate(regex_options.capture_group_defaults)
},
)
# and sanitized default
entry.add_variables(
variables_to_add={
f"{regex_options.capture_group_names[i]}_sanitized": sanitize_filename(
default.apply_formatter(
variable_dict=source_variables_and_overrides_dict
)
)
for i, default in enumerate(regex_options.capture_group_defaults)
},
)
# There is a capture, add the source variables to the entry as
# {source_var}_capture_1, {source_var}_capture_2, ...
else:
# Add the value...
entry.add_variables(
variables_to_add={
regex_options.capture_group_names[i]: capture
for i, capture in enumerate(maybe_capture)
},
)
# And the sanitized value
entry.add_variables(
variables_to_add={
f"{regex_options.capture_group_names[i]}_sanitized": sanitize_filename(
capture
)
for i, capture in enumerate(maybe_capture)
},
)
return entry

View file

@ -1,17 +1,20 @@
import copy
import re
from typing import Any
from typing import Dict
import mergedeep
import pytest
from expected_transaction_log import assert_transaction_log_matches
from ytdl_sub.config.preset import Preset
from ytdl_sub.config.config_file import ConfigFile
from ytdl_sub.subscriptions.subscription import Subscription
from ytdl_sub.utils.exceptions import RegexNoMatchException
from ytdl_sub.utils.exceptions import ValidationException
@pytest.fixture
def regex_subscription_dict(output_directory):
def regex_subscription_dict_base(output_directory):
return {
"preset": "music_video",
"download": {"url": "https://youtube.com/playlist?list=PL5BC0FC26BECA5A35"},
@ -24,13 +27,6 @@ def regex_subscription_dict(output_directory):
"regex": {
# tests that skip_if_match_fails defaults to True
"from": {
"title": {
"match": [
"should not cap (.+) - (.+)",
".*\\[(.+) - (Feb.+)]", # should filter out march video
],
"capture_group_names": ["title_type", "title_date"],
},
"description": {
"match": [".*http:\\/\\/(.+).com.*"],
"capture_group_names": ["description_website"],
@ -48,58 +44,149 @@ def regex_subscription_dict(output_directory):
},
},
},
"nfo_tags": {
"tags": {
"title_cap_1": "{title_type}",
"title_cap_1_sanitized": "{title_type_sanitized}",
"title_cap_2": "{title_date}",
"desc_cap": "{description_website}",
"upload_date_both_caps": "{upload_captured_year} and {upload_captured_month}",
"override_with_capture_variable": "{contains_regex_default}",
"override_with_capture_variable_sanitized": "{contains_regex_sanitized_default}",
}
},
"overrides": {
"in_regex_default": "in regex default",
"contains_regex_default": "contains {title_type}",
"contains_regex_sanitized_default": "contains {title_type_sanitized}",
},
}
@pytest.fixture
def regex_subscription_dict_no_match_fails(regex_subscription_dict):
regex_subscription_dict["regex"]["skip_if_match_fails"] = False
return regex_subscription_dict
def regex_subscription_dict(regex_subscription_dict_base, output_directory):
return mergedeep.merge(
regex_subscription_dict_base,
{
"regex": {
# tests that skip_if_match_fails defaults to True
"from": {
"title": {
"match": [
"should not cap (.+) - (.+)",
".*\\[(.+) - (Feb.+)]", # should filter out march video
],
"capture_group_names": ["title_type", "title_date"],
},
},
},
"nfo_tags": {
"tags": {
"title_cap_1": "{title_type}",
"title_cap_1_sanitized": "{title_type_sanitized}",
"title_cap_2": "{title_date}",
"desc_cap": "{description_website}",
"upload_date_both_caps": "{upload_captured_year} and {upload_captured_month}",
"override_with_capture_variable": "{contains_regex_default}",
"override_with_capture_variable_sanitized": "{contains_regex_sanitized_default}",
}
},
"overrides": {
"contains_regex_default": "contains {title_type}",
"contains_regex_sanitized_default": "contains {title_type_sanitized}",
},
},
)
@pytest.fixture
def regex_subscription_dict_exclude(regex_subscription_dict_base, output_directory):
return mergedeep.merge(
regex_subscription_dict_base,
{
"regex": {
# tests that skip_if_match_fails defaults to True
"from": {
"title": {
"exclude": [
"should not cap",
".*Feb.*", # should filter out march video
],
},
},
},
},
)
@pytest.fixture
def regex_subscription_dict_match_and_exclude(regex_subscription_dict_base, output_directory):
return mergedeep.merge(
regex_subscription_dict_base,
{
"regex": {
# tests that skip_if_match_fails defaults to True
"from": {
"title": {
"match": [
"should not cap (.+) - (.+)",
".*\\[(.+) - (Feb.+)]", # should filter out march video
],
"capture_group_names": ["title_type", "title_date"],
"exclude": [
"should not cap",
".*27.*", # should filter out Feb 27th video
],
},
},
},
"nfo_tags": {
"tags": {
"title_cap_1": "{title_type}",
"title_cap_1_sanitized": "{title_type_sanitized}",
"title_cap_2": "{title_date}",
"desc_cap": "{description_website}",
"upload_date_both_caps": "{upload_captured_year} and {upload_captured_month}",
"override_with_capture_variable": "{contains_regex_default}",
"override_with_capture_variable_sanitized": "{contains_regex_sanitized_default}",
}
},
"overrides": {
"contains_regex_default": "contains {title_type}",
"contains_regex_sanitized_default": "contains {title_type_sanitized}",
},
},
)
@pytest.fixture
def playlist_subscription(music_video_config, regex_subscription_dict):
playlist_preset = Preset.from_dict(
return Subscription.from_dict(
config=music_video_config,
preset_name="regex_capture_playlist_test",
preset_dict=regex_subscription_dict,
)
return Subscription.from_preset(
preset=playlist_preset,
@pytest.fixture
def playlist_subscription_no_match_fails(
music_video_config: ConfigFile, regex_subscription_dict: Dict[str, Any]
):
regex_subscription_dict["regex"]["skip_if_match_fails"] = False
return Subscription.from_dict(
config=music_video_config,
preset_name="regex_capture_playlist_test",
preset_dict=regex_subscription_dict,
)
@pytest.fixture
def playlist_subscription_no_match_fails(
music_video_config, regex_subscription_dict_no_match_fails
):
playlist_preset = Preset.from_dict(
def playlist_subscription_exclude(
music_video_config: ConfigFile, regex_subscription_dict_exclude: Dict[str, Any]
) -> Subscription:
return Subscription.from_dict(
config=music_video_config,
preset_name="regex_capture_playlist_test",
preset_dict=regex_subscription_dict_no_match_fails,
preset_name="regex_exclude_playlist_test",
preset_dict=regex_subscription_dict_exclude,
)
return Subscription.from_preset(
preset=playlist_preset,
@pytest.fixture
def playlist_subscription_match_and_exclude(
music_video_config: ConfigFile, regex_subscription_dict_match_and_exclude: Dict[str, Any]
) -> Subscription:
return Subscription.from_dict(
config=music_video_config,
preset_name="regex_match_and_exclude_playlist_test",
preset_dict=regex_subscription_dict_match_and_exclude,
)
@ -113,6 +200,27 @@ class TestRegex:
transaction_log_summary_file_name="plugins/test_regex.txt",
)
def test_regex_excludes_success(self, playlist_subscription_exclude, output_directory):
# Should only contain the march video
transaction_log = playlist_subscription_exclude.download(dry_run=True)
assert_transaction_log_matches(
output_directory=output_directory,
transaction_log=transaction_log,
transaction_log_summary_file_name="plugins/test_regex_exclude.txt",
)
def test_regex_match_and_excludes_success(
self, playlist_subscription_match_and_exclude, output_directory
):
# Should only contain the Feb 1st video
transaction_log = playlist_subscription_match_and_exclude.download(dry_run=True)
assert_transaction_log_matches(
output_directory=output_directory,
transaction_log=transaction_log,
transaction_log_summary_file_name="plugins/test_regex_match_and_exclude.txt",
regenerate_transaction_log=True,
)
def test_regex_fails_no_match(self, playlist_subscription_no_match_fails, output_directory):
with pytest.raises(
RegexNoMatchException,
@ -122,6 +230,34 @@ class TestRegex:
):
_ = playlist_subscription_no_match_fails.download(dry_run=True)
def test_regex_fails_capture_group_with_only_excludes(
self, regex_subscription_dict_exclude, music_video_config
):
regex_subscription_dict_exclude["regex"]["from"]["title"]["capture_group_names"] = ["uid"]
with pytest.raises(
ValidationException,
match=re.escape(
"capture group parameters requires at least one `match` to be specified"
),
):
_ = Subscription.from_dict(
config=music_video_config,
preset_name="test_regex_fails_capture_group_is_source_variable",
preset_dict=regex_subscription_dict_exclude,
)
def test_regex_fails_no_match_or_exclude(self, regex_subscription_dict, music_video_config):
del regex_subscription_dict["regex"]["from"]["title"]["match"]
with pytest.raises(
ValidationException,
match=re.escape("must specify either `match` or `exclude`"),
):
_ = Subscription.from_dict(
config=music_video_config,
preset_name="test_regex_fails_capture_group_is_source_variable",
preset_dict=regex_subscription_dict,
)
def test_regex_fails_capture_group_is_source_variable(
self, regex_subscription_dict, music_video_config
):

View file

@ -0,0 +1,15 @@
Files created:
----------------------------------------
{output_directory}
.ytdl-sub-regex_exclude_playlist_test-download-archive.json
{output_directory}/Project Zombie
Project Zombie - Jesse's Minecraft Server [Trailer - Mar.21]-thumb.jpg
Project Zombie - Jesse's Minecraft Server [Trailer - Mar.21].info.json
Project Zombie - Jesse's Minecraft Server [Trailer - Mar.21].mp4
Project Zombie - Jesse's Minecraft Server [Trailer - Mar.21].nfo
NFO tags:
musicvideo:
album: Music Videos
artist: Project Zombie
title: Jesse's Minecraft Server [Trailer - Mar.21]
year: 2011

View file

@ -0,0 +1,22 @@
Files created:
----------------------------------------
{output_directory}
.ytdl-sub-regex_match_and_exclude_playlist_test-download-archive.json
{output_directory}/Project Zombie
Project Zombie - Jesse's Minecraft Server [Trailer - Feb.1]-thumb.jpg
Project Zombie - Jesse's Minecraft Server [Trailer - Feb.1].info.json
Project Zombie - Jesse's Minecraft Server [Trailer - Feb.1].mp4
Project Zombie - Jesse's Minecraft Server [Trailer - Feb.1].nfo
NFO tags:
musicvideo:
album: Music Videos
artist: Project Zombie
desc_cap: www.jesseminecraft.webs
override_with_capture_variable: contains Trailer
override_with_capture_variable_sanitized: contains Trailer
title: Jesse's Minecraft Server [Trailer - Feb.1]
title_cap_1: Trailer
title_cap_1_sanitized: Trailer
title_cap_2: Feb.1
upload_date_both_caps: First and Second containing in regex default
year: 2011