danvics/ytdl-sub
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

tests

Browse source
This commit is contained in:
Jesse Bannon 2023-05-22 10:08:18 -07:00
parent 5b0750be4f
commit 624b259da6
4 changed files with 261 additions and 87 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

103
src/ytdl_sub/plugins/regex.py
View file

@ -40,7 +40,7 @@ class SourceVariableRegex(StrictDictValidator):
if self._match is None and self._exclude is None:
raise self._validation_exception("must specify either `match` or `exclude`")
if self._match is None and (self._capture_group_defaults or self._capture_group_names):
if self._match is None and (self._capture_group_defaults or self._capture_group_names.list):
raise self._validation_exception(
"capture group parameters requires at least one `match` to be specified"
)
@ -56,7 +56,7 @@ class SourceVariableRegex(StrictDictValidator):
)
# If there are capture groups, ensure there are capture group names
if len(self._capture_group_names.list) != self._match.num_capture_groups:
if self._match and (len(self._capture_group_names.list) != self._match.num_capture_groups):
raise self._validation_exception(
f"number of capture group names must match number of capture groups, "
f"{len(self._capture_group_names.list)} != {self._match.num_capture_groups}"
@ -66,7 +66,7 @@ class SourceVariableRegex(StrictDictValidator):
def match(self) -> Optional[RegexListValidator]:
"""
List of regex strings to try to match against a source variable. Each regex
string must have the same number of capture groups. Can not be used with ``match``
string must have the same number of capture groups.
"""
return self._match
@ -74,7 +74,8 @@ class SourceVariableRegex(StrictDictValidator):
def exclude(self) -> Optional[RegexListValidator]:
"""
List of regex strings to try to match against a source variable. If one of the regex strings
match, then the entry will be skipped.
match, then the entry will be skipped. If both ``exclude`` and ``match`` are specified,
entries will get skipped if the regex matches against both ``exclude`` and ``match``.
"""
return self._exclude
@ -332,62 +333,62 @@ class RegexPlugin(Plugin[RegexOptions]):
):
return self._try_skip_entry(entry=entry, source_var=source_var)
maybe_capture: Optional[List[str]] = None
# If match is present
if regex_options.match is not None:
maybe_capture = regex_options.match.match_any(
input_str=entry_variable_dict[source_var]
)
# If no capture
if maybe_capture is None:
# and no defaults
if not regex_options.has_defaults:
return self._try_skip_entry(entry=entry, source_var=source_var)
# And nothing matched
if maybe_capture is None:
# and no defaults
if not regex_options.has_defaults:
return self._try_skip_entry(entry=entry, source_var=source_var)
# otherwise, use defaults (apply them using the original entry source dict)
source_variables_and_overrides_dict = dict(
entry_variable_dict, **self.overrides.dict_with_format_strings
)
# otherwise, use defaults (apply them using the original entry source dict)
source_variables_and_overrides_dict = dict(
entry_variable_dict, **self.overrides.dict_with_format_strings
)
# add both the default...
entry.add_variables(
variables_to_add={
regex_options.capture_group_names[i]: default.apply_formatter(
variable_dict=source_variables_and_overrides_dict
)
for i, default in enumerate(regex_options.capture_group_defaults)
},
)
# and sanitized default
entry.add_variables(
variables_to_add={
f"{regex_options.capture_group_names[i]}_sanitized": sanitize_filename(
default.apply_formatter(
# add both the default...
entry.add_variables(
variables_to_add={
regex_options.capture_group_names[i]: default.apply_formatter(
variable_dict=source_variables_and_overrides_dict
)
)
for i, default in enumerate(regex_options.capture_group_defaults)
},
)
# There is a capture, add the source variables to the entry as
# {source_var}_capture_1, {source_var}_capture_2, ...
else:
# Add the value...
entry.add_variables(
variables_to_add={
regex_options.capture_group_names[i]: capture
for i, capture in enumerate(maybe_capture)
},
)
# And the sanitized value
entry.add_variables(
variables_to_add={
f"{regex_options.capture_group_names[i]}_sanitized": sanitize_filename(
capture
)
for i, capture in enumerate(maybe_capture)
},
)
for i, default in enumerate(regex_options.capture_group_defaults)
},
)
# and sanitized default
entry.add_variables(
variables_to_add={
f"{regex_options.capture_group_names[i]}_sanitized": sanitize_filename(
default.apply_formatter(
variable_dict=source_variables_and_overrides_dict
)
)
for i, default in enumerate(regex_options.capture_group_defaults)
},
)
# There is a capture, add the source variables to the entry as
# {source_var}_capture_1, {source_var}_capture_2, ...
else:
# Add the value...
entry.add_variables(
variables_to_add={
regex_options.capture_group_names[i]: capture
for i, capture in enumerate(maybe_capture)
},
)
# And the sanitized value
entry.add_variables(
variables_to_add={
f"{regex_options.capture_group_names[i]}_sanitized": sanitize_filename(
capture
)
for i, capture in enumerate(maybe_capture)
},
)
return entry

208
tests/e2e/plugins/test_regex.py
View file

@ -1,17 +1,20 @@
import copy
import re
from typing import Any
from typing import Dict
import mergedeep
import pytest
from expected_transaction_log import assert_transaction_log_matches
from ytdl_sub.config.preset import Preset
from ytdl_sub.config.config_file import ConfigFile
from ytdl_sub.subscriptions.subscription import Subscription
from ytdl_sub.utils.exceptions import RegexNoMatchException
from ytdl_sub.utils.exceptions import ValidationException
@pytest.fixture
def regex_subscription_dict(output_directory):
def regex_subscription_dict_base(output_directory):
return {
"preset": "music_video",
"download": {"url": "https://youtube.com/playlist?list=PL5BC0FC26BECA5A35"},
@ -24,13 +27,6 @@ def regex_subscription_dict(output_directory):
"regex": {
# tests that skip_if_match_fails defaults to True
"from": {
"title": {
"match": [
"should not cap (.+) - (.+)",
".*\\[(.+) - (Feb.+)]", # should filter out march video
],
"capture_group_names": ["title_type", "title_date"],
},
"description": {
"match": [".*http:\\/\\/(.+).com.*"],
"capture_group_names": ["description_website"],
@ -48,58 +44,149 @@ def regex_subscription_dict(output_directory):
},
},
},
"nfo_tags": {
"tags": {
"title_cap_1": "{title_type}",
"title_cap_1_sanitized": "{title_type_sanitized}",
"title_cap_2": "{title_date}",
"desc_cap": "{description_website}",
"upload_date_both_caps": "{upload_captured_year} and {upload_captured_month}",
"override_with_capture_variable": "{contains_regex_default}",
"override_with_capture_variable_sanitized": "{contains_regex_sanitized_default}",
}
},
"overrides": {
"in_regex_default": "in regex default",
"contains_regex_default": "contains {title_type}",
"contains_regex_sanitized_default": "contains {title_type_sanitized}",
},
}
@pytest.fixture
def regex_subscription_dict_no_match_fails(regex_subscription_dict):
regex_subscription_dict["regex"]["skip_if_match_fails"] = False
return regex_subscription_dict
def regex_subscription_dict(regex_subscription_dict_base, output_directory):
return mergedeep.merge(
regex_subscription_dict_base,
{
"regex": {
# tests that skip_if_match_fails defaults to True
"from": {
"title": {
"match": [
"should not cap (.+) - (.+)",
".*\\[(.+) - (Feb.+)]", # should filter out march video
],
"capture_group_names": ["title_type", "title_date"],
},
},
},
"nfo_tags": {
"tags": {
"title_cap_1": "{title_type}",
"title_cap_1_sanitized": "{title_type_sanitized}",
"title_cap_2": "{title_date}",
"desc_cap": "{description_website}",
"upload_date_both_caps": "{upload_captured_year} and {upload_captured_month}",
"override_with_capture_variable": "{contains_regex_default}",
"override_with_capture_variable_sanitized": "{contains_regex_sanitized_default}",
}
},
"overrides": {
"contains_regex_default": "contains {title_type}",
"contains_regex_sanitized_default": "contains {title_type_sanitized}",
},
},
)
@pytest.fixture
def regex_subscription_dict_exclude(regex_subscription_dict_base, output_directory):
return mergedeep.merge(
regex_subscription_dict_base,
{
"regex": {
# tests that skip_if_match_fails defaults to True
"from": {
"title": {
"exclude": [
"should not cap",
".*Feb.*", # should filter out march video
],
},
},
},
},
)
@pytest.fixture
def regex_subscription_dict_match_and_exclude(regex_subscription_dict_base, output_directory):
return mergedeep.merge(
regex_subscription_dict_base,
{
"regex": {
# tests that skip_if_match_fails defaults to True
"from": {
"title": {
"match": [
"should not cap (.+) - (.+)",
".*\\[(.+) - (Feb.+)]", # should filter out march video
],
"capture_group_names": ["title_type", "title_date"],
"exclude": [
"should not cap",
".*27.*", # should filter out Feb 27th video
],
},
},
},
"nfo_tags": {
"tags": {
"title_cap_1": "{title_type}",
"title_cap_1_sanitized": "{title_type_sanitized}",
"title_cap_2": "{title_date}",
"desc_cap": "{description_website}",
"upload_date_both_caps": "{upload_captured_year} and {upload_captured_month}",
"override_with_capture_variable": "{contains_regex_default}",
"override_with_capture_variable_sanitized": "{contains_regex_sanitized_default}",
}
},
"overrides": {
"contains_regex_default": "contains {title_type}",
"contains_regex_sanitized_default": "contains {title_type_sanitized}",
},
},
)
@pytest.fixture
def playlist_subscription(music_video_config, regex_subscription_dict):
playlist_preset = Preset.from_dict(
return Subscription.from_dict(
config=music_video_config,
preset_name="regex_capture_playlist_test",
preset_dict=regex_subscription_dict,
)
return Subscription.from_preset(
preset=playlist_preset,
@pytest.fixture
def playlist_subscription_no_match_fails(
music_video_config: ConfigFile, regex_subscription_dict: Dict[str, Any]
):
regex_subscription_dict["regex"]["skip_if_match_fails"] = False
return Subscription.from_dict(
config=music_video_config,
preset_name="regex_capture_playlist_test",
preset_dict=regex_subscription_dict,
)
@pytest.fixture
def playlist_subscription_no_match_fails(
music_video_config, regex_subscription_dict_no_match_fails
):
playlist_preset = Preset.from_dict(
def playlist_subscription_exclude(
music_video_config: ConfigFile, regex_subscription_dict_exclude: Dict[str, Any]
) -> Subscription:
return Subscription.from_dict(
config=music_video_config,
preset_name="regex_capture_playlist_test",
preset_dict=regex_subscription_dict_no_match_fails,
preset_name="regex_exclude_playlist_test",
preset_dict=regex_subscription_dict_exclude,
)
return Subscription.from_preset(
preset=playlist_preset,
@pytest.fixture
def playlist_subscription_match_and_exclude(
music_video_config: ConfigFile, regex_subscription_dict_match_and_exclude: Dict[str, Any]
) -> Subscription:
return Subscription.from_dict(
config=music_video_config,
preset_name="regex_match_and_exclude_playlist_test",
preset_dict=regex_subscription_dict_match_and_exclude,
)
@ -113,6 +200,27 @@ class TestRegex:
transaction_log_summary_file_name="plugins/test_regex.txt",
)
def test_regex_excludes_success(self, playlist_subscription_exclude, output_directory):
# Should only contain the march video
transaction_log = playlist_subscription_exclude.download(dry_run=True)
assert_transaction_log_matches(
output_directory=output_directory,
transaction_log=transaction_log,
transaction_log_summary_file_name="plugins/test_regex_exclude.txt",
)
def test_regex_match_and_excludes_success(
self, playlist_subscription_match_and_exclude, output_directory
):
# Should only contain the Feb 1st video
transaction_log = playlist_subscription_match_and_exclude.download(dry_run=True)
assert_transaction_log_matches(
output_directory=output_directory,
transaction_log=transaction_log,
transaction_log_summary_file_name="plugins/test_regex_match_and_exclude.txt",
regenerate_transaction_log=True,
)
def test_regex_fails_no_match(self, playlist_subscription_no_match_fails, output_directory):
with pytest.raises(
RegexNoMatchException,
@ -122,6 +230,34 @@ class TestRegex:
):
_ = playlist_subscription_no_match_fails.download(dry_run=True)
def test_regex_fails_capture_group_with_only_excludes(
self, regex_subscription_dict_exclude, music_video_config
):
regex_subscription_dict_exclude["regex"]["from"]["title"]["capture_group_names"] = ["uid"]
with pytest.raises(
ValidationException,
match=re.escape(
"capture group parameters requires at least one `match` to be specified"
),
):
_ = Subscription.from_dict(
config=music_video_config,
preset_name="test_regex_fails_capture_group_is_source_variable",
preset_dict=regex_subscription_dict_exclude,
)
def test_regex_fails_no_match_or_exclude(self, regex_subscription_dict, music_video_config):
del regex_subscription_dict["regex"]["from"]["title"]["match"]
with pytest.raises(
ValidationException,
match=re.escape("must specify either `match` or `exclude`"),
):
_ = Subscription.from_dict(
config=music_video_config,
preset_name="test_regex_fails_capture_group_is_source_variable",
preset_dict=regex_subscription_dict,
)
def test_regex_fails_capture_group_is_source_variable(
self, regex_subscription_dict, music_video_config
):

15
tests/resources/transaction_log_summaries/plugins/test_regex_exclude.txt Normal file
View file

@ -0,0 +1,15 @@
Files created:
----------------------------------------
{output_directory}
.ytdl-sub-regex_exclude_playlist_test-download-archive.json
{output_directory}/Project Zombie
Project Zombie - Jesse's Minecraft Server [Trailer - Mar.21]-thumb.jpg
Project Zombie - Jesse's Minecraft Server [Trailer - Mar.21].info.json
Project Zombie - Jesse's Minecraft Server [Trailer - Mar.21].mp4
Project Zombie - Jesse's Minecraft Server [Trailer - Mar.21].nfo
NFO tags:
musicvideo:
album: Music Videos
artist: Project Zombie
title: Jesse's Minecraft Server [Trailer - Mar.21]
year: 2011

22
tests/resources/transaction_log_summaries/plugins/test_regex_match_and_exclude.txt Normal file
View file

@ -0,0 +1,22 @@
Files created:
----------------------------------------
{output_directory}
.ytdl-sub-regex_match_and_exclude_playlist_test-download-archive.json
{output_directory}/Project Zombie
Project Zombie - Jesse's Minecraft Server [Trailer - Feb.1]-thumb.jpg
Project Zombie - Jesse's Minecraft Server [Trailer - Feb.1].info.json
Project Zombie - Jesse's Minecraft Server [Trailer - Feb.1].mp4
Project Zombie - Jesse's Minecraft Server [Trailer - Feb.1].nfo
NFO tags:
musicvideo:
album: Music Videos
artist: Project Zombie
desc_cap: www.jesseminecraft.webs
override_with_capture_variable: contains Trailer
override_with_capture_variable_sanitized: contains Trailer
title: Jesse's Minecraft Server [Trailer - Feb.1]
title_cap_1: Trailer
title_cap_1_sanitized: Trailer
title_cap_2: Feb.1
upload_date_both_caps: First and Second containing in regex default
year: 2011