[DEV] Unit test filter plugin (#1066)
This commit is contained in:
Jesse Bannon
GitHub
parent
513d1f5a97
commit
fd5882051a
10 changed files with 315 additions and 438 deletions
|
|
@ -10,7 +10,7 @@ from ytdl_sub.script.types.resolvable import Boolean
|
||||||
from ytdl_sub.script.types.resolvable import Float
|
from ytdl_sub.script.types.resolvable import Float
|
||||||
from ytdl_sub.script.types.resolvable import Integer
|
from ytdl_sub.script.types.resolvable import Integer
|
||||||
from ytdl_sub.script.types.resolvable import String
|
from ytdl_sub.script.types.resolvable import String
|
||||||
from ytdl_sub.script.utils.exceptions import RuntimeException
|
from ytdl_sub.script.utils.exceptions import FunctionRuntimeException
|
||||||
|
|
||||||
|
|
||||||
def _re_output_to_array(re_out: Match[AnyStr] | None) -> Array:
|
def _re_output_to_array(re_out: Match[AnyStr] | None) -> Array:
|
||||||
|
|
@ -115,12 +115,12 @@ class RegexFunctions:
|
||||||
# ["2020-02-27", "2020", "02"]
|
# ["2020-02-27", "2020", "02"]
|
||||||
"""
|
"""
|
||||||
if len(regex_array) == 0:
|
if len(regex_array) == 0:
|
||||||
raise RuntimeException("regex_array must contain at least a single element")
|
raise FunctionRuntimeException("regex_array must contain at least a single element")
|
||||||
|
|
||||||
regex_list: List[String] = []
|
regex_list: List[String] = []
|
||||||
for regex in regex_array.value:
|
for regex in regex_array.value:
|
||||||
if not isinstance(regex, String):
|
if not isinstance(regex, String):
|
||||||
raise RuntimeException("All regex_array elements must be strings")
|
raise FunctionRuntimeException("All regex_array elements must be strings")
|
||||||
regex_list.append(regex)
|
regex_list.append(regex)
|
||||||
|
|
||||||
if default is None:
|
if default is None:
|
||||||
|
|
@ -129,13 +129,13 @@ class RegexFunctions:
|
||||||
RegexFunctions.regex_capture_groups(regex).value != num_capture_groups
|
RegexFunctions.regex_capture_groups(regex).value != num_capture_groups
|
||||||
for regex in regex_list[1:]
|
for regex in regex_list[1:]
|
||||||
):
|
):
|
||||||
raise RuntimeException(
|
raise FunctionRuntimeException(
|
||||||
"regex_array elements must contain the same number of capture groups"
|
"regex_array elements must contain the same number of capture groups"
|
||||||
)
|
)
|
||||||
elif any(
|
elif any(
|
||||||
RegexFunctions.regex_capture_groups(regex).value > len(default) for regex in regex_list
|
RegexFunctions.regex_capture_groups(regex).value > len(default) for regex in regex_list
|
||||||
):
|
):
|
||||||
raise RuntimeException(
|
raise FunctionRuntimeException(
|
||||||
"When using %regex_capture_array, number of regex capture groups must be less than "
|
"When using %regex_capture_array, number of regex capture groups must be less than "
|
||||||
"or equal to the number of defaults"
|
"or equal to the number of defaults"
|
||||||
)
|
)
|
||||||
|
|
@ -147,7 +147,7 @@ class RegexFunctions:
|
||||||
break
|
break
|
||||||
|
|
||||||
if len(output) == 0 and not default:
|
if len(output) == 0 and not default:
|
||||||
raise RuntimeException(
|
raise FunctionRuntimeException(
|
||||||
f"no regex strings were captured for input string {string.value}"
|
f"no regex strings were captured for input string {string.value}"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -1,299 +0,0 @@
|
||||||
import re
|
|
||||||
from typing import Any
|
|
||||||
from typing import Dict
|
|
||||||
|
|
||||||
import mergedeep
|
|
||||||
import pytest
|
|
||||||
from expected_transaction_log import assert_transaction_log_matches
|
|
||||||
|
|
||||||
from ytdl_sub.config.config_file import ConfigFile
|
|
||||||
from ytdl_sub.script.utils.exceptions import RuntimeException
|
|
||||||
from ytdl_sub.script.utils.exceptions import UserThrownRuntimeError
|
|
||||||
from ytdl_sub.subscriptions.subscription import Subscription
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.fixture
|
|
||||||
def regex_subscription_dict_base(output_directory):
|
|
||||||
return {
|
|
||||||
"preset": "Jellyfin Music Videos",
|
|
||||||
# override the output directory with our fixture-generated dir
|
|
||||||
"output_options": {"output_directory": output_directory},
|
|
||||||
"format": "best[height<=480]", # download the worst format so it is fast
|
|
||||||
"filter_include": ["{%not(%is_null(description_website))}"],
|
|
||||||
"overrides": {
|
|
||||||
"in_regex_default": "in regex default",
|
|
||||||
"url": "https://youtube.com/playlist?list=PL5BC0FC26BECA5A35",
|
|
||||||
"upload_capture": """{
|
|
||||||
%regex_capture_many_with_defaults(
|
|
||||||
upload_date_standardized,
|
|
||||||
["([0-9]+)-([0-9]+)-27"],
|
|
||||||
["First", %concat("Second containing ", in_regex_default)]
|
|
||||||
)
|
|
||||||
}""",
|
|
||||||
"upload_captured_year": "{%array_at(upload_capture, 1)}",
|
|
||||||
"upload_captured_month": "{%array_at(upload_capture, 2)}",
|
|
||||||
"description_capture": """{
|
|
||||||
%regex_capture_many_with_defaults(
|
|
||||||
description,
|
|
||||||
[".*http:\\/\\/(.+).com.*"],
|
|
||||||
[null]
|
|
||||||
)
|
|
||||||
}""",
|
|
||||||
"description_website": "{%array_at(description_capture, 1)}",
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.fixture
|
|
||||||
def regex_subscription_dict(regex_subscription_dict_base, output_directory):
|
|
||||||
return mergedeep.merge(
|
|
||||||
regex_subscription_dict_base,
|
|
||||||
{
|
|
||||||
"nfo_tags": {
|
|
||||||
"tags": {
|
|
||||||
"title_cap_1": "{title_type}",
|
|
||||||
"title_cap_1_sanitized": "{title_type_sanitized}",
|
|
||||||
"title_cap_2": "{title_date}",
|
|
||||||
"desc_cap": "{description_website}",
|
|
||||||
"upload_date_both_caps": "{upload_captured_year} and {upload_captured_month}",
|
|
||||||
"override_with_capture_variable": "{contains_regex_default}",
|
|
||||||
"override_with_capture_variable_sanitized": "{contains_regex_sanitized_default}",
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"filter_exclude": ["{%is_null(title_type)}"],
|
|
||||||
"overrides": {
|
|
||||||
"title_capture_list": """{
|
|
||||||
%regex_capture_many_with_defaults(
|
|
||||||
title,
|
|
||||||
[ "should not cap (.+) - (.+)", ".*\\[(.+) - (Feb.+)]" ],
|
|
||||||
[ null, null ]
|
|
||||||
)
|
|
||||||
}""",
|
|
||||||
"title_type": "{%array_at(title_capture_list, 1)}",
|
|
||||||
"title_date": "{%array_at(title_capture_list, 2)}",
|
|
||||||
"contains_regex_default": "contains {title_type}",
|
|
||||||
"contains_regex_sanitized_default": "contains {title_type_sanitized}",
|
|
||||||
},
|
|
||||||
},
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.fixture
|
|
||||||
def regex_subscription_dict_exclude(regex_subscription_dict_base, output_directory):
|
|
||||||
return mergedeep.merge(
|
|
||||||
regex_subscription_dict_base,
|
|
||||||
{
|
|
||||||
"filter_exclude": [
|
|
||||||
"""{
|
|
||||||
%regex_search_any(
|
|
||||||
title,
|
|
||||||
[ "should not cap", ".*Feb.*" ]
|
|
||||||
)
|
|
||||||
}"""
|
|
||||||
]
|
|
||||||
},
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.fixture
|
|
||||||
def regex_subscription_dict_match_and_exclude(regex_subscription_dict_base, output_directory):
|
|
||||||
return mergedeep.merge(
|
|
||||||
regex_subscription_dict_base,
|
|
||||||
{
|
|
||||||
"regex": {
|
|
||||||
# tests that skip_if_match_fails defaults to True
|
|
||||||
"from": {
|
|
||||||
"title": {
|
|
||||||
"match": [
|
|
||||||
"should not cap (.+) - (.+)",
|
|
||||||
".*\\[(.+) - (Feb.+)]", # should filter out march video
|
|
||||||
],
|
|
||||||
"capture_group_names": ["title_type", "title_date"],
|
|
||||||
"exclude": [
|
|
||||||
"should not cap",
|
|
||||||
".*27.*", # should filter out Feb 27th video
|
|
||||||
],
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
"nfo_tags": {
|
|
||||||
"tags": {
|
|
||||||
"title_cap_1": "{title_type}",
|
|
||||||
"title_cap_1_sanitized": "{title_type_sanitized}",
|
|
||||||
"title_cap_2": "{title_date}",
|
|
||||||
"desc_cap": "{description_website}",
|
|
||||||
"upload_date_both_caps": "{upload_captured_year} and {upload_captured_month}",
|
|
||||||
"override_with_capture_variable": "{contains_regex_default}",
|
|
||||||
"override_with_capture_variable_sanitized": "{contains_regex_sanitized_default}",
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"overrides": {
|
|
||||||
"contains_regex_default": "contains {title_type}",
|
|
||||||
"contains_regex_sanitized_default": "contains {title_type_sanitized}",
|
|
||||||
},
|
|
||||||
},
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.fixture
|
|
||||||
def regex_subscription_dict_match_and_exclude_override_variable(
|
|
||||||
regex_subscription_dict_base, output_directory
|
|
||||||
):
|
|
||||||
return mergedeep.merge(
|
|
||||||
regex_subscription_dict_base,
|
|
||||||
{
|
|
||||||
"regex": {
|
|
||||||
# tests that skip_if_match_fails defaults to True
|
|
||||||
"from": {
|
|
||||||
"override_title": {
|
|
||||||
"exclude": [
|
|
||||||
"should not cap",
|
|
||||||
".*Feb.*", # should filter out march video
|
|
||||||
],
|
|
||||||
},
|
|
||||||
"override_description": {
|
|
||||||
"match": [".*http:\\/\\/(.+).com.*"],
|
|
||||||
"capture_group_names": ["override_description_website"],
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
"overrides": {"override_title": "{title}", "override_description": "{description}"},
|
|
||||||
},
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.fixture
|
|
||||||
def playlist_subscription(default_config, regex_subscription_dict):
|
|
||||||
return Subscription.from_dict(
|
|
||||||
config=default_config,
|
|
||||||
preset_name="regex_capture_playlist_test",
|
|
||||||
preset_dict=regex_subscription_dict,
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.fixture
|
|
||||||
def playlist_subscription_no_match_fails(
|
|
||||||
default_config: ConfigFile, regex_subscription_dict: Dict[str, Any]
|
|
||||||
):
|
|
||||||
regex_subscription_dict["overrides"][
|
|
||||||
"title_capture_list"
|
|
||||||
] = """{
|
|
||||||
%regex_capture_many_required(
|
|
||||||
title,
|
|
||||||
[ "should not cap (.+) - (.+)", ".*\\[(.+) - (Feb.+)]" ]
|
|
||||||
)
|
|
||||||
}"""
|
|
||||||
|
|
||||||
return Subscription.from_dict(
|
|
||||||
config=default_config,
|
|
||||||
preset_name="regex_capture_playlist_test",
|
|
||||||
preset_dict=regex_subscription_dict,
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.fixture
|
|
||||||
def playlist_subscription_exclude(
|
|
||||||
default_config: ConfigFile, regex_subscription_dict_exclude: Dict[str, Any]
|
|
||||||
) -> Subscription:
|
|
||||||
return Subscription.from_dict(
|
|
||||||
config=default_config,
|
|
||||||
preset_name="regex_exclude_playlist_test",
|
|
||||||
preset_dict=regex_subscription_dict_exclude,
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.fixture
|
|
||||||
def playlist_subscription_overrides(
|
|
||||||
default_config: ConfigFile,
|
|
||||||
regex_subscription_dict_match_and_exclude_override_variable: Dict[str, Any],
|
|
||||||
) -> Subscription:
|
|
||||||
return Subscription.from_dict(
|
|
||||||
config=default_config,
|
|
||||||
preset_name="regex_using_overrides_test",
|
|
||||||
preset_dict=regex_subscription_dict_match_and_exclude_override_variable,
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.fixture
|
|
||||||
def playlist_subscription_match_and_exclude(
|
|
||||||
default_config: ConfigFile, regex_subscription_dict_match_and_exclude: Dict[str, Any]
|
|
||||||
) -> Subscription:
|
|
||||||
return Subscription.from_dict(
|
|
||||||
config=default_config,
|
|
||||||
preset_name="regex_match_and_exclude_playlist_test",
|
|
||||||
preset_dict=regex_subscription_dict_match_and_exclude,
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
class TestFilter:
|
|
||||||
def test_regex_success(self, playlist_subscription, output_directory):
|
|
||||||
# Only dry run is needed to see if capture variables are created
|
|
||||||
transaction_log = playlist_subscription.download(dry_run=True)
|
|
||||||
assert_transaction_log_matches(
|
|
||||||
output_directory=output_directory,
|
|
||||||
transaction_log=transaction_log,
|
|
||||||
transaction_log_summary_file_name="plugins/test_regex.txt",
|
|
||||||
)
|
|
||||||
|
|
||||||
def test_regex_excludes_success(self, playlist_subscription_exclude, output_directory):
|
|
||||||
# Should only contain the march video
|
|
||||||
transaction_log = playlist_subscription_exclude.download(dry_run=True)
|
|
||||||
assert_transaction_log_matches(
|
|
||||||
output_directory=output_directory,
|
|
||||||
transaction_log=transaction_log,
|
|
||||||
transaction_log_summary_file_name="plugins/test_regex_exclude.txt",
|
|
||||||
)
|
|
||||||
|
|
||||||
def test_regex_match_and_excludes_success(
|
|
||||||
self, playlist_subscription_match_and_exclude, output_directory
|
|
||||||
):
|
|
||||||
# Should only contain the Feb 1st video
|
|
||||||
transaction_log = playlist_subscription_match_and_exclude.download(dry_run=True)
|
|
||||||
assert_transaction_log_matches(
|
|
||||||
output_directory=output_directory,
|
|
||||||
transaction_log=transaction_log,
|
|
||||||
transaction_log_summary_file_name="plugins/test_regex_match_and_exclude.txt",
|
|
||||||
)
|
|
||||||
|
|
||||||
def test_regex_using_overrides_success(self, playlist_subscription_overrides, output_directory):
|
|
||||||
# Should only contain the march video
|
|
||||||
transaction_log = playlist_subscription_overrides.download(dry_run=True)
|
|
||||||
assert_transaction_log_matches(
|
|
||||||
output_directory=output_directory,
|
|
||||||
transaction_log=transaction_log,
|
|
||||||
transaction_log_summary_file_name="plugins/test_regex_overrides.txt",
|
|
||||||
)
|
|
||||||
|
|
||||||
def test_regex_fails_no_match(self, playlist_subscription_no_match_fails, output_directory):
|
|
||||||
with pytest.raises(
|
|
||||||
RuntimeException,
|
|
||||||
match=re.escape(
|
|
||||||
"no regex strings were captured for input string Jesse's Minecraft Server [Trailer - Mar.21]"
|
|
||||||
),
|
|
||||||
):
|
|
||||||
_ = playlist_subscription_no_match_fails.download(dry_run=True)
|
|
||||||
|
|
||||||
def test_regex_fails_unequal_defaults(self, regex_subscription_dict, default_config):
|
|
||||||
regex_subscription_dict["overrides"][
|
|
||||||
"title_capture_list"
|
|
||||||
] = """{
|
|
||||||
%regex_capture_many_with_defaults(
|
|
||||||
title,
|
|
||||||
[ "should not cap (.+) - (.+)", ".*\\[(.+) - (Feb.+)]" ],
|
|
||||||
[ "one default, expects >= 2" ]
|
|
||||||
)
|
|
||||||
}"""
|
|
||||||
|
|
||||||
subscription = Subscription.from_dict(
|
|
||||||
config=default_config,
|
|
||||||
preset_name="regex_capture_playlist_test",
|
|
||||||
preset_dict=regex_subscription_dict,
|
|
||||||
)
|
|
||||||
|
|
||||||
with pytest.raises(
|
|
||||||
RuntimeException,
|
|
||||||
match=re.escape(
|
|
||||||
"When using %regex_capture_array, number of regex capture groups must be less than or equal to the number of defaults"
|
|
||||||
),
|
|
||||||
):
|
|
||||||
_ = subscription.download(dry_run=True)
|
|
||||||
|
|
@ -0,0 +1,105 @@
|
||||||
|
Files created:
|
||||||
|
----------------------------------------
|
||||||
|
{output_directory}
|
||||||
|
.ytdl-sub-subscription_test-download-archive.json
|
||||||
|
{output_directory}/subscription_test
|
||||||
|
Mock Entry 20-1.info.json
|
||||||
|
Mock Entry 20-1.jpg
|
||||||
|
Mock Entry 20-1.mp4
|
||||||
|
Video Tags:
|
||||||
|
album: Music Videos
|
||||||
|
artist: subscription_test
|
||||||
|
genre: ytdl-sub
|
||||||
|
premiered: 2020-08-08
|
||||||
|
title: Mock Entry 20-1
|
||||||
|
year: 2020
|
||||||
|
Mock Entry 20-1.nfo
|
||||||
|
NFO tags:
|
||||||
|
musicvideo:
|
||||||
|
album: Music Videos
|
||||||
|
artist: subscription_test
|
||||||
|
desc_cap: Description
|
||||||
|
genre: ytdl-sub
|
||||||
|
override_with_capture_variable: contains 20
|
||||||
|
override_with_capture_variable_sanitized: contains 20
|
||||||
|
premiered: 2020-08-08
|
||||||
|
title: Mock Entry 20-1
|
||||||
|
title_cap_1: 20
|
||||||
|
title_cap_1_sanitized: 20
|
||||||
|
title_cap_2: 1
|
||||||
|
upload_date_both_caps: First and Second containing in regex default
|
||||||
|
Mock Entry 20-2.info.json
|
||||||
|
Mock Entry 20-2.jpg
|
||||||
|
Mock Entry 20-2.mp4
|
||||||
|
Video Tags:
|
||||||
|
album: Music Videos
|
||||||
|
artist: subscription_test
|
||||||
|
genre: ytdl-sub
|
||||||
|
premiered: 2020-08-08
|
||||||
|
title: Mock Entry 20-2
|
||||||
|
year: 2020
|
||||||
|
Mock Entry 20-2.nfo
|
||||||
|
NFO tags:
|
||||||
|
musicvideo:
|
||||||
|
album: Music Videos
|
||||||
|
artist: subscription_test
|
||||||
|
desc_cap: Description
|
||||||
|
genre: ytdl-sub
|
||||||
|
override_with_capture_variable: contains 20
|
||||||
|
override_with_capture_variable_sanitized: contains 20
|
||||||
|
premiered: 2020-08-08
|
||||||
|
title: Mock Entry 20-2
|
||||||
|
title_cap_1: 20
|
||||||
|
title_cap_1_sanitized: 20
|
||||||
|
title_cap_2: 2
|
||||||
|
upload_date_both_caps: First and Second containing in regex default
|
||||||
|
Mock Entry 20-3.info.json
|
||||||
|
Mock Entry 20-3.jpg
|
||||||
|
Mock Entry 20-3.mp4
|
||||||
|
Video Tags:
|
||||||
|
album: Music Videos
|
||||||
|
artist: subscription_test
|
||||||
|
genre: ytdl-sub
|
||||||
|
premiered: 2020-08-07
|
||||||
|
title: Mock Entry 20-3
|
||||||
|
year: 2020
|
||||||
|
Mock Entry 20-3.nfo
|
||||||
|
NFO tags:
|
||||||
|
musicvideo:
|
||||||
|
album: Music Videos
|
||||||
|
artist: subscription_test
|
||||||
|
desc_cap: Description
|
||||||
|
genre: ytdl-sub
|
||||||
|
override_with_capture_variable: contains 20
|
||||||
|
override_with_capture_variable_sanitized: contains 20
|
||||||
|
premiered: 2020-08-07
|
||||||
|
title: Mock Entry 20-3
|
||||||
|
title_cap_1: 20
|
||||||
|
title_cap_1_sanitized: 20
|
||||||
|
title_cap_2: 3
|
||||||
|
upload_date_both_caps: First and Second containing in regex default
|
||||||
|
Mock Entry 21-1.info.json
|
||||||
|
Mock Entry 21-1.jpg
|
||||||
|
Mock Entry 21-1.mp4
|
||||||
|
Video Tags:
|
||||||
|
album: Music Videos
|
||||||
|
artist: subscription_test
|
||||||
|
genre: ytdl-sub
|
||||||
|
premiered: 2021-08-08
|
||||||
|
title: Mock Entry 21-1
|
||||||
|
year: 2021
|
||||||
|
Mock Entry 21-1.nfo
|
||||||
|
NFO tags:
|
||||||
|
musicvideo:
|
||||||
|
album: Music Videos
|
||||||
|
artist: subscription_test
|
||||||
|
desc_cap: Description
|
||||||
|
genre: ytdl-sub
|
||||||
|
override_with_capture_variable: contains 21
|
||||||
|
override_with_capture_variable_sanitized: contains 21
|
||||||
|
premiered: 2021-08-08
|
||||||
|
title: Mock Entry 21-1
|
||||||
|
title_cap_1: 21
|
||||||
|
title_cap_1_sanitized: 21
|
||||||
|
title_cap_2: 1
|
||||||
|
upload_date_both_caps: First and Second containing in regex default
|
||||||
|
|
@ -0,0 +1,23 @@
|
||||||
|
Files created:
|
||||||
|
----------------------------------------
|
||||||
|
{output_directory}
|
||||||
|
.ytdl-sub-subscription_test-download-archive.json
|
||||||
|
{output_directory}/subscription_test
|
||||||
|
Mock Entry 21-1.info.json
|
||||||
|
Mock Entry 21-1.jpg
|
||||||
|
Mock Entry 21-1.mp4
|
||||||
|
Video Tags:
|
||||||
|
album: Music Videos
|
||||||
|
artist: subscription_test
|
||||||
|
genre: ytdl-sub
|
||||||
|
premiered: 2021-08-08
|
||||||
|
title: Mock Entry 21-1
|
||||||
|
year: 2021
|
||||||
|
Mock Entry 21-1.nfo
|
||||||
|
NFO tags:
|
||||||
|
musicvideo:
|
||||||
|
album: Music Videos
|
||||||
|
artist: subscription_test
|
||||||
|
genre: ytdl-sub
|
||||||
|
premiered: 2021-08-08
|
||||||
|
title: Mock Entry 21-1
|
||||||
|
|
@ -1,55 +0,0 @@
|
||||||
Files created:
|
|
||||||
----------------------------------------
|
|
||||||
{output_directory}
|
|
||||||
.ytdl-sub-regex_capture_playlist_test-download-archive.json
|
|
||||||
{output_directory}/regex_capture_playlist_test
|
|
||||||
Jesse's Minecraft Server [Trailer - Feb.1].info.json
|
|
||||||
Jesse's Minecraft Server [Trailer - Feb.1].jpg
|
|
||||||
Jesse's Minecraft Server [Trailer - Feb.1].mp4
|
|
||||||
Video Tags:
|
|
||||||
album: Music Videos
|
|
||||||
artist: regex_capture_playlist_test
|
|
||||||
genre: ytdl-sub
|
|
||||||
premiered: 2011-02-01
|
|
||||||
title: Jesse's Minecraft Server [Trailer - Feb.1]
|
|
||||||
year: 2011
|
|
||||||
Jesse's Minecraft Server [Trailer - Feb.1].nfo
|
|
||||||
NFO tags:
|
|
||||||
musicvideo:
|
|
||||||
album: Music Videos
|
|
||||||
artist: regex_capture_playlist_test
|
|
||||||
desc_cap: www.jesseminecraft.webs
|
|
||||||
genre: ytdl-sub
|
|
||||||
override_with_capture_variable: contains Trailer
|
|
||||||
override_with_capture_variable_sanitized: contains Trailer
|
|
||||||
premiered: 2011-02-01
|
|
||||||
title: Jesse's Minecraft Server [Trailer - Feb.1]
|
|
||||||
title_cap_1: Trailer
|
|
||||||
title_cap_1_sanitized: Trailer
|
|
||||||
title_cap_2: Feb.1
|
|
||||||
upload_date_both_caps: First and Second containing in regex default
|
|
||||||
Jesse's Minecraft Server [Trailer - Feb.27].info.json
|
|
||||||
Jesse's Minecraft Server [Trailer - Feb.27].jpg
|
|
||||||
Jesse's Minecraft Server [Trailer - Feb.27].mp4
|
|
||||||
Video Tags:
|
|
||||||
album: Music Videos
|
|
||||||
artist: regex_capture_playlist_test
|
|
||||||
genre: ytdl-sub
|
|
||||||
premiered: 2011-02-27
|
|
||||||
title: Jesse's Minecraft Server [Trailer - Feb.27]
|
|
||||||
year: 2011
|
|
||||||
Jesse's Minecraft Server [Trailer - Feb.27].nfo
|
|
||||||
NFO tags:
|
|
||||||
musicvideo:
|
|
||||||
album: Music Videos
|
|
||||||
artist: regex_capture_playlist_test
|
|
||||||
desc_cap: jesseminecraft.webs
|
|
||||||
genre: ytdl-sub
|
|
||||||
override_with_capture_variable: contains Trailer
|
|
||||||
override_with_capture_variable_sanitized: contains Trailer
|
|
||||||
premiered: 2011-02-27
|
|
||||||
title: Jesse's Minecraft Server [Trailer - Feb.27]
|
|
||||||
title_cap_1: Trailer
|
|
||||||
title_cap_1_sanitized: Trailer
|
|
||||||
title_cap_2: Feb.27
|
|
||||||
upload_date_both_caps: 2011 and 02
|
|
||||||
|
|
@ -1,23 +0,0 @@
|
||||||
Files created:
|
|
||||||
----------------------------------------
|
|
||||||
{output_directory}
|
|
||||||
.ytdl-sub-regex_exclude_playlist_test-download-archive.json
|
|
||||||
{output_directory}/regex_exclude_playlist_test
|
|
||||||
Jesse's Minecraft Server [Trailer - Mar.21].info.json
|
|
||||||
Jesse's Minecraft Server [Trailer - Mar.21].jpg
|
|
||||||
Jesse's Minecraft Server [Trailer - Mar.21].mp4
|
|
||||||
Video Tags:
|
|
||||||
album: Music Videos
|
|
||||||
artist: regex_exclude_playlist_test
|
|
||||||
genre: ytdl-sub
|
|
||||||
premiered: 2011-03-21
|
|
||||||
title: Jesse's Minecraft Server [Trailer - Mar.21]
|
|
||||||
year: 2011
|
|
||||||
Jesse's Minecraft Server [Trailer - Mar.21].nfo
|
|
||||||
NFO tags:
|
|
||||||
musicvideo:
|
|
||||||
album: Music Videos
|
|
||||||
artist: regex_exclude_playlist_test
|
|
||||||
genre: ytdl-sub
|
|
||||||
premiered: 2011-03-21
|
|
||||||
title: Jesse's Minecraft Server [Trailer - Mar.21]
|
|
||||||
|
|
@ -1,30 +0,0 @@
|
||||||
Files created:
|
|
||||||
----------------------------------------
|
|
||||||
{output_directory}
|
|
||||||
.ytdl-sub-regex_match_and_exclude_playlist_test-download-archive.json
|
|
||||||
{output_directory}/regex_match_and_exclude_playlist_test
|
|
||||||
Jesse's Minecraft Server [Trailer - Feb.1].info.json
|
|
||||||
Jesse's Minecraft Server [Trailer - Feb.1].jpg
|
|
||||||
Jesse's Minecraft Server [Trailer - Feb.1].mp4
|
|
||||||
Video Tags:
|
|
||||||
album: Music Videos
|
|
||||||
artist: regex_match_and_exclude_playlist_test
|
|
||||||
genre: ytdl-sub
|
|
||||||
premiered: 2011-02-01
|
|
||||||
title: Jesse's Minecraft Server [Trailer - Feb.1]
|
|
||||||
year: 2011
|
|
||||||
Jesse's Minecraft Server [Trailer - Feb.1].nfo
|
|
||||||
NFO tags:
|
|
||||||
musicvideo:
|
|
||||||
album: Music Videos
|
|
||||||
artist: regex_match_and_exclude_playlist_test
|
|
||||||
desc_cap: www.jesseminecraft.webs
|
|
||||||
genre: ytdl-sub
|
|
||||||
override_with_capture_variable: contains Trailer
|
|
||||||
override_with_capture_variable_sanitized: contains Trailer
|
|
||||||
premiered: 2011-02-01
|
|
||||||
title: Jesse's Minecraft Server [Trailer - Feb.1]
|
|
||||||
title_cap_1: Trailer
|
|
||||||
title_cap_1_sanitized: Trailer
|
|
||||||
title_cap_2: Feb.1
|
|
||||||
upload_date_both_caps: First and Second containing in regex default
|
|
||||||
|
|
@ -1,23 +0,0 @@
|
||||||
Files created:
|
|
||||||
----------------------------------------
|
|
||||||
{output_directory}
|
|
||||||
.ytdl-sub-regex_using_overrides_test-download-archive.json
|
|
||||||
{output_directory}/regex_using_overrides_test
|
|
||||||
Jesse's Minecraft Server [Trailer - Mar.21].info.json
|
|
||||||
Jesse's Minecraft Server [Trailer - Mar.21].jpg
|
|
||||||
Jesse's Minecraft Server [Trailer - Mar.21].mp4
|
|
||||||
Video Tags:
|
|
||||||
album: Music Videos
|
|
||||||
artist: regex_using_overrides_test
|
|
||||||
genre: ytdl-sub
|
|
||||||
premiered: 2011-03-21
|
|
||||||
title: Jesse's Minecraft Server [Trailer - Mar.21]
|
|
||||||
year: 2011
|
|
||||||
Jesse's Minecraft Server [Trailer - Mar.21].nfo
|
|
||||||
NFO tags:
|
|
||||||
musicvideo:
|
|
||||||
album: Music Videos
|
|
||||||
artist: regex_using_overrides_test
|
|
||||||
genre: ytdl-sub
|
|
||||||
premiered: 2011-03-21
|
|
||||||
title: Jesse's Minecraft Server [Trailer - Mar.21]
|
|
||||||
145
tests/unit/plugins/test_filter.py
Normal file
145
tests/unit/plugins/test_filter.py
Normal file
|
|
@ -0,0 +1,145 @@
|
||||||
|
from typing import Any
|
||||||
|
from typing import Dict
|
||||||
|
|
||||||
|
import mergedeep
|
||||||
|
import pytest
|
||||||
|
from expected_transaction_log import assert_transaction_log_matches
|
||||||
|
|
||||||
|
from ytdl_sub.config.config_file import ConfigFile
|
||||||
|
from ytdl_sub.subscriptions.subscription import Subscription
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def regex_subscription_dict_base(output_directory):
|
||||||
|
return {
|
||||||
|
"preset": "Jellyfin Music Videos",
|
||||||
|
# override the output directory with our fixture-generated dir
|
||||||
|
"output_options": {"output_directory": output_directory},
|
||||||
|
"format": "best[height<=480]", # download the worst format so it is fast
|
||||||
|
"filter_include": ["{%not(%is_null(description_website))}"],
|
||||||
|
"overrides": {
|
||||||
|
"in_regex_default": "in regex default",
|
||||||
|
"url": "https://youtube.com/playlist?list=PL5BC0FC26BECA5A35",
|
||||||
|
"upload_capture": """{
|
||||||
|
%regex_capture_many_with_defaults(
|
||||||
|
upload_date_standardized,
|
||||||
|
["([0-9]+)-([0-9]+)-27"],
|
||||||
|
["First", %concat("Second containing ", in_regex_default)]
|
||||||
|
)
|
||||||
|
}""",
|
||||||
|
"upload_captured_year": "{%array_at(upload_capture, 1)}",
|
||||||
|
"upload_captured_month": "{%array_at(upload_capture, 2)}",
|
||||||
|
"description_capture": """{
|
||||||
|
%regex_capture_many_with_defaults(
|
||||||
|
description,
|
||||||
|
["The (.*)"],
|
||||||
|
[null]
|
||||||
|
)
|
||||||
|
}""",
|
||||||
|
"description_website": "{%array_at(description_capture, 1)}",
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def regex_subscription_dict(regex_subscription_dict_base, output_directory):
|
||||||
|
return mergedeep.merge(
|
||||||
|
regex_subscription_dict_base,
|
||||||
|
{
|
||||||
|
"nfo_tags": {
|
||||||
|
"tags": {
|
||||||
|
"title_cap_1": "{title_type}",
|
||||||
|
"title_cap_1_sanitized": "{title_type_sanitized}",
|
||||||
|
"title_cap_2": "{title_date}",
|
||||||
|
"desc_cap": "{description_website}",
|
||||||
|
"upload_date_both_caps": "{upload_captured_year} and {upload_captured_month}",
|
||||||
|
"override_with_capture_variable": "{contains_regex_default}",
|
||||||
|
"override_with_capture_variable_sanitized": "{contains_regex_sanitized_default}",
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"filter_exclude": ["{%is_null(title_type)}"],
|
||||||
|
"overrides": {
|
||||||
|
"title_capture_list": """{
|
||||||
|
%regex_capture_many_with_defaults(
|
||||||
|
title,
|
||||||
|
[ "should not cap (.+) - (.+)", "Mock Entry (.*)-(.*)" ],
|
||||||
|
[ null, null ]
|
||||||
|
)
|
||||||
|
}""",
|
||||||
|
"title_type": "{%array_at(title_capture_list, 1)}",
|
||||||
|
"title_date": "{%array_at(title_capture_list, 2)}",
|
||||||
|
"contains_regex_default": "contains {title_type}",
|
||||||
|
"contains_regex_sanitized_default": "contains {title_type_sanitized}",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def regex_subscription_dict_exclude(regex_subscription_dict_base, output_directory):
|
||||||
|
return mergedeep.merge(
|
||||||
|
regex_subscription_dict_base,
|
||||||
|
{
|
||||||
|
"filter_exclude": [
|
||||||
|
"""{
|
||||||
|
%regex_search_any(
|
||||||
|
title,
|
||||||
|
[ "should not cap", "Mock Entry 20-.*" ]
|
||||||
|
)
|
||||||
|
}"""
|
||||||
|
]
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def playlist_subscription(
|
||||||
|
config: ConfigFile, subscription_name: str, regex_subscription_dict: Dict[str, Any]
|
||||||
|
):
|
||||||
|
return Subscription.from_dict(
|
||||||
|
config=config,
|
||||||
|
preset_name=subscription_name,
|
||||||
|
preset_dict=regex_subscription_dict,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def playlist_subscription_exclude(
|
||||||
|
config: ConfigFile, subscription_name: str, regex_subscription_dict_exclude: Dict[str, Any]
|
||||||
|
) -> Subscription:
|
||||||
|
return Subscription.from_dict(
|
||||||
|
config=config,
|
||||||
|
preset_name=subscription_name,
|
||||||
|
preset_dict=regex_subscription_dict_exclude,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class TestFilter:
|
||||||
|
def test_filter_regex_success(
|
||||||
|
self, mock_download_collection_entries, playlist_subscription, output_directory
|
||||||
|
):
|
||||||
|
with mock_download_collection_entries(
|
||||||
|
is_youtube_channel=False, num_urls=1, is_extracted_audio=False, is_dry_run=True
|
||||||
|
):
|
||||||
|
transaction_log = playlist_subscription.download(dry_run=True)
|
||||||
|
|
||||||
|
assert_transaction_log_matches(
|
||||||
|
output_directory=output_directory,
|
||||||
|
transaction_log=transaction_log,
|
||||||
|
transaction_log_summary_file_name="plugins/filter/test_regex.txt",
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_filter_regex_excludes_success(
|
||||||
|
self, mock_download_collection_entries, playlist_subscription_exclude, output_directory
|
||||||
|
):
|
||||||
|
# Should only contain the march video
|
||||||
|
with mock_download_collection_entries(
|
||||||
|
is_youtube_channel=False, num_urls=1, is_extracted_audio=False, is_dry_run=True
|
||||||
|
):
|
||||||
|
transaction_log = playlist_subscription_exclude.download(dry_run=True)
|
||||||
|
|
||||||
|
assert_transaction_log_matches(
|
||||||
|
output_directory=output_directory,
|
||||||
|
transaction_log=transaction_log,
|
||||||
|
transaction_log_summary_file_name="plugins/filter/test_regex_exclude.txt",
|
||||||
|
)
|
||||||
|
|
@ -1,10 +1,10 @@
|
||||||
import pytest
|
import pytest
|
||||||
from unit.script.conftest import single_variable_output
|
from unit.script.conftest import single_variable_output
|
||||||
|
|
||||||
from ytdl_sub.script.script import Script
|
from ytdl_sub.script.utils.exceptions import FunctionRuntimeException
|
||||||
|
|
||||||
|
|
||||||
class TestNumericFunctions:
|
class TestRegexFunctions:
|
||||||
@pytest.mark.parametrize(
|
@pytest.mark.parametrize(
|
||||||
"values, expected_output",
|
"values, expected_output",
|
||||||
[
|
[
|
||||||
|
|
@ -59,3 +59,37 @@ class TestNumericFunctions:
|
||||||
def test_regex_sub(self, values: str, expected_output: str):
|
def test_regex_sub(self, values: str, expected_output: str):
|
||||||
output = single_variable_output(f"{{%regex_sub({values})}}")
|
output = single_variable_output(f"{{%regex_sub({values})}}")
|
||||||
assert output == expected_output
|
assert output == expected_output
|
||||||
|
|
||||||
|
def test_regex_capture_many_fails_no_match(self):
|
||||||
|
with pytest.raises(
|
||||||
|
FunctionRuntimeException,
|
||||||
|
match="no regex strings were captured for input string the string",
|
||||||
|
):
|
||||||
|
single_variable_output(
|
||||||
|
f"""{{
|
||||||
|
%regex_capture_many(
|
||||||
|
"the string",
|
||||||
|
[
|
||||||
|
"no (.*) match",
|
||||||
|
"not here either (.*)"
|
||||||
|
]
|
||||||
|
)
|
||||||
|
}}"""
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_regex_capture_many_fails_unequal_capture_groups(self):
|
||||||
|
with pytest.raises(
|
||||||
|
FunctionRuntimeException,
|
||||||
|
match="regex_array elements must contain the same number of capture groups",
|
||||||
|
):
|
||||||
|
single_variable_output(
|
||||||
|
f"""{{
|
||||||
|
%regex_capture_many(
|
||||||
|
"the string",
|
||||||
|
[
|
||||||
|
"no (.*) match",
|
||||||
|
"(.*) not equal to 1 (.*)"
|
||||||
|
]
|
||||||
|
)
|
||||||
|
}}"""
|
||||||
|
)
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue