danvics/ytdl-sub
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

refactor plugin name, matching now working

Browse source
This commit is contained in:
jbannon 2022-07-14 05:38:15 +00:00
parent bdf69722f0
commit 3ca6af38dc
8 changed files with 88 additions and 60 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
src/ytdl_sub/config/preset_class_mappings.py
View file

@ -13,7 +13,7 @@ from ytdl_sub.plugins.music_tags import MusicTagsPlugin
from ytdl_sub.plugins.nfo_tags import NfoTagsPlugin
from ytdl_sub.plugins.output_directory_nfo_tags import OutputDirectoryNfoTagsPlugin
from ytdl_sub.plugins.plugin import Plugin
from ytdl_sub.plugins.regex_capture import RegexCapturePlugin
from ytdl_sub.plugins.regex import RegexPlugin
class DownloadStrategyMapping:
@ -108,7 +108,7 @@ class PluginMapping:
"music_tags": MusicTagsPlugin,
"nfo_tags": NfoTagsPlugin,
"output_directory_nfo_tags": OutputDirectoryNfoTagsPlugin,
"regex": RegexCapturePlugin,
"regex": RegexPlugin,
}
@classmethod

32
src/ytdl_sub/plugins/regex_capture.py → src/ytdl_sub/plugins/regex.py
View file

@ -5,7 +5,7 @@ from typing import Optional
from ytdl_sub.entries.entry import Entry
from ytdl_sub.plugins.plugin import Plugin
from ytdl_sub.plugins.plugin import PluginOptions
from ytdl_sub.utils.exceptions import ValidationException
from ytdl_sub.utils.exceptions import RegexNoMatchException
from ytdl_sub.utils.logger import Logger
from ytdl_sub.validators.regex_validator import RegexListValidator
from ytdl_sub.validators.strict_dict_validator import StrictDictValidator
@ -20,7 +20,7 @@ def _source_var_name(source_variable: str, capture_group_idx: int) -> str:
return f"{source_variable}_capture_{capture_group_idx+1}"
class SourceVariableRegexCapture(StrictDictValidator):
class SourceVariableRegex(StrictDictValidator):
_required_keys = {"match"}
_optional_keys = {"defaults"}
@ -70,27 +70,26 @@ class SourceVariableRegexCapture(StrictDictValidator):
return self._defaults.list if self.has_defaults else None
class FromSourceVariablesRegexCapture(StrictDictValidator):
class FromSourceVariablesRegex(StrictDictValidator):
_optional_keys = Entry.source_variables()
_allow_extra_keys = True
def __init__(self, name, value):
super().__init__(name, value)
self.source_variable_capture_dict: Dict[str, SourceVariableRegexCapture] = {
key: self._validate_key(key=key, validator=SourceVariableRegexCapture)
for key in self._keys
self.source_variable_capture_dict: Dict[str, SourceVariableRegex] = {
key: self._validate_key(key=key, validator=SourceVariableRegex) for key in self._keys
}
class RegexCaptureOptions(PluginOptions):
class RegexOptions(PluginOptions):
_required_keys = {"from"}
_optional_keys = {"skip_if_match_fails"}
def __init__(self, name, value):
super().__init__(name, value)
self._from = self._validate_key(key="from", validator=FromSourceVariablesRegexCapture)
self._from = self._validate_key(key="from", validator=FromSourceVariablesRegex)
self.skip_if_match_fails: bool = self._validate_key_if_present(
key="skip_if_match_fails", validator=BoolValidator, default=False
).value
@ -111,7 +110,7 @@ class RegexCaptureOptions(PluginOptions):
)
@property
def source_variable_capture_dict(self) -> Dict[str, SourceVariableRegexCapture]:
def source_variable_capture_dict(self) -> Dict[str, SourceVariableRegex]:
"""
Returns
-------
@ -135,8 +134,8 @@ class RegexCaptureOptions(PluginOptions):
return added_source_vars
class RegexCapturePlugin(Plugin[RegexCaptureOptions]):
plugin_options_type = RegexCaptureOptions
class RegexPlugin(Plugin[RegexOptions]):
plugin_options_type = RegexOptions
def modify_entry(self, entry: Entry) -> Optional[Entry]:
"""
@ -158,7 +157,7 @@ class RegexCapturePlugin(Plugin[RegexCaptureOptions]):
# Iterate each source var to capture and add to the entry
for source_var, regex_options in self.plugin_options.source_variable_capture_dict.items():
maybe_capture = regex_options.capture_list.capture_any(
maybe_capture = regex_options.capture_list.match_any(
input_str=entry_variable_dict[source_var]
)
@ -169,14 +168,15 @@ class RegexCapturePlugin(Plugin[RegexCaptureOptions]):
# Skip the entry if toggled
if self.plugin_options.skip_if_match_fails:
logger.info(
"Entry with title '%s' failed to match regex, skipping.", entry.title
"Regex failed to match '%s' from '%s', skipping.",
source_var,
entry.title,
)
return None
# Otherwise, error
raise ValidationException(
f"Failed to capture {source_var} from an entry with the value:\n"
f"{entry_variable_dict[source_var]}"
raise RegexNoMatchException(
f"Regex failed to match '{source_var}' from '{entry.title}'"
)
# otherwise, use defaults (apply them using the original entry source dict)

8
src/ytdl_sub/subscriptions/subscription.py
View file

@ -273,7 +273,13 @@ class Subscription:
# First, modify the entry with all plugins
for plugin in plugins:
entry = plugin.modify_entry(entry)
# Break out of this plugin loop if entry is None, it is indicated to not DL it
if (entry := plugin.modify_entry(entry)) is None:
break
# If entry is None from the broken out loop, continue over the other entries
if entry is None:
continue
# Then, post-process the entry with all plugins
for plugin in plugins:

4
src/ytdl_sub/utils/exceptions.py
View file

@ -20,3 +20,7 @@ class FileNotFoundException(ValidationException):
class InvalidYamlException(ValidationException):
"""User yaml that is invalid"""
class RegexNoMatchException(ValidationException):
"""Regex failed to match during download"""

9
src/ytdl_sub/validators/regex_validator.py
View file

@ -28,7 +28,7 @@ class RegexValidator(StringValidator):
"""
return self._compiled_regex.groups
def capture(self, input_str: str) -> Optional[List[str]]:
def match(self, input_str: str) -> Optional[List[str]]:
"""
Parameters
----------
@ -68,7 +68,7 @@ class RegexListValidator(ListValidator[RegexValidator]):
"""
return self._num_capture_groups
def capture_any(self, input_str: str) -> Optional[List[str]]:
def match_any(self, input_str: str) -> Optional[List[str]]:
"""
Parameters
----------
@ -77,9 +77,10 @@ class RegexListValidator(ListValidator[RegexValidator]):
Returns
-------
List of captures on the first regex that matches. None if no regexes match.
List of captures on the first regex that matches. If the regex has no capture groups, then
the list will be emtpy. None is returned if the input_str failed to match
"""
for reg in self._list:
if (maybe_capture := reg.capture(input_str)) is not None:
if (maybe_capture := reg.match(input_str)) is not None:
return maybe_capture
return None

52
tests/e2e/plugins/test_regex_capture.py → tests/e2e/plugins/test_regex.py
View file

@ -1,12 +1,15 @@
import re
import pytest
from e2e.expected_transaction_log import assert_transaction_log_matches
from ytdl_sub.config.preset import Preset
from ytdl_sub.subscriptions.subscription import Subscription
from ytdl_sub.utils.exceptions import RegexNoMatchException
@pytest.fixture
def regex_capture_subscription_dict(output_directory):
def regex_subscription_dict(output_directory):
return {
"preset": "yt_music_video_playlist",
"youtube": {"playlist_url": "https://youtube.com/playlist?list=PL5BC0FC26BECA5A35"},
@ -15,15 +18,14 @@ def regex_capture_subscription_dict(output_directory):
# download the worst format so it is fast
"ytdl_options": {
"format": "best[height<=480]",
"postprocessor_args": {"ffmpeg": ["-bitexact"]}, # Must add this for reproducibility
},
"regex": {
"skip_if_match_fails": False,
"skip_if_match_fails": True,
"from": {
"title": {
"match": [
"should not cap (.+) - (.+)",
".*\\[(.+) - (.+)]",
".*\\[(.+) - (Feb.+)]", # should filter out march video
],
},
"description": {"match": [".*http:\\/\\/(.+).com.*"]},
@ -50,11 +52,18 @@ def regex_capture_subscription_dict(output_directory):
@pytest.fixture
def playlist_subscription(music_video_config, regex_capture_subscription_dict):
def regex_subscription_dict_no_match_fails(regex_subscription_dict):
# tests that skip_if_match_fails defaults to False
del regex_subscription_dict["regex"]["skip_if_match_fails"]
return regex_subscription_dict
@pytest.fixture
def playlist_subscription(music_video_config, regex_subscription_dict):
playlist_preset = Preset.from_dict(
config=music_video_config,
preset_name="regex_capture_playlist_test",
preset_dict=regex_capture_subscription_dict,
preset_dict=regex_subscription_dict,
)
return Subscription.from_preset(
@ -63,12 +72,37 @@ def playlist_subscription(music_video_config, regex_capture_subscription_dict):
)
class TestRegexCapture:
def test_regex_capture_success(self, playlist_subscription, output_directory):
@pytest.fixture
def playlist_subscription_no_match_fails(
music_video_config, regex_subscription_dict_no_match_fails
):
playlist_preset = Preset.from_dict(
config=music_video_config,
preset_name="regex_capture_playlist_test",
preset_dict=regex_subscription_dict_no_match_fails,
)
return Subscription.from_preset(
preset=playlist_preset,
config=music_video_config,
)
class TestRegex:
def test_regex_success(self, playlist_subscription, output_directory):
# Only dry run is needed to see if capture variables are created
transaction_log = playlist_subscription.download(dry_run=True)
assert_transaction_log_matches(
output_directory=output_directory,
transaction_log=transaction_log,
transaction_log_summary_file_name="plugins/test_regex_mapping.txt",
transaction_log_summary_file_name="plugins/test_regex.txt",
)
def test_regex_fails_no_match(self, playlist_subscription_no_match_fails, output_directory):
with pytest.raises(
RegexNoMatchException,
match=re.escape(
"Regex failed to match 'title' from 'Jesse's Minecraft Server [Trailer - Mar.21]'"
),
):
_ = playlist_subscription_no_match_fails.download(dry_run=True)

14
tests/e2e/resources/transaction_log_summaries/plugins/test_regex_mapping.txt → tests/e2e/resources/transaction_log_summaries/plugins/test_regex.txt
View file

@ -28,18 +28,4 @@ Project Zombie - Jesse's Minecraft Server [Trailer - Feb.27].nfo
title_cap_1: Trailer
title_cap_2: Feb.27
upload_date_both_caps: 2011 and 02
year: 2011
Project Zombie - Jesse's Minecraft Server [Trailer - Mar.21]-thumb.jpg
Project Zombie - Jesse's Minecraft Server [Trailer - Mar.21].mp4
Project Zombie - Jesse's Minecraft Server [Trailer - Mar.21].nfo
NFO tags:
musicvideo:
album: Music Videos
artist: Project Zombie
artist_cap_always_default: Always default
desc_cap: jesseminecraft.webs
title: Jesse's Minecraft Server [Trailer - Mar.21]
title_cap_1: Trailer
title_cap_2: Mar.21
upload_date_both_caps: First and Second
year: 2011

25
tests/unit/validators/test_regex_validator.py
View file

@ -6,21 +6,20 @@ from ytdl_sub.validators.regex_validator import RegexValidator
@pytest.mark.parametrize(
"regex_value, input_str, matches, captures",
"regex_value, input_str, expected_output",
[
("^match this$", "match this", True, None),
("^my (.+) cap", "my first cap", True, ["first"]),
(".* (.+) - (.+) two", "my other - capped two", True, ["other", "capped"]),
("failed match", "nope", False, None),
("^match this$", "match this", []),
("^my (.+) cap", "my first cap", ["first"]),
(".* (.+) - (.+) two", "my other - capped two", ["other", "capped"]),
("failed match", "nope", None),
],
)
def test_regex_validator(regex_value, input_str, matches, captures):
def test_regex_validator(regex_value, input_str, expected_output):
regex_validator = RegexValidator(name="good_regex_validator", value=regex_value)
assert regex_validator._name == "good_regex_validator"
assert regex_validator._value == regex_value
assert regex_validator.is_match(input_str=input_str) is matches
assert regex_validator.capture(input_str=input_str) == captures
assert regex_validator.match(input_str=input_str) == expected_output
@pytest.mark.parametrize("regex_value", ["(", "??"])
@ -29,17 +28,16 @@ def test_regex_validator_fails_bad_value(regex_value):
_ = RegexValidator(name="fail", value=regex_value)
def test_regex_list_validator():
def test_regex_list_validator_matches():
regex_list_validator_raw_value = ["try matching this", "try matching that", "how about this"]
regex_list = RegexListValidator(name="list val", value=regex_list_validator_raw_value)
for raw_val in regex_list_validator_raw_value:
assert regex_list.matches_any(raw_val)
assert regex_list.capture_any(raw_val) is None
assert regex_list.match_any(raw_val) == []
def test_regex_list_validator_capture():
def test_regex_list_validator_captures():
regex_list_validator_raw_value = ["try (.+) this", "try (.+) that", "how (.+) this"]
regex_list = RegexListValidator(name="list val", value=regex_list_validator_raw_value)
@ -50,8 +48,7 @@ def test_regex_list_validator_capture():
]
for capture_str, expected_capture in captures_test:
assert regex_list.matches_any(capture_str)
assert regex_list.capture_any(capture_str) == [expected_capture]
assert regex_list.match_any(capture_str) == [expected_capture]
def test_regex_list_validator_invalid_regex():