danvics/ytdl-sub
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

updated with name, default, better docs

Browse source
This commit is contained in:
jbannon 2022-07-17 06:14:20 +00:00
parent 9646a3bd65
commit 142d87d14e
4 changed files with 66 additions and 48 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

11
docs/config.rst
View file

@ -192,6 +192,17 @@ nfo_output_directory
-------------------------------------------------------------------------------
regex
'''''
.. autoclass:: ytdl_sub.plugins.regex.RegexOptions()
:members: skip_if_match_fails
.. autoclass:: ytdl_sub.plugins.regex.SourceVariableRegex()
:members: match, capture_group_names, capture_group_defaults
:member-order: bysource
-------------------------------------------------------------------------------
.. _subscription_yaml:
subscription.yaml

84
src/ytdl_sub/plugins/regex.py
View file

@ -20,13 +20,13 @@ logger = Logger.get(name="regex")
class SourceVariableRegex(StrictDictValidator):
_required_keys = {"match"}
_optional_keys = {"defaults", "capture_group_names"}
_optional_keys = {"capture_group_defaults", "capture_group_names"}
def __init__(self, name, value):
super().__init__(name, value)
self._match = self._validate_key(key="match", validator=RegexListValidator)
self._defaults = self._validate_key_if_present(
key="defaults", validator=ListFormatterValidator
self._capture_group_defaults = self._validate_key_if_present(
key="capture_group_defaults", validator=ListFormatterValidator
)
self._capture_group_names = self._validate_key_if_present(
key="capture_group_names", validator=SourceVariableNameListValidator, default=[]
@ -34,12 +34,12 @@ class SourceVariableRegex(StrictDictValidator):
# If defaults are to be used, ensure there are the same number of defaults as there are
# capture groups
if self._defaults is not None and self._match.num_capture_groups != len(
self._defaults.list
if self._capture_group_defaults is not None and self._match.num_capture_groups != len(
self._capture_group_defaults.list
):
raise self._validation_exception(
f"number of defaults must match number of capture groups, "
f"{len(self._defaults.list)} != {self._match.num_capture_groups}"
f"{len(self._capture_group_defaults.list)} != {self._match.num_capture_groups}"
)
# If there are capture groups, ensure there are capture group names
@ -50,14 +50,31 @@ class SourceVariableRegex(StrictDictValidator):
)
@property
def capture_list(self) -> RegexListValidator:
def match(self) -> RegexListValidator:
"""
Returns
-------
List of regex captures
Required. List of regex strings to try to match against a source variable. Each regex
string must have the same number of capture groups.
"""
return self._match
@property
def capture_group_names(self) -> Optional[List[str]]:
"""
Optional (only when no capture groups are in the regex string). List of names to store the
capture group values to. These and ``_sanitized`` versions will be available to use as
source variables. The list's length must be equal to the number of match capture groups.
"""
return [validator.value for validator in self._capture_group_names.list]
@property
def capture_group_defaults(self) -> Optional[List[StringFormatterValidator]]:
"""
Optional. List of string format validators to use for capture group defaults if a
source variable cannot be matched. The list's length must be equal to the number of match
capture groups.
"""
return self._capture_group_defaults.list if self.has_defaults else None
@property
def has_defaults(self) -> bool:
"""
@ -65,25 +82,7 @@ class SourceVariableRegex(StrictDictValidator):
-------
True if a validation exception should be raised if not captured. False otherwise.
"""
return self._defaults is not None
@property
def defaults(self) -> Optional[List[StringFormatterValidator]]:
"""
Returns
-------
List of string format validators to use for the defaults
"""
return self._defaults.list if self.has_defaults else None
@property
def capture_group_names(self) -> List[str]:
"""
Returns
-------
List of new capture group names
"""
return [validator.value for validator in self._capture_group_names.list]
return self._capture_group_defaults is not None
class FromSourceVariablesRegex(StrictDictValidator):
@ -115,16 +114,19 @@ class RegexOptions(PluginOptions):
skip_if_match_fails: True
from:
# For each entry's `title` value...
title:
# Match with capture groups act as a filter.
# Perform this regex match on it to act as a filter.
# This will only download videos with "Official Video" in it.
match: "\[Official Video\]"
match: '\[Official Video\]'
# For each entry's `description` value...
description:
# Match with capture groups and defaults.
# This tries to scrape a date from the description and produce new source variables
match: "([0-9]{4})-([0-9]{2})-([0-9]{2})"
# Each capture group creates these new source variables, respectively
# Each capture group creates these new source variables, respectively, as well
# a sanitized version, i.e. `sanitized_captured_upload_year`
capture_group_names:
- "captured_upload_year"
- "captured_upload_month"
@ -144,10 +146,18 @@ class RegexOptions(PluginOptions):
def __init__(self, name, value):
super().__init__(name, value)
self._from = self._validate_key(key="from", validator=FromSourceVariablesRegex)
self.skip_if_match_fails: bool = self._validate_key_if_present(
key="skip_if_match_fails", validator=BoolValidator, default=False
self._skip_if_match_fails: bool = self._validate_key_if_present(
key="skip_if_match_fails", validator=BoolValidator, default=True
).value
@property
def skip_if_match_fails(self) -> Optional[bool]:
"""
Defaults to True. If True, when any match fails and has no defaults, the entry will be
skipped. If False, ytdl-sub will error and all downloads will not proceed.
"""
return self._skip_if_match_fails
def validate_with_variables(
self, source_variables: List[str], override_variables: List[str]
) -> None:
@ -226,9 +236,7 @@ class RegexPlugin(Plugin[RegexOptions]):
# Iterate each source var to capture and add to the entry
for source_var, regex_options in self.plugin_options.source_variable_capture_dict.items():
maybe_capture = regex_options.capture_list.match_any(
input_str=entry_variable_dict[source_var]
)
maybe_capture = regex_options.match.match_any(input_str=entry_variable_dict[source_var])
# If no capture
if maybe_capture is None:
@ -257,7 +265,7 @@ class RegexPlugin(Plugin[RegexOptions]):
regex_options.capture_group_names[i]: default.apply_formatter(
variable_dict=source_variables_and_overrides_dict
)
for i, default in enumerate(regex_options.defaults)
for i, default in enumerate(regex_options.capture_group_defaults)
},
)
# There is a capture, add the source variables to the entry as

11
tests/e2e/plugins/test_regex.py
View file

@ -22,7 +22,7 @@ def regex_subscription_dict(output_directory):
"format": "best[height<=480]",
},
"regex": {
"skip_if_match_fails": True,
# tests that skip_if_match_fails defaults to True
"from": {
"title": {
"match": [
@ -41,7 +41,7 @@ def regex_subscription_dict(output_directory):
"upload_captured_year",
"upload_captured_month",
],
"defaults": [
"capture_group_defaults": [
"First",
"Second containing {in_regex_default}",
],
@ -49,7 +49,7 @@ def regex_subscription_dict(output_directory):
"artist": {
"match": ["Never (.*) capture"],
"capture_group_names": ["always_default"],
"defaults": ["Always default"],
"capture_group_defaults": ["Always default"],
},
},
},
@ -72,8 +72,7 @@ def regex_subscription_dict(output_directory):
@pytest.fixture
def regex_subscription_dict_no_match_fails(regex_subscription_dict):
# tests that skip_if_match_fails defaults to False
del regex_subscription_dict["regex"]["skip_if_match_fails"]
regex_subscription_dict["regex"]["skip_if_match_fails"] = False
return regex_subscription_dict
@ -177,7 +176,7 @@ class TestRegex:
)
def test_regex_fails_unequal_defaults(self, regex_subscription_dict, music_video_config):
regex_subscription_dict["regex"]["from"]["title"]["defaults"] = ["1 != 2"]
regex_subscription_dict["regex"]["from"]["title"]["capture_group_defaults"] = ["1 != 2"]
with pytest.raises(
ValidationException,
match=re.escape("number of defaults must match number of capture groups, 1 != 2"),

8
tests/e2e/youtube/test_playlist_as_kodi_music_videos.py
View file

@ -52,7 +52,7 @@ def expected_playlist_download():
return ExpectedDownloads(
expected_downloads=[
# Download mapping
ExpectedDownloadFile(path=Path(".ytdl-sub-jmc-download-archive.json"), md5="9f785c29194a6ecfba6a6b4018763ddc"),
ExpectedDownloadFile(path=Path(".ytdl-sub-music_video_playlist_test-download-archive.json"), md5="9f785c29194a6ecfba6a6b4018763ddc"),
# Entry files
ExpectedDownloadFile(path=Path("JMC - Jesse's Minecraft Server [Trailer - Feb.1]-thumb.jpg"), md5="b232d253df621aa770b780c1301d364d"),
@ -87,16 +87,16 @@ def single_video_subscription_dict(subscription_dict):
@pytest.fixture
def single_video_subscription(config, single_video_subscription_dict):
def single_video_subscription(music_video_config, single_video_subscription_dict):
single_video_preset = Preset.from_dict(
config=config,
config=music_video_config,
preset_name="music_video_single_video_test",
preset_dict=single_video_subscription_dict,
)
return Subscription.from_preset(
preset=single_video_preset,
config=config,
config=music_video_config,
)