danvics/ytdl-sub
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

e2e tested, need more failure tests, no dupe variable names, add docs

Browse source
This commit is contained in:
jbannon 2022-07-17 01:00:59 +00:00
parent 87837bbc29
commit ce0a4406b9
6 changed files with 137 additions and 51 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

86
src/ytdl_sub/plugins/regex.py
View file

@ -8,6 +8,7 @@ from ytdl_sub.plugins.plugin import PluginOptions
from ytdl_sub.utils.exceptions import RegexNoMatchException
from ytdl_sub.utils.logger import Logger
from ytdl_sub.validators.regex_validator import RegexListValidator
from ytdl_sub.validators.source_variable_validator import SourceVariableNameListValidator
from ytdl_sub.validators.strict_dict_validator import StrictDictValidator
from ytdl_sub.validators.string_formatter_validators import ListFormatterValidator
from ytdl_sub.validators.string_formatter_validators import StringFormatterValidator
@ -16,14 +17,10 @@ from ytdl_sub.validators.validators import BoolValidator
logger = Logger.get(name="regex")
def _source_var_name(source_variable: str, capture_group_idx: int) -> str:
return f"{source_variable}_capture_{capture_group_idx+1}"
class SourceVariableRegex(StrictDictValidator):
_required_keys = {"match"}
_optional_keys = {"defaults"}
_optional_keys = {"defaults", "capture_group_names"}
def __init__(self, name, value):
super().__init__(name, value)
@ -31,6 +28,9 @@ class SourceVariableRegex(StrictDictValidator):
self._defaults = self._validate_key_if_present(
key="defaults", validator=ListFormatterValidator
)
self._capture_group_names = self._validate_key_if_present(
key="capture_group_names", validator=SourceVariableNameListValidator, default=[]
)
# If defaults are to be used, ensure there are the same number of defaults as there are
# capture groups
@ -42,6 +42,13 @@ class SourceVariableRegex(StrictDictValidator):
f"{len(self._defaults.list)} != {self._match.num_capture_groups}"
)
# If there are capture groups, ensure there are capture group names
if len(self._capture_group_names.list) != self._match.num_capture_groups:
raise self._validation_exception(
f"Number of capture group names must match number of capture groups, "
f"{len(self._capture_group_names.list)} != {self._match.num_capture_groups}"
)
@property
def capture_list(self) -> RegexListValidator:
"""
@ -69,6 +76,15 @@ class SourceVariableRegex(StrictDictValidator):
"""
return self._defaults.list if self.has_defaults else None
@property
def capture_group_names(self) -> List[str]:
"""
Returns
-------
List of new capture group names
"""
return [validator.value for validator in self._capture_group_names.list]
class FromSourceVariablesRegex(StrictDictValidator):
@ -83,9 +99,9 @@ class FromSourceVariablesRegex(StrictDictValidator):
class RegexOptions(PluginOptions):
"""
Performs regex matching on an entry's source variables. Regex can be used to either capture
groups to create new source variables or filter entries from proceeding with download.
r"""
Performs regex matching on an entry's source variables. Regex can be used to filter entries
from proceeding with download or capture groups to create new source variables.
Usage:
@ -94,25 +110,32 @@ class RegexOptions(PluginOptions):
presets:
my_example_preset:
regex:
skip_if_match_fails: False
from:
title:
# Match with no defaults act as a filter.
# This will only download videos with "Official Video" in it.
match: "\[Official Video\]"
description:
# Match with capture groups and defaults.
# This tries to scape a date from the description and produce new source variables
# {description_capture_1}, {description_capture_2}, {description_capture_3}
match: "([0-9]{4})-([0-9]{2})-([0-9]{2})"
capture_group_defaults:
- "{upload_year}"
- "{upload_month}"
- "{upload_day}"
# By default, if any match fails and has no defaults, the entry will be skipped.
# If set to False, ytdl-sub will error and stop all downloads from proceeding.
skip_if_match_fails: True
TODO: add test with override variables in the capture group defaults, and override
variable
referencing a capture variable
from:
title:
# Match with capture groups act as a filter.
# This will only download videos with "Official Video" in it.
match: "\[Official Video\]"
description:
# Match with capture groups and defaults.
# This tries to scrape a date from the description and produce new source variables
match: "([0-9]{4})-([0-9]{2})-([0-9]{2})"
# Each capture group creates these new source variables, respectively
capture_group_names:
- "captured_upload_year"
- "captured_upload_month"
- "captured_upload_day"
# And if the string does not match, use these as respective default values for the
# new source variables.
capture_group_defaults:
- "{upload_year}"
- "{upload_month}"
- "{upload_day}"
"""
_required_keys = {"from"}
@ -156,11 +179,8 @@ class RegexOptions(PluginOptions):
List of new source variables created via regex capture
"""
added_source_vars: List[str] = []
for source_var, regex_options in self.source_variable_capture_dict.items():
added_source_vars.extend(
_source_var_name(source_var, idx)
for idx in range(regex_options.capture_list.num_capture_groups)
)
for regex_options in self.source_variable_capture_dict.values():
added_source_vars.extend(regex_options.capture_group_names)
return added_source_vars
@ -216,7 +236,7 @@ class RegexPlugin(Plugin[RegexOptions]):
)
entry.add_variables(
variables_to_add={
_source_var_name(source_var, i): default.apply_formatter(
regex_options.capture_group_names[i]: default.apply_formatter(
variable_dict=source_variables_and_overrides_dict
)
for i, default in enumerate(regex_options.defaults)
@ -227,7 +247,7 @@ class RegexPlugin(Plugin[RegexOptions]):
else:
entry.add_variables(
variables_to_add={
_source_var_name(source_var, i): capture
regex_options.capture_group_names[i]: capture
for i, capture in enumerate(maybe_capture)
},
)

4
src/ytdl_sub/utils/exceptions.py
View file

@ -10,6 +10,10 @@ class StringFormattingVariableNotFoundException(StringFormattingException):
"""Tried to format a string but the variable was not found"""
class InvalidVariableNameException(ValidationException):
"""A user defined variable name is invalid"""
class DownloadArchiveException(ValueError):
"""Any user or file errors caused by download archive or mapping files"""

20
src/ytdl_sub/validators/source_variable_validator.py Normal file
View file

@ -0,0 +1,20 @@
from ytdl_sub.utils.exceptions import InvalidVariableNameException
from ytdl_sub.validators.string_formatter_validators import is_valid_source_variable_name
from ytdl_sub.validators.validators import ListValidator
from ytdl_sub.validators.validators import StringValidator
class SourceVariableNameValidator(StringValidator):
_expected_value_type_name = "source variable name"
def __init__(self, name, value):
super().__init__(name, value)
try:
_ = is_valid_source_variable_name(self.value, raise_exception=True)
except InvalidVariableNameException as exc:
raise self._validation_exception(exc) from exc
class SourceVariableNameListValidator(ListValidator[SourceVariableNameValidator]):
_inner_list_type = SourceVariableNameValidator
_expected_value_type_name = "source variable name list"

42
src/ytdl_sub/validators/string_formatter_validators.py
View file

@ -5,12 +5,45 @@ from typing import Dict
from typing import List
from typing import final
from ytdl_sub.utils.exceptions import InvalidVariableNameException
from ytdl_sub.utils.exceptions import StringFormattingException
from ytdl_sub.utils.exceptions import StringFormattingVariableNotFoundException
from ytdl_sub.validators.validators import ListValidator
from ytdl_sub.validators.validators import LiteralDictValidator
from ytdl_sub.validators.validators import Validator
_fields_validator = re.compile(r"{([a-z][a-z0-9_]+?)}")
_fields_validator_exception_message: str = (
"{variable_names} must start with a lowercase letter, should only contain lowercase letters, "
"numbers, underscores, and have a single open and close bracket."
)
def is_valid_source_variable_name(input_str: str, raise_exception: bool = False) -> bool:
"""
Parameters
----------
input_str
String to see if it can be a source variable
raise_exception
Raise InvalidVariableNameException False.
Returns
-------
True if it is. False otherwise.
Raises
------
InvalidVariableNameException
If raise_exception and output is False
"""
# Add brackets around it to pretend its a StringFormatter, see if it captures
is_source_variable_name = len(re.findall(_fields_validator, f"{{{input_str}}}")) > 0
if not is_source_variable_name and raise_exception:
raise InvalidVariableNameException(_fields_validator_exception_message)
return is_source_variable_name
class StringFormatterValidator(Validator):
"""
@ -44,8 +77,6 @@ class StringFormatterValidator(Validator):
"Format variable '{variable_name}' does not exist. Available variables: {available_fields}"
)
__fields_validator = re.compile(r"{([a-z][a-z0-9_]+?)}")
__max_format_recursion = 3
def __validate_and_get_format_variables(self) -> List[str]:
@ -70,14 +101,11 @@ class StringFormatterValidator(Validator):
exception_class=StringFormattingException,
)
format_variables: List[str] = list(
re.findall(StringFormatterValidator.__fields_validator, self.format_string)
)
format_variables: List[str] = list(re.findall(_fields_validator, self.format_string))
if len(format_variables) != open_bracket_count:
raise self._validation_exception(
"{variable_names} must start with a lowercase letter, should only contain lowercase"
"letters, numbers, underscores, and have a single open and close bracket.",
error_message=_fields_validator_exception_message,
exception_class=StringFormattingException,
)

32
tests/e2e/plugins/test_regex.py
View file

@ -27,32 +27,44 @@ def regex_subscription_dict(output_directory):
"should not cap (.+) - (.+)",
".*\\[(.+) - (Feb.+)]", # should filter out march video
],
"capture_group_names": ["title_type", "title_date"],
},
"description": {
"match": [".*http:\\/\\/(.+).com.*"],
"capture_group_names": ["description_website"],
},
"description": {"match": [".*http:\\/\\/(.+).com.*"]},
"upload_date_standardized": {
"match": ["([0-9]+)-([0-9]+)-27"],
"capture_group_names": [
"upload_captured_year",
"upload_captured_month",
],
"defaults": [
"First",
"Second containing {in_regex_default}",
],
},
"artist": {"match": ["Never (.*) capture"], "defaults": ["Always default"]},
"artist": {
"match": ["Never (.*) capture"],
"capture_group_names": ["always_default"],
"defaults": ["Always default"],
},
},
},
"nfo_tags": {
"tags": {
"title_cap_1": "{title_capture_1}",
"title_cap_2": "{title_capture_2}",
"desc_cap": "{description_capture_1}",
"upload_date_both_caps": "{upload_date_standardized_capture_1} and {upload_date_standardized_capture_2}",
"artist_cap_always_default": "{artist_capture_1}",
"override_with_capture_variable": "{contains_regex_default}"
"title_cap_1": "{title_type}",
"title_cap_2": "{title_date}",
"desc_cap": "{description_website}",
"upload_date_both_caps": "{upload_captured_year} and {upload_captured_month}",
"artist_cap_always_default": "{always_default}",
"override_with_capture_variable": "{contains_regex_default}",
}
},
"overrides": {
"in_regex_default": "in regex default",
"contains_regex_default": "contains {title_capture_1}"
}
"contains_regex_default": "contains {title_type}",
},
}

4
tests/e2e/resources/transaction_log_summaries/plugins/test_regex.txt
View file

@ -10,10 +10,11 @@ Project Zombie - Jesse's Minecraft Server [Trailer - Feb.1].nfo
artist: Project Zombie
artist_cap_always_default: Always default
desc_cap: www.jesseminecraft.webs
override_with_capture_variable: contains Trailer
title: Jesse's Minecraft Server [Trailer - Feb.1]
title_cap_1: Trailer
title_cap_2: Feb.1
upload_date_both_caps: First and Second
upload_date_both_caps: First and Second containing in regex default
year: 2011
Project Zombie - Jesse's Minecraft Server [Trailer - Feb.27]-thumb.jpg
Project Zombie - Jesse's Minecraft Server [Trailer - Feb.27].mp4
@ -24,6 +25,7 @@ Project Zombie - Jesse's Minecraft Server [Trailer - Feb.27].nfo
artist: Project Zombie
artist_cap_always_default: Always default
desc_cap: jesseminecraft.webs
override_with_capture_variable: contains Trailer
title: Jesse's Minecraft Server [Trailer - Feb.27]
title_cap_1: Trailer
title_cap_2: Feb.27