danvics/ytdl-sub
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

[FEATURE] Regex capture and filtering on source variables plugin (#100)

Browse source 
* actually working

* changed plugin format. Need unit tests, documentation, update e2e to filter

* refactor plugin name, matching now working

* description work in progress, need more intricate testing

* tests looks good, regenerate output

* e2e tested, need more failure tests, no dupe variable names, add docs

* unit tests passing again

* more fail tests

* updated with name, default, better docs

* sanitized vars created
This commit is contained in:
Jesse Bannon 2022-07-16 23:39:46 -07:00 committed by GitHub
parent 119a0c212f
commit 53adbbcb22
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
24 changed files with 840 additions and 112 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

11
docs/config.rst
View file

@ -192,6 +192,17 @@ nfo_output_directory
-------------------------------------------------------------------------------
regex
'''''
.. autoclass:: ytdl_sub.plugins.regex.RegexOptions()
:members: skip_if_match_fails
.. autoclass:: ytdl_sub.plugins.regex.SourceVariableRegex()
:members: match, capture_group_names, capture_group_defaults
:member-order: bysource
-------------------------------------------------------------------------------
.. _subscription_yaml:
subscription.yaml

23
src/ytdl_sub/config/preset.py
View file

@ -83,6 +83,10 @@ class Preset(StrictDictValidator):
# and ensure required keys are present.
_optional_keys = PRESET_KEYS
@property
def _source_variables(self) -> List[str]:
return self.downloader.downloader_entry_type.source_variables()
def __validate_and_get_downloader(self, downloader_source: str) -> Type[Downloader]:
return self._validate_key(key=downloader_source, validator=DownloadStrategyValidator).get(
downloader_source=downloader_source
@ -141,6 +145,9 @@ class Preset(StrictDictValidator):
plugin = PluginMapping.get(plugin=key)
plugin_options = self._validate_key(key=key, validator=plugin.plugin_options_type)
plugin_options.validate_with_variables(
source_variables=self._source_variables, override_variables=self.overrides.keys
)
plugins.append((plugin, plugin_options))
@ -156,12 +163,20 @@ class Preset(StrictDictValidator):
# If the formatter supports source variables, set the formatter variables to include
# both source and override variables
if not isinstance(formatter_validator, OverridesStringFormatterValidator):
source_variables = {
source_var: "dummy_string"
for source_var in self.downloader.downloader_entry_type.source_variables()
}
source_variables = {source_var: "dummy_string" for source_var in self._source_variables}
variable_dict = dict(source_variables, **variable_dict)
# For all plugins, add in any extra added source variables
for _, plugin_options in self.plugins:
added_plugin_variables = {
source_var: "dummy_string" for source_var in plugin_options.added_source_variables()
}
# sanity check plugin variables do not override source variables
expected_len = len(variable_dict) + len(added_plugin_variables)
variable_dict = dict(variable_dict, **added_plugin_variables)
assert len(variable_dict) == expected_len, "plugin variables overwrote source variables"
_ = formatter_validator.apply_formatter(variable_dict=variable_dict)
def __recursive_preset_validate(

2
src/ytdl_sub/config/preset_class_mappings.py
View file

@ -13,6 +13,7 @@ from ytdl_sub.plugins.music_tags import MusicTagsPlugin
from ytdl_sub.plugins.nfo_tags import NfoTagsPlugin
from ytdl_sub.plugins.output_directory_nfo_tags import OutputDirectoryNfoTagsPlugin
from ytdl_sub.plugins.plugin import Plugin
from ytdl_sub.plugins.regex import RegexPlugin
class DownloadStrategyMapping:
@ -107,6 +108,7 @@ class PluginMapping:
"music_tags": MusicTagsPlugin,
"nfo_tags": NfoTagsPlugin,
"output_directory_nfo_tags": OutputDirectoryNfoTagsPlugin,
"regex": RegexPlugin,
}
@classmethod

27
src/ytdl_sub/entries/base_entry.py
View file

@ -22,6 +22,8 @@ class BaseEntry(ABC):
self._working_directory = working_directory
self._kwargs = entry_dict
self._additional_variables: Dict[str, str] = {}
def kwargs_contains(self, key: str) -> bool:
"""Returns whether internal kwargs contains the specified key"""
return key in self._kwargs
@ -39,3 +41,28 @@ class BaseEntry(ABC):
The working directory
"""
return self._working_directory
def add_variables(self, variables_to_add: Dict[str, str]) -> "BaseEntry":
"""
Parameters
----------
variables_to_add
Variables to add to this entry
Returns
-------
self
Raises
------
ValueError
If a variable trying to be added already exists as a source variable
"""
for variable_name in variables_to_add.keys():
if self.kwargs_contains(variable_name):
raise ValueError(
f"Cannot add variable '{variable_name}': already exists in the kwargs"
)
self._additional_variables = dict(self._additional_variables, **variables_to_add)
return self

13
src/ytdl_sub/entries/variables/entry_variables.py
View file

@ -37,6 +37,14 @@ class SourceVariables:
"""
return self.kwargs("extractor")
def _added_variables(self: BaseEntry) -> Dict[str, str]:
"""
Returns
-------
Dict of variables added to this entry
"""
return self._additional_variables
@classmethod
def source_variables(cls) -> List[str]:
"""
@ -54,7 +62,10 @@ class SourceVariables:
-------
Dictionary containing all variables
"""
return {source_var: getattr(self, source_var) for source_var in self.source_variables()}
source_variable_dict = {
source_var: getattr(self, source_var) for source_var in self.source_variables()
}
return dict(source_variable_dict, **self._added_variables())
class EntryVariables(SourceVariables):

34
src/ytdl_sub/plugins/plugin.py
View file

@ -1,5 +1,6 @@
from abc import ABC
from typing import Generic
from typing import List
from typing import Optional
from typing import Type
from typing import TypeVar
@ -19,6 +20,35 @@ class PluginOptions(StrictDictValidator):
Class that defines the parameters to a plugin
"""
# pylint: disable=no-self-use
def added_source_variables(self) -> List[str]:
"""
If the plugin adds source variables, list them here.
Returns
-------
List of added source variables this plugin creates
"""
return []
# pylint: disable=unused-argument
def validate_with_variables(
self, source_variables: List[str], override_variables: List[str]
) -> None:
"""
Optional validation after init with the session's source and override variables.
Parameters
----------
source_variables
Available source variables when running the plugin
override_variables
Available override variables when running the plugin
"""
return None
# pylint: enable=no-self-use,unused-argument
PluginOptionsT = TypeVar("PluginOptionsT", bound=PluginOptions)
@ -43,6 +73,7 @@ class Plugin(DownloadArchiver, Generic[PluginOptionsT], ABC):
# TODO pass yaml snake case name in the class somewhere, and use it for the logger
self._logger = Logger.get(self.__class__.__name__)
# pylint: disable=no-self-use
def modify_entry(self, entry: Entry) -> Optional[Entry]:
"""
For each entry downloaded, modify the entry in some way before sending it to
@ -57,6 +88,9 @@ class Plugin(DownloadArchiver, Generic[PluginOptionsT], ABC):
-------
The entry or None, indicating not to move it to the output directory
"""
return entry
# pylint: enable=no-self-use
def post_process_entry(self, entry: Entry) -> Optional[FileMetadata]:
"""

311
src/ytdl_sub/plugins/regex.py Normal file
View file

@ -0,0 +1,311 @@
from typing import Dict
from typing import List
from typing import Optional
from yt_dlp.utils import sanitize_filename
from ytdl_sub.entries.entry import Entry
from ytdl_sub.plugins.plugin import Plugin
from ytdl_sub.plugins.plugin import PluginOptions
from ytdl_sub.utils.exceptions import RegexNoMatchException
from ytdl_sub.utils.logger import Logger
from ytdl_sub.validators.regex_validator import RegexListValidator
from ytdl_sub.validators.source_variable_validator import SourceVariableNameListValidator
from ytdl_sub.validators.strict_dict_validator import StrictDictValidator
from ytdl_sub.validators.string_formatter_validators import ListFormatterValidator
from ytdl_sub.validators.string_formatter_validators import StringFormatterValidator
from ytdl_sub.validators.validators import BoolValidator
logger = Logger.get(name="regex")
class SourceVariableRegex(StrictDictValidator):
_required_keys = {"match"}
_optional_keys = {"capture_group_defaults", "capture_group_names"}
def __init__(self, name, value):
super().__init__(name, value)
self._match = self._validate_key(key="match", validator=RegexListValidator)
self._capture_group_defaults = self._validate_key_if_present(
key="capture_group_defaults", validator=ListFormatterValidator
)
self._capture_group_names = self._validate_key_if_present(
key="capture_group_names", validator=SourceVariableNameListValidator, default=[]
)
# If defaults are to be used, ensure there are the same number of defaults as there are
# capture groups
if self._capture_group_defaults is not None and self._match.num_capture_groups != len(
self._capture_group_defaults.list
):
raise self._validation_exception(
f"number of defaults must match number of capture groups, "
f"{len(self._capture_group_defaults.list)} != {self._match.num_capture_groups}"
)
# If there are capture groups, ensure there are capture group names
if len(self._capture_group_names.list) != self._match.num_capture_groups:
raise self._validation_exception(
f"number of capture group names must match number of capture groups, "
f"{len(self._capture_group_names.list)} != {self._match.num_capture_groups}"
)
@property
def match(self) -> RegexListValidator:
"""
Required. List of regex strings to try to match against a source variable. Each regex
string must have the same number of capture groups.
"""
return self._match
@property
def capture_group_names(self) -> Optional[List[str]]:
"""
Optional (only when no capture groups are in the regex string). List of names to store the
capture group values to. These and ``_sanitized`` versions will be available to use as
source variables. The list's length must be equal to the number of match capture groups.
"""
return [validator.value for validator in self._capture_group_names.list]
@property
def capture_group_defaults(self) -> Optional[List[StringFormatterValidator]]:
"""
Optional. List of string format validators to use for capture group defaults if a
source variable cannot be matched. The list's length must be equal to the number of match
capture groups.
"""
return self._capture_group_defaults.list if self.has_defaults else None
@property
def has_defaults(self) -> bool:
"""
Returns
-------
True if a validation exception should be raised if not captured. False otherwise.
"""
return self._capture_group_defaults is not None
class FromSourceVariablesRegex(StrictDictValidator):
_optional_keys = Entry.source_variables()
_allow_extra_keys = True
def __init__(self, name, value):
super().__init__(name, value)
self.source_variable_capture_dict: Dict[str, SourceVariableRegex] = {
key: self._validate_key(key=key, validator=SourceVariableRegex) for key in self._keys
}
class RegexOptions(PluginOptions):
r"""
Performs regex matching on an entry's source variables. Regex can be used to filter entries
from proceeding with download or capture groups to create new source variables.
Usage:
.. code-block:: yaml
presets:
my_example_preset:
regex:
# By default, if any match fails and has no defaults, the entry will be skipped.
# If set to False, ytdl-sub will error and stop all downloads from proceeding.
skip_if_match_fails: True
from:
# For each entry's `title` value...
title:
# Perform this regex match on it to act as a filter.
# This will only download videos with "Official Video" in it.
match: '\[Official Video\]'
# For each entry's `description` value...
description:
# Match with capture groups and defaults.
# This tries to scrape a date from the description and produce new source variables
match: "([0-9]{4})-([0-9]{2})-([0-9]{2})"
# Each capture group creates these new source variables, respectively, as well
# a sanitized version, i.e. `captured_upload_year_sanitized`
capture_group_names:
- "captured_upload_year"
- "captured_upload_month"
- "captured_upload_day"
# And if the string does not match, use these as respective default values for the
# new source variables.
capture_group_defaults:
- "{upload_year}"
- "{upload_month}"
- "{upload_day}"
"""
_required_keys = {"from"}
_optional_keys = {"skip_if_match_fails"}
def __init__(self, name, value):
super().__init__(name, value)
self._from = self._validate_key(key="from", validator=FromSourceVariablesRegex)
self._skip_if_match_fails: bool = self._validate_key_if_present(
key="skip_if_match_fails", validator=BoolValidator, default=True
).value
@property
def skip_if_match_fails(self) -> Optional[bool]:
"""
Defaults to True. If True, when any match fails and has no defaults, the entry will be
skipped. If False, ytdl-sub will error and all downloads will not proceed.
"""
return self._skip_if_match_fails
def validate_with_variables(
self, source_variables: List[str], override_variables: List[str]
) -> None:
"""
Ensures each source variable capture group is valid
Parameters
----------
source_variables
Available source variables when running the plugin
override_variables
Available override variables when running the plugin
"""
for key, regex_options in self.source_variable_capture_dict.items():
# Ensure each variable getting captured is a source variable
if key not in source_variables:
raise self._validation_exception(
f"cannot regex capture '{key}' because it is not a source variable"
)
# Ensure the capture group names are not existing source/override variables
for capture_group_name in regex_options.capture_group_names:
if capture_group_name in source_variables:
raise self._validation_exception(
f"'{capture_group_name}' cannot be used as a capture group name because it "
f"is a source variable"
)
if capture_group_name in override_variables:
raise self._validation_exception(
f"'{capture_group_name}' cannot be used as a capture group name because it "
f"is an override variable"
)
@property
def source_variable_capture_dict(self) -> Dict[str, SourceVariableRegex]:
"""
Returns
-------
Dict of { source variable: capture options }
"""
return self._from.source_variable_capture_dict
def added_source_variables(self) -> List[str]:
"""
Returns
-------
List of new source variables created via regex capture
"""
added_source_vars: List[str] = []
for regex_options in self.source_variable_capture_dict.values():
added_source_vars.extend(regex_options.capture_group_names)
added_source_vars.extend(
f"{capture_group_name}_sanitized"
for capture_group_name in regex_options.capture_group_names
)
return added_source_vars
class RegexPlugin(Plugin[RegexOptions]):
plugin_options_type = RegexOptions
def modify_entry(self, entry: Entry) -> Optional[Entry]:
"""
Parameters
----------
entry
Entry to add source variables to
Returns
-------
Entry with regex capture variables added to its source variables
Raises
------
ValidationException
If no capture and no defaults
"""
entry_variable_dict = entry.to_dict()
# Iterate each source var to capture and add to the entry
for source_var, regex_options in self.plugin_options.source_variable_capture_dict.items():
maybe_capture = regex_options.match.match_any(input_str=entry_variable_dict[source_var])
# If no capture
if maybe_capture is None:
# and no defaults
if not regex_options.has_defaults:
# Skip the entry if toggled
if self.plugin_options.skip_if_match_fails:
logger.info(
"Regex failed to match '%s' from '%s', skipping.",
source_var,
entry.title,
)
return None
# Otherwise, error
raise RegexNoMatchException(
f"Regex failed to match '{source_var}' from '{entry.title}'"
)
# otherwise, use defaults (apply them using the original entry source dict)
source_variables_and_overrides_dict = dict(
entry_variable_dict, **self.overrides.dict_with_format_strings
)
# add both the default...
entry.add_variables(
variables_to_add={
regex_options.capture_group_names[i]: default.apply_formatter(
variable_dict=source_variables_and_overrides_dict
)
for i, default in enumerate(regex_options.capture_group_defaults)
},
)
# and sanitized default
entry.add_variables(
variables_to_add={
f"{regex_options.capture_group_names[i]}_sanitized": sanitize_filename(
default.apply_formatter(
variable_dict=source_variables_and_overrides_dict
)
)
for i, default in enumerate(regex_options.capture_group_defaults)
},
)
# There is a capture, add the source variables to the entry as
# {source_var}_capture_1, {source_var}_capture_2, ...
else:
# Add the value...
entry.add_variables(
variables_to_add={
regex_options.capture_group_names[i]: capture
for i, capture in enumerate(maybe_capture)
},
)
# And the sanitized value
entry.add_variables(
variables_to_add={
f"{regex_options.capture_group_names[i]}_sanitized": sanitize_filename(
capture
)
for i, capture in enumerate(maybe_capture)
},
)
return entry

39
src/ytdl_sub/subscriptions/subscription.py
View file

@ -3,6 +3,7 @@ import copy
import os
import shutil
from pathlib import Path
from typing import Dict
from typing import List
from typing import Optional
from typing import Tuple
@ -271,6 +272,17 @@ class Subscription:
if isinstance(entry, tuple):
entry, entry_metadata = entry
# First, modify the entry with all plugins
for plugin in plugins:
# Break out of this plugin loop if entry is None, it is indicated to not DL it
if (entry := plugin.modify_entry(entry)) is None:
break
# If entry is None from the broken out loop, continue over the other entries
if entry is None:
continue
# Then, post-process the entry with all plugins
for plugin in plugins:
optional_plugin_entry_metadata = plugin.post_process_entry(entry)
if optional_plugin_entry_metadata:
@ -307,3 +319,30 @@ class Subscription:
preset_options=preset,
config_options=config.config_options,
)
@classmethod
def from_dict(cls, config: ConfigFile, preset_name: str, preset_dict: Dict) -> "Subscription":
"""
Creates a subscription from a preset dict
Parameters
----------
config:
Validated instance of the config
preset_name:
Name of the preset
preset_dict:
The preset config in dict format
Returns
-------
Initialized subscription
"""
return cls.from_preset(
preset=Preset.from_dict(
config=config,
preset_name=preset_name,
preset_dict=preset_dict,
),
config=config,
)

8
src/ytdl_sub/utils/exceptions.py
View file

@ -10,6 +10,10 @@ class StringFormattingVariableNotFoundException(StringFormattingException):
"""Tried to format a string but the variable was not found"""
class InvalidVariableNameException(ValidationException):
"""A user defined variable name is invalid"""
class DownloadArchiveException(ValueError):
"""Any user or file errors caused by download archive or mapping files"""
@ -20,3 +24,7 @@ class FileNotFoundException(ValidationException):
class InvalidYamlException(ValidationException):
"""User yaml that is invalid"""
class RegexNoMatchException(ValidationException):
"""Regex failed to match during download"""

45
src/ytdl_sub/validators/regex_validator.py
View file

@ -28,33 +28,20 @@ class RegexValidator(StringValidator):
"""
return self._compiled_regex.groups
def is_match(self, input_str: str) -> bool:
def match(self, input_str: str) -> Optional[List[str]]:
"""
Parameters
----------
input_str
String to match against the regex
String to regex match
Returns
-------
True if input_str matches. False otherwise.
"""
return self._compiled_regex.search(input_str) is not None
def capture(self, input_str: str) -> Optional[List[str]]:
"""
Parameters
----------
input_str
String to try to regex capture from
Returns
-------
List of captures (will always be >= 1). None if there are no captures.
List of captures. If the regex has no capture groups, then the list will be emtpy.
None is returned if the input_str failed to match
"""
if match := self._compiled_regex.search(input_str):
if len(to_return := list(match.groups())) > 0:
return to_return
return list(match.groups())
return None
@ -70,20 +57,18 @@ class RegexListValidator(ListValidator[RegexValidator]):
"each regex in a list must have the same number of capture groups"
)
def matches_any(self, input_str: str) -> bool:
"""
Parameters
----------
input_str
String to match against any regexes in the list
self._num_capture_groups = self._list[0].num_capture_groups
@property
def num_capture_groups(self) -> int:
"""
Returns
-------
True if at least one matches. False if none match
Number of capture groups. All regexes in the list will have the same number.
"""
return any(reg.is_match(input_str) for reg in self._list)
return self._num_capture_groups
def capture_any(self, input_str: str) -> Optional[List[str]]:
def match_any(self, input_str: str) -> Optional[List[str]]:
"""
Parameters
----------
@ -92,10 +77,10 @@ class RegexListValidator(ListValidator[RegexValidator]):
Returns
-------
List of captures (will always be >= 1) on the first regex that matches. None if
no regexes match.
List of captures on the first regex that matches. If the regex has no capture groups, then
the list will be emtpy. None is returned if the input_str failed to match
"""
for reg in self._list:
if maybe_capture := reg.capture(input_str):
if (maybe_capture := reg.match(input_str)) is not None:
return maybe_capture
return None

20
src/ytdl_sub/validators/source_variable_validator.py Normal file
View file

@ -0,0 +1,20 @@
from ytdl_sub.utils.exceptions import InvalidVariableNameException
from ytdl_sub.validators.string_formatter_validators import is_valid_source_variable_name
from ytdl_sub.validators.validators import ListValidator
from ytdl_sub.validators.validators import StringValidator
class SourceVariableNameValidator(StringValidator):
_expected_value_type_name = "source variable name"
def __init__(self, name, value):
super().__init__(name, value)
try:
_ = is_valid_source_variable_name(self.value, raise_exception=True)
except InvalidVariableNameException as exc:
raise self._validation_exception(exc) from exc
class SourceVariableNameListValidator(ListValidator[SourceVariableNameValidator]):
_inner_list_type = SourceVariableNameValidator
_expected_value_type_name = "source variable name list"

47
src/ytdl_sub/validators/string_formatter_validators.py
View file

@ -5,11 +5,45 @@ from typing import Dict
from typing import List
from typing import final
from ytdl_sub.utils.exceptions import InvalidVariableNameException
from ytdl_sub.utils.exceptions import StringFormattingException
from ytdl_sub.utils.exceptions import StringFormattingVariableNotFoundException
from ytdl_sub.validators.validators import ListValidator
from ytdl_sub.validators.validators import LiteralDictValidator
from ytdl_sub.validators.validators import Validator
_fields_validator = re.compile(r"{([a-z][a-z0-9_]+?)}")
_fields_validator_exception_message: str = (
"{variable_names} must start with a lowercase letter, should only contain lowercase letters, "
"numbers, underscores, and have a single open and close bracket."
)
def is_valid_source_variable_name(input_str: str, raise_exception: bool = False) -> bool:
"""
Parameters
----------
input_str
String to see if it can be a source variable
raise_exception
Raise InvalidVariableNameException False.
Returns
-------
True if it is. False otherwise.
Raises
------
InvalidVariableNameException
If raise_exception and output is False
"""
# Add brackets around it to pretend its a StringFormatter, see if it captures
is_source_variable_name = len(re.findall(_fields_validator, f"{{{input_str}}}")) > 0
if not is_source_variable_name and raise_exception:
raise InvalidVariableNameException(_fields_validator_exception_message)
return is_source_variable_name
class StringFormatterValidator(Validator):
"""
@ -43,8 +77,6 @@ class StringFormatterValidator(Validator):
"Format variable '{variable_name}' does not exist. Available variables: {available_fields}"
)
__fields_validator = re.compile(r"{([a-z_]+?)}")
__max_format_recursion = 3
def __validate_and_get_format_variables(self) -> List[str]:
@ -69,14 +101,11 @@ class StringFormatterValidator(Validator):
exception_class=StringFormattingException,
)
format_variables: List[str] = list(
re.findall(StringFormatterValidator.__fields_validator, self.format_string)
)
format_variables: List[str] = list(re.findall(_fields_validator, self.format_string))
if len(format_variables) != open_bracket_count:
raise self._validation_exception(
"{variable_names} should only contain "
"lowercase letters and underscores with a single open and close bracket.",
error_message=_fields_validator_exception_message,
exception_class=StringFormattingException,
)
@ -177,6 +206,10 @@ class OverridesStringFormatterValidator(StringFormatterValidator):
# pylint: enable=line-too-long
class ListFormatterValidator(ListValidator[StringFormatterValidator]):
_inner_list_type = StringFormatterValidator
class DictFormatterValidator(LiteralDictValidator):
"""
A dict made up of

9
src/ytdl_sub/validators/validators.py
View file

@ -112,6 +112,15 @@ class ListValidator(Validator, ABC, Generic[T]):
for i, val in enumerate(self._value)
]
@property
def list(self) -> List[T]:
"""
Returns
-------
The list
"""
return self._list
class DictValidator(Validator):
"""

12
tests/e2e/conftest.py
View file

@ -2,8 +2,20 @@ import tempfile
import pytest
from ytdl_sub.config.config_file import ConfigFile
@pytest.fixture()
def output_directory():
with tempfile.TemporaryDirectory() as temp_dir:
yield temp_dir
@pytest.fixture()
def music_video_config():
return ConfigFile.from_file_path(config_path="examples/kodi_music_videos_config.yaml")
@pytest.fixture()
def channel_as_tv_show_config():
return ConfigFile.from_file_path(config_path="examples/kodi_tv_shows_config.yaml")

0
tests/e2e/plugins/__init__.py Normal file
View file

208
tests/e2e/plugins/test_regex.py Normal file
View file

@ -0,0 +1,208 @@
import copy
import re
import pytest
from e2e.expected_transaction_log import assert_transaction_log_matches
from ytdl_sub.config.preset import Preset
from ytdl_sub.subscriptions.subscription import Subscription
from ytdl_sub.utils.exceptions import RegexNoMatchException
from ytdl_sub.utils.exceptions import ValidationException
@pytest.fixture
def regex_subscription_dict(output_directory):
return {
"preset": "yt_music_video_playlist",
"youtube": {"playlist_url": "https://youtube.com/playlist?list=PL5BC0FC26BECA5A35"},
# override the output directory with our fixture-generated dir
"output_options": {"output_directory": output_directory},
# download the worst format so it is fast
"ytdl_options": {
"format": "best[height<=480]",
},
"regex": {
# tests that skip_if_match_fails defaults to True
"from": {
"title": {
"match": [
"should not cap (.+) - (.+)",
".*\\[(.+) - (Feb.+)]", # should filter out march video
],
"capture_group_names": ["title_type", "title_date"],
},
"description": {
"match": [".*http:\\/\\/(.+).com.*"],
"capture_group_names": ["description_website"],
},
"upload_date_standardized": {
"match": ["([0-9]+)-([0-9]+)-27"],
"capture_group_names": [
"upload_captured_year",
"upload_captured_month",
],
"capture_group_defaults": [
"First",
"Second containing {in_regex_default}",
],
},
"artist": {
"match": ["Never (.*) capture"],
"capture_group_names": ["always_default"],
"capture_group_defaults": ["Always default"],
},
},
},
"nfo_tags": {
"tags": {
"title_cap_1": "{title_type}",
"title_cap_1_sanitized": "{title_type_sanitized}",
"title_cap_2": "{title_date}",
"desc_cap": "{description_website}",
"upload_date_both_caps": "{upload_captured_year} and {upload_captured_month}",
"artist_cap_always_default": "{always_default}",
"artist_cap_always_default_sanitized": "{always_default_sanitized}",
"override_with_capture_variable": "{contains_regex_default}",
"override_with_capture_variable_sanitized": "{contains_regex_sanitized_default}",
}
},
"overrides": {
"in_regex_default": "in regex default",
"contains_regex_default": "contains {title_type}",
"contains_regex_sanitized_default": "contains {title_type_sanitized}",
},
}
@pytest.fixture
def regex_subscription_dict_no_match_fails(regex_subscription_dict):
regex_subscription_dict["regex"]["skip_if_match_fails"] = False
return regex_subscription_dict
@pytest.fixture
def playlist_subscription(music_video_config, regex_subscription_dict):
playlist_preset = Preset.from_dict(
config=music_video_config,
preset_name="regex_capture_playlist_test",
preset_dict=regex_subscription_dict,
)
return Subscription.from_preset(
preset=playlist_preset,
config=music_video_config,
)
@pytest.fixture
def playlist_subscription_no_match_fails(
music_video_config, regex_subscription_dict_no_match_fails
):
playlist_preset = Preset.from_dict(
config=music_video_config,
preset_name="regex_capture_playlist_test",
preset_dict=regex_subscription_dict_no_match_fails,
)
return Subscription.from_preset(
preset=playlist_preset,
config=music_video_config,
)
class TestRegex:
def test_regex_success(self, playlist_subscription, output_directory):
# Only dry run is needed to see if capture variables are created
transaction_log = playlist_subscription.download(dry_run=True)
assert_transaction_log_matches(
output_directory=output_directory,
transaction_log=transaction_log,
transaction_log_summary_file_name="plugins/test_regex.txt",
)
def test_regex_fails_no_match(self, playlist_subscription_no_match_fails, output_directory):
with pytest.raises(
RegexNoMatchException,
match=re.escape(
"Regex failed to match 'title' from 'Jesse's Minecraft Server [Trailer - Mar.21]'"
),
):
_ = playlist_subscription_no_match_fails.download(dry_run=True)
def test_regex_fails_capture_group_is_source_variable(
self, regex_subscription_dict, music_video_config
):
regex_subscription_dict["regex"]["from"]["title"]["capture_group_names"][0] = "uid"
with pytest.raises(
ValidationException,
match=re.escape(
"'uid' cannot be used as a capture group name because it is a source variable"
),
):
_ = Subscription.from_dict(
config=music_video_config,
preset_name="test_regex_fails_capture_group_is_source_variable",
preset_dict=regex_subscription_dict,
)
def test_regex_fails_capture_group_is_override_variable(
self, regex_subscription_dict, music_video_config
):
regex_subscription_dict["regex"]["from"]["title"]["capture_group_names"][
0
] = "in_regex_default"
with pytest.raises(
ValidationException,
match=re.escape(
"'in_regex_default' cannot be used as a capture group name because it is an override variable"
),
):
_ = Subscription.from_dict(
config=music_video_config,
preset_name="test_regex_fails_capture_group_is_override_variable",
preset_dict=regex_subscription_dict,
)
def test_regex_fails_source_variable_does_not_exist(
self, regex_subscription_dict, music_video_config
):
regex_subscription_dict["regex"]["from"]["dne"] = copy.deepcopy(
regex_subscription_dict["regex"]["from"]["title"]
)
with pytest.raises(
ValidationException,
match=re.escape("cannot regex capture 'dne' because it is not a source variable"),
):
_ = Subscription.from_dict(
config=music_video_config,
preset_name="test_regex_fails_source_variable_does_not_exist",
preset_dict=regex_subscription_dict,
)
def test_regex_fails_unequal_defaults(self, regex_subscription_dict, music_video_config):
regex_subscription_dict["regex"]["from"]["title"]["capture_group_defaults"] = ["1 != 2"]
with pytest.raises(
ValidationException,
match=re.escape("number of defaults must match number of capture groups, 1 != 2"),
):
_ = Subscription.from_dict(
config=music_video_config,
preset_name="test_regex_fails_unequal_defaults",
preset_dict=regex_subscription_dict,
)
def test_regex_fails_unequal_capture_group_names(
self, regex_subscription_dict, music_video_config
):
regex_subscription_dict["regex"]["from"]["title"]["capture_group_names"].append("unequal")
with pytest.raises(
ValidationException,
match=re.escape(
"number of capture group names must match number of capture groups, 3 != 2"
),
):
_ = Subscription.from_dict(
config=music_video_config,
preset_name="test_regex_fails_unequal_capture_group_names",
preset_dict=regex_subscription_dict,
)

39
tests/e2e/resources/transaction_log_summaries/plugins/test_regex.txt Normal file
View file

@ -0,0 +1,39 @@
Files created in '{output_directory}'
----------------------------------------
.ytdl-sub-regex_capture_playlist_test-download-archive.json
Project Zombie - Jesse's Minecraft Server [Trailer - Feb.1]-thumb.jpg
Project Zombie - Jesse's Minecraft Server [Trailer - Feb.1].mp4
Project Zombie - Jesse's Minecraft Server [Trailer - Feb.1].nfo
NFO tags:
musicvideo:
album: Music Videos
artist: Project Zombie
artist_cap_always_default: Always default
artist_cap_always_default_sanitized: Always default
desc_cap: www.jesseminecraft.webs
override_with_capture_variable: contains Trailer
override_with_capture_variable_sanitized: contains Trailer
title: Jesse's Minecraft Server [Trailer - Feb.1]
title_cap_1: Trailer
title_cap_1_sanitized: Trailer
title_cap_2: Feb.1
upload_date_both_caps: First and Second containing in regex default
year: 2011
Project Zombie - Jesse's Minecraft Server [Trailer - Feb.27]-thumb.jpg
Project Zombie - Jesse's Minecraft Server [Trailer - Feb.27].mp4
Project Zombie - Jesse's Minecraft Server [Trailer - Feb.27].nfo
NFO tags:
musicvideo:
album: Music Videos
artist: Project Zombie
artist_cap_always_default: Always default
artist_cap_always_default_sanitized: Always default
desc_cap: jesseminecraft.webs
override_with_capture_variable: contains Trailer
override_with_capture_variable_sanitized: contains Trailer
title: Jesse's Minecraft Server [Trailer - Feb.27]
title_cap_1: Trailer
title_cap_1_sanitized: Trailer
title_cap_2: Feb.27
upload_date_both_caps: 2011 and 02
year: 2011

2
tests/e2e/resources/transaction_log_summaries/youtube/test_playlist.txt
View file

@ -1,6 +1,6 @@
Files created in '{output_directory}'
----------------------------------------
.ytdl-sub-jmc-download-archive.json
.ytdl-sub-music_video_playlist_test-download-archive.json
JMC - Jesse's Minecraft Server [Trailer - Feb.1]-thumb.jpg
JMC - Jesse's Minecraft Server [Trailer - Feb.1].mp4
JMC - Jesse's Minecraft Server [Trailer - Feb.1].nfo

36
tests/e2e/youtube/test_playlist_as_kodi_music_videos.py
View file

@ -8,28 +8,12 @@ from e2e.expected_download import ExpectedDownloads
from e2e.expected_transaction_log import assert_transaction_log_matches
import ytdl_sub.downloaders.downloader
from ytdl_sub.config.config_file import ConfigFile
from ytdl_sub.config.preset import Preset
from ytdl_sub.subscriptions.subscription import Subscription
@pytest.fixture
def config_path():
return "examples/kodi_music_videos_config.yaml"
@pytest.fixture
def subscription_name():
return "jmc"
@pytest.fixture
def config(config_path):
return ConfigFile.from_file_path(config_path=config_path)
@pytest.fixture
def subscription_dict(output_directory, subscription_name):
def subscription_dict(output_directory):
return {
"preset": "yt_music_video_playlist",
"youtube": {"playlist_url": "https://youtube.com/playlist?list=PL5BC0FC26BECA5A35"},
@ -48,16 +32,16 @@ def subscription_dict(output_directory, subscription_name):
@pytest.fixture
def playlist_subscription(config, subscription_name, subscription_dict):
def playlist_subscription(music_video_config, subscription_dict):
playlist_preset = Preset.from_dict(
config=config,
preset_name=subscription_name,
config=music_video_config,
preset_name="music_video_playlist_test",
preset_dict=subscription_dict,
)
return Subscription.from_preset(
preset=playlist_preset,
config=config,
config=music_video_config,
)
@ -68,7 +52,7 @@ def expected_playlist_download():
return ExpectedDownloads(
expected_downloads=[
# Download mapping
ExpectedDownloadFile(path=Path(".ytdl-sub-jmc-download-archive.json"), md5="9f785c29194a6ecfba6a6b4018763ddc"),
ExpectedDownloadFile(path=Path(".ytdl-sub-music_video_playlist_test-download-archive.json"), md5="9f785c29194a6ecfba6a6b4018763ddc"),
# Entry files
ExpectedDownloadFile(path=Path("JMC - Jesse's Minecraft Server [Trailer - Feb.1]-thumb.jpg"), md5="b232d253df621aa770b780c1301d364d"),
@ -103,16 +87,16 @@ def single_video_subscription_dict(subscription_dict):
@pytest.fixture
def single_video_subscription(config, subscription_name, single_video_subscription_dict):
def single_video_subscription(music_video_config, single_video_subscription_dict):
single_video_preset = Preset.from_dict(
config=config,
preset_name=subscription_name,
config=music_video_config,
preset_name="music_video_single_video_test",
preset_dict=single_video_subscription_dict,
)
return Subscription.from_preset(
preset=single_video_preset,
config=config,
config=music_video_config,
)

29
tests/e2e/youtube/test_split_video.py
View file

@ -1,24 +1,15 @@
from pathlib import Path
from tempfile import NamedTemporaryFile
import mergedeep
import pytest
from conftest import assert_debug_log
from e2e.expected_download import ExpectedDownloadFile
from e2e.expected_download import ExpectedDownloads
from e2e.expected_transaction_log import assert_transaction_log_matches
import ytdl_sub.downloaders.downloader
from ytdl_sub.config.config_file import ConfigFile
from ytdl_sub.config.preset import Preset
from ytdl_sub.subscriptions.subscription import Subscription
@pytest.fixture
def config_path():
return "examples/kodi_music_videos_config.yaml"
@pytest.fixture
def split_timestamps_file_path():
timestamps = [
@ -37,17 +28,7 @@ def split_timestamps_file_path():
@pytest.fixture
def subscription_name():
return "jmc"
@pytest.fixture
def config(config_path):
return ConfigFile.from_file_path(config_path=config_path)
@pytest.fixture
def subscription_dict(output_directory, subscription_name, split_timestamps_file_path):
def subscription_dict(output_directory, split_timestamps_file_path):
return {
"preset": "yt_music_video",
"youtube": {
@ -73,16 +54,16 @@ def subscription_dict(output_directory, subscription_name, split_timestamps_file
@pytest.fixture
def single_video_subscription(config, subscription_name, subscription_dict):
def single_video_subscription(music_video_config, subscription_dict):
single_video_preset = Preset.from_dict(
config=config,
preset_name=subscription_name,
config=music_video_config,
preset_name="split_video_test",
preset_dict=subscription_dict,
)
return Subscription.from_preset(
preset=single_video_preset,
config=config,
config=music_video_config,
)

0
tests/unit/plugins/__init__.py Normal file
View file

1
tests/unit/plugins/test_regex_capture.py Normal file
View file

@ -0,0 +1 @@

25
tests/unit/validators/test_regex_validator.py
View file

@ -6,21 +6,20 @@ from ytdl_sub.validators.regex_validator import RegexValidator
@pytest.mark.parametrize(
"regex_value, input_str, matches, captures",
"regex_value, input_str, expected_output",
[
("^match this$", "match this", True, None),
("^my (.+) cap", "my first cap", True, ["first"]),
(".* (.+) - (.+) two", "my other - capped two", True, ["other", "capped"]),
("failed match", "nope", False, None),
("^match this$", "match this", []),
("^my (.+) cap", "my first cap", ["first"]),
(".* (.+) - (.+) two", "my other - capped two", ["other", "capped"]),
("failed match", "nope", None),
],
)
def test_regex_validator(regex_value, input_str, matches, captures):
def test_regex_validator(regex_value, input_str, expected_output):
regex_validator = RegexValidator(name="good_regex_validator", value=regex_value)
assert regex_validator._name == "good_regex_validator"
assert regex_validator._value == regex_value
assert regex_validator.is_match(input_str=input_str) is matches
assert regex_validator.capture(input_str=input_str) == captures
assert regex_validator.match(input_str=input_str) == expected_output
@pytest.mark.parametrize("regex_value", ["(", "??"])
@ -29,17 +28,16 @@ def test_regex_validator_fails_bad_value(regex_value):
_ = RegexValidator(name="fail", value=regex_value)
def test_regex_list_validator():
def test_regex_list_validator_matches():
regex_list_validator_raw_value = ["try matching this", "try matching that", "how about this"]
regex_list = RegexListValidator(name="list val", value=regex_list_validator_raw_value)
for raw_val in regex_list_validator_raw_value:
assert regex_list.matches_any(raw_val)
assert regex_list.capture_any(raw_val) is None
assert regex_list.match_any(raw_val) == []
def test_regex_list_validator_capture():
def test_regex_list_validator_captures():
regex_list_validator_raw_value = ["try (.+) this", "try (.+) that", "how (.+) this"]
regex_list = RegexListValidator(name="list val", value=regex_list_validator_raw_value)
@ -50,8 +48,7 @@ def test_regex_list_validator_capture():
]
for capture_str, expected_capture in captures_test:
assert regex_list.matches_any(capture_str)
assert regex_list.capture_any(capture_str) == [expected_capture]
assert regex_list.match_any(capture_str) == [expected_capture]
def test_regex_list_validator_invalid_regex():

11
tests/unit/validators/test_string_formatter_validator.py
View file

@ -19,8 +19,8 @@ def error_message_unequal_brackets_str():
@pytest.fixture
def error_message_unequal_regex_matches_str():
return (
"{variable_names} should only contain lowercase letters and "
"underscores with a single open and close bracket."
"{variable_names} must start with a lowercase letter, should only contain lowercase "
"letters, numbers, underscores, and have a single open and close bracket."
)
@ -29,11 +29,11 @@ def error_message_unequal_regex_matches_str():
)
class TestStringFormatterValidator(object):
def test_parse(self, string_formatter_class):
format_string = "Here is my {var_one} and {var_two} 💩"
format_string = "Here is my {var_one1} and {var_two} 💩"
validator = string_formatter_class(name="test_format_variables", value=format_string)
assert validator.format_string == format_string
assert validator.format_variables == ["var_one", "var_two"]
assert validator.format_variables == ["var_one1", "var_two"]
def test_format_variables(self, string_formatter_class):
format_string = "No vars 💩"
@ -64,7 +64,8 @@ class TestStringFormatterValidator(object):
@pytest.mark.parametrize(
"format_string",
[
"Try {var1} numbers",
"Try {1starting_number}",
"Try {_underscore_first}",
"Try {VAR1} caps w/numbers",
"Try {internal{bracket}}",
"Try }backwards{ facing",