danvics/ytdl-sub
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

[FEATURE] Ability to use overrides in regex plugin (#631)

Browse source 
* [FEATURE] Ability to use overrides in regex plugin

* lint

* more tests

* more docs

* docs
This commit is contained in:
Jesse Bannon 2023-06-17 13:02:35 -07:00 committed by GitHub
parent ac5561cb2e
commit a993beff0e
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
6 changed files with 147 additions and 40 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
docs/config.rst
View file

@ -213,7 +213,7 @@ regex
.. autoclass:: ytdl_sub.plugins.regex.RegexOptions()
:members: skip_if_match_fails
.. autoclass:: ytdl_sub.plugins.regex.SourceVariableRegex()
.. autoclass:: ytdl_sub.plugins.regex.VariableRegex()
:members: match, capture_group_names, capture_group_defaults, exclude
:member-order: bysource
:exclude-members: partial_validate

110
src/ytdl_sub/plugins/regex.py
View file

@ -11,6 +11,7 @@ from ytdl_sub.plugins.plugin import Plugin
from ytdl_sub.plugins.plugin import PluginOptions
from ytdl_sub.plugins.plugin import PluginPriority
from ytdl_sub.utils.exceptions import RegexNoMatchException
from ytdl_sub.utils.exceptions import StringFormattingVariableNotFoundException
from ytdl_sub.utils.logger import Logger
from ytdl_sub.validators.regex_validator import RegexListValidator
from ytdl_sub.validators.source_variable_validator import SourceVariableNameListValidator
@ -22,7 +23,7 @@ from ytdl_sub.validators.validators import BoolValidator
logger = Logger.get(name="regex")
class SourceVariableRegex(StrictDictValidator):
class VariableRegex(StrictDictValidator):
_optional_keys = {"match", "exclude", "capture_group_defaults", "capture_group_names"}
@ -65,7 +66,7 @@ class SourceVariableRegex(StrictDictValidator):
@property
def match(self) -> Optional[RegexListValidator]:
"""
List of regex strings to try to match against a source variable. Each regex
List of regex strings to try to match against a source or override variable. Each regex
string must have the same number of capture groups.
"""
return self._match
@ -73,9 +74,10 @@ class SourceVariableRegex(StrictDictValidator):
@property
def exclude(self) -> Optional[RegexListValidator]:
"""
List of regex strings to try to match against a source variable. If one of the regex strings
match, then the entry will be skipped. If both ``exclude`` and ``match`` are specified,
entries will get skipped if the regex matches against both ``exclude`` and ``match``.
List of regex strings to try to match against a source or override variable. If one of the
regex strings match, then the entry will be skipped. If both ``exclude`` and ``match`` are
specified, entries will get skipped if the regex matches against both ``exclude`` and
``match``.
"""
return self._exclude
@ -114,19 +116,25 @@ class FromSourceVariablesRegex(StrictDictValidator):
def __init__(self, name, value):
super().__init__(name, value)
self.source_variable_capture_dict: Dict[str, SourceVariableRegex] = {
key: self._validate_key(key=key, validator=SourceVariableRegex) for key in self._keys
self.variable_capture_dict: Dict[str, VariableRegex] = {
key: self._validate_key(key=key, validator=VariableRegex) for key in self._keys
}
class RegexOptions(PluginOptions):
r"""
Performs regex matching on an entry's source variables. Regex can be used to filter entries
from proceeding with download or capture groups to create new source variables. NOTE to
Performs regex matching on an entry's source or override variables. Regex can be used to filter
entries from proceeding with download or capture groups to create new source variables. NOTE to
use backslashes anywhere in your regex, i.e. ``\d``, you must add another backslash escape. This
means ``\d`` should be written as ``\\d``. This is because YAML requires an escape for any
backslash usage.
If you want to regex-search multiple source variables to create a logical OR effect, you can
create an override variable that contains the concatenation of them, and search that with regex.
For example, creating the override variable ``"title_and_description": "{title} {description}"``
and using ``title_and_description`` can regex match/exclude from either ``title`` or
``description``.
Usage:
.. code-block:: yaml
@ -216,9 +224,9 @@ class RegexOptions(PluginOptions):
"""
for key, regex_options in self.source_variable_capture_dict.items():
# Ensure each variable getting captured is a source variable
if key not in source_variables:
if key not in source_variables and key not in override_variables:
raise self._validation_exception(
f"cannot regex capture '{key}' because it is not a source variable"
f"cannot regex capture '{key}' because it is not a source or override variable"
)
# Ensure the capture group names are not existing source/override variables
@ -235,13 +243,13 @@ class RegexOptions(PluginOptions):
)
@property
def source_variable_capture_dict(self) -> Dict[str, SourceVariableRegex]:
def source_variable_capture_dict(self) -> Dict[str, VariableRegex]:
"""
Returns
-------
Dict of { source variable: capture options }
"""
return self._from.source_variable_capture_dict
return self._from.variable_capture_dict
def added_source_variables(self) -> List[str]:
"""
@ -267,28 +275,56 @@ class RegexPlugin(Plugin[RegexOptions]):
)
@classmethod
def _add_processed_regex_source_var(cls, entry: Entry, source_var: str) -> None:
def _add_processed_regex_variable_name(cls, entry: Entry, source_var: str) -> None:
if not entry.kwargs_contains(YTDL_SUB_REGEX_SOURCE_VARS):
entry.add_kwargs({YTDL_SUB_REGEX_SOURCE_VARS: []})
entry.kwargs(YTDL_SUB_REGEX_SOURCE_VARS).append(source_var)
@classmethod
def _contains_processed_regex_source_var(cls, entry: Entry, source_var: str) -> bool:
return source_var in entry.kwargs_get(YTDL_SUB_REGEX_SOURCE_VARS, [])
def _contains_processed_regex_variable(cls, entry: Entry, variable_name: str) -> bool:
return variable_name in entry.kwargs_get(YTDL_SUB_REGEX_SOURCE_VARS, [])
def _try_skip_entry(self, entry: Entry, source_var: str) -> None:
def _try_skip_entry(self, entry: Entry, variable_name: str) -> None:
# Skip the entry if toggled
if self.plugin_options.skip_if_match_fails:
logger.info(
"Regex failed to match '%s' from '%s', skipping.",
source_var,
variable_name,
entry.title,
)
return None
# Otherwise, error
raise RegexNoMatchException(f"Regex failed to match '{source_var}' from '{entry.title}'")
raise RegexNoMatchException(f"Regex failed to match '{variable_name}' from '{entry.title}'")
def _can_process_at_metadata_stage(self, entry: Entry, variable_name: str) -> bool:
# If the variable is an override...
if variable_name in self.overrides.dict:
# Try to see if it can resolve
try:
self.overrides.apply_formatter(
formatter=self.overrides.dict[variable_name],
entry=entry,
)
# If it can not from missing variables (from post-metadata stage), return False
except StringFormattingVariableNotFoundException:
return False
# If it is a source variable and not present, return false
elif variable_name not in entry.to_dict():
return False
return True
def _get_regex_input_string(self, entry: Entry, variable_name: str) -> str:
# Apply override formatter if it's an override
if variable_name in self.overrides.dict:
return self.overrides.apply_formatter(
formatter=self.overrides.dict[variable_name],
entry=entry,
)
# Otherwise pluck from the entry's source variable
return entry.to_dict()[variable_name]
def _modify_entry_metadata(self, entry: Entry, is_metadata_stage: bool) -> Optional[Entry]:
"""
@ -308,46 +344,50 @@ class RegexPlugin(Plugin[RegexOptions]):
ValidationException
If no capture and no defaults
"""
entry_variable_dict = entry.to_dict()
# Iterate each source var to capture and add to the entry
for source_var, regex_options in self.plugin_options.source_variable_capture_dict.items():
for (
variable_name,
regex_options,
) in self.plugin_options.source_variable_capture_dict.items():
# Record which regex source variables are processed, to
# process as many variables as possible in the metadata stage, then the rest
# after the media file has been downloaded.
if self._contains_processed_regex_source_var(entry, source_var):
if self._contains_processed_regex_variable(entry, variable_name):
continue
# Continue if the source var isn't in the dict since entry variables could be added
# after the metadata stage
if is_metadata_stage and source_var not in entry_variable_dict:
# If it's the metadata stage, and it can't be processed, skip until post-metadata
if is_metadata_stage and not self._can_process_at_metadata_stage(
entry=entry, variable_name=variable_name
):
continue
self._add_processed_regex_source_var(entry, source_var)
self._add_processed_regex_variable_name(entry, variable_name)
regex_input_str = self._get_regex_input_string(
entry=entry,
variable_name=variable_name,
)
if (
regex_options.exclude is not None
and regex_options.exclude.match_any(input_str=entry_variable_dict[source_var])
is not None
and regex_options.exclude.match_any(input_str=regex_input_str) is not None
):
return self._try_skip_entry(entry=entry, source_var=source_var)
return self._try_skip_entry(entry=entry, variable_name=variable_name)
# If match is present
if regex_options.match is not None:
maybe_capture = regex_options.match.match_any(
input_str=entry_variable_dict[source_var]
)
maybe_capture = regex_options.match.match_any(input_str=regex_input_str)
# And nothing matched
if maybe_capture is None:
# and no defaults
if not regex_options.has_defaults:
return self._try_skip_entry(entry=entry, source_var=source_var)
return self._try_skip_entry(entry=entry, variable_name=variable_name)
# otherwise, use defaults (apply them using the original entry source dict)
source_variables_and_overrides_dict = dict(
entry_variable_dict, **self.overrides.dict_with_format_strings
entry.to_dict(), **self.overrides.dict_with_format_strings
)
# add both the default...

2
tests/e2e/plugins/test_music_tags.py
View file

@ -1,8 +1,6 @@
import re
import pytest
from expected_download import assert_expected_downloads
from expected_transaction_log import assert_transaction_log_matches
from ytdl_sub.subscriptions.subscription import Subscription
from ytdl_sub.utils.exceptions import ValidationException

53
tests/e2e/plugins/test_regex.py
View file

@ -146,6 +146,33 @@ def regex_subscription_dict_match_and_exclude(regex_subscription_dict_base, outp
)
@pytest.fixture
def regex_subscription_dict_match_and_exclude_override_variable(
regex_subscription_dict_base, output_directory
):
return mergedeep.merge(
regex_subscription_dict_base,
{
"regex": {
# tests that skip_if_match_fails defaults to True
"from": {
"override_title": {
"exclude": [
"should not cap",
".*Feb.*", # should filter out march video
],
},
"override_description": {
"match": [".*http:\\/\\/(.+).com.*"],
"capture_group_names": ["override_description_website"],
},
},
},
"overrides": {"override_title": "{title}", "override_description": "{description}"},
},
)
@pytest.fixture
def playlist_subscription(music_video_config, regex_subscription_dict):
return Subscription.from_dict(
@ -179,6 +206,18 @@ def playlist_subscription_exclude(
)
@pytest.fixture
def playlist_subscription_overrides(
music_video_config: ConfigFile,
regex_subscription_dict_match_and_exclude_override_variable: Dict[str, Any],
) -> Subscription:
return Subscription.from_dict(
config=music_video_config,
preset_name="regex_using_overrides_test",
preset_dict=regex_subscription_dict_match_and_exclude_override_variable,
)
@pytest.fixture
def playlist_subscription_match_and_exclude(
music_video_config: ConfigFile, regex_subscription_dict_match_and_exclude: Dict[str, Any]
@ -221,6 +260,16 @@ class TestRegex:
regenerate_transaction_log=True,
)
def test_regex_using_overrides_success(self, playlist_subscription_overrides, output_directory):
# Should only contain the march video
transaction_log = playlist_subscription_overrides.download(dry_run=True)
assert_transaction_log_matches(
output_directory=output_directory,
transaction_log=transaction_log,
transaction_log_summary_file_name="plugins/test_regex_overrides.txt",
regenerate_transaction_log=True,
)
def test_regex_fails_no_match(self, playlist_subscription_no_match_fails, output_directory):
with pytest.raises(
RegexNoMatchException,
@ -300,7 +349,9 @@ class TestRegex:
)
with pytest.raises(
ValidationException,
match=re.escape("cannot regex capture 'dne' because it is not a source variable"),
match=re.escape(
"cannot regex capture 'dne' because it is not a source or override variable"
),
):
_ = Subscription.from_dict(
config=music_video_config,

5
tests/e2e/plugins/test_split_by_chapters.py
View file

@ -32,7 +32,9 @@ def yt_album_as_chapters_with_regex_preset_dict(yt_album_as_chapters_preset_dict
{
"regex": {
"from": {
"chapter_title": {
# Ensure regex can handle override variables that come from the
# post-metadata stage
"override_chapter_title": {
"match": r"\d+\. (.+)",
"capture_group_names": "captured_track_title",
"capture_group_defaults": "{chapter_title}",
@ -54,6 +56,7 @@ def yt_album_as_chapters_with_regex_preset_dict(yt_album_as_chapters_preset_dict
}
},
"overrides": {
"override_chapter_title": "{chapter_title}",
"track_title": "{captured_track_title}",
"track_album": "{captured_track_album}",
"track_artist": "{captured_track_artist}",

15
tests/resources/transaction_log_summaries/plugins/test_regex_overrides.txt Normal file
View file

@ -0,0 +1,15 @@
Files created:
----------------------------------------
{output_directory}
.ytdl-sub-regex_using_overrides_test-download-archive.json
{output_directory}/Project Zombie
Project Zombie - Jesse's Minecraft Server [Trailer - Mar.21]-thumb.jpg
Project Zombie - Jesse's Minecraft Server [Trailer - Mar.21].info.json
Project Zombie - Jesse's Minecraft Server [Trailer - Mar.21].mp4
Project Zombie - Jesse's Minecraft Server [Trailer - Mar.21].nfo
NFO tags:
musicvideo:
album: Music Videos
artist: Project Zombie
title: Jesse's Minecraft Server [Trailer - Mar.21]
year: 2011