From 1b57d79c8fd211af5f2ef4f76afa6df9892c0665 Mon Sep 17 00:00:00 2001 From: Jesse Bannon Date: Sun, 29 Sep 2024 08:41:09 -0700 Subject: [PATCH] [BREAKING CHANGE] Remove regex plugin (#1067) Regex plugin has been removed in favor of scripting. The function [regex_capture_many](https://ytdl-sub.readthedocs.io/en/latest/config_reference/scripting/scripting_functions.html#regex-capture-many) has been created to replicate the plugin's behavior. See the following converted example: `regex, now deprecated` ``` regex: from: title: match: - ".*? - (.*)" # Captures 'Song' from 'Emily Hopkins - Some - Song' capture_group_names: - "captured_track_title" capture_group_defaults: - "{title}" overrides: track_title: "{captured_track_title}" ``` `scripting equivalent` ``` overrides: # Captures 'Song' from 'Emily Hopkins - Some - Song' captured_track_title: >- { %regex_capture_many( title, [ ".*? - (.*)" ], [ title ] ) } track_title: "{%array_at(captured_track_title, 1)}" ``` ## Motivation: Regex was a unique plugin that added custom variables based on user input. This made the backend need to support both `overrides` and 'plugin user variables'. Removing `regex` plugin will consolidate this logic into only `overrides`, making it much easier to maintain. --- docs/source/config_reference/plugins.rst | 115 ----- docs/source/deprecation_notices.rst | 39 ++ src/ytdl_sub/config/plugin/plugin_mapping.py | 3 - src/ytdl_sub/plugins/regex.py | 456 ------------------ .../music/albums_from_playlists.yaml | 20 +- tests/e2e/plugins/test_regex.py | 402 --------------- tests/e2e/plugins/test_split_by_chapters.py | 48 +- 7 files changed, 69 insertions(+), 1014 deletions(-) delete mode 100644 src/ytdl_sub/plugins/regex.py delete mode 100644 tests/e2e/plugins/test_regex.py diff --git a/docs/source/config_reference/plugins.rst b/docs/source/config_reference/plugins.rst index cd0da7f9..c3161a27 100644 --- a/docs/source/config_reference/plugins.rst +++ b/docs/source/config_reference/plugins.rst @@ -770,121 +770,6 @@ In addition, any override variable defined will automatically create a ``sanitiz for use. In the example above, ``output_directory_sanitized`` will exist and perform sanitization on the value when used. ----------------------------------------------------------------------------------------------------- - -regex ------ -.. attention:: - - This plugin will eventually be deprecated and replaced by scripting functions. - You can replicate the example below using the following. - - .. code-block:: yaml - - # Only includes videos with 'Official Video' - filter_include: - - >- - { %contains( %lower(title), "official video" ) } - - # Excludes videos with '#short' in its description - filter_exclude: - - >- - { %contains( %lower(description), '#short' ) } - - # Creates a capture array with defaults, and assigns - # each capture group to its own variable - overrides: - description_date_capture: >- - { - %regex_capture_many( - description, - [ "([0-9]{4})-([0-9]{2})-([0-9]{2})" ], - [ upload_year, upload_month, upload_day ] - ) - } - captured_upload_year: >- - { %array_at(description_date_capture, 1) } - captured_upload_month: >- - { %array_at(description_date_capture, 2) } - captured_upload_day: >- - { %array_at(description_date_capture, 3) } - -Performs regex matching on an entry's source or override variables. Regex can be used to filter -entries from proceeding with download or capture groups to create new source variables. - -NOTE that YAML differentiates between single-quote (``'``) and double-quote (``"``), which can -affect regex. Double-quote implies string literals, i.e. ``"\n"`` is the literal chars ``\n``, -whereas single-quote, ``'\n'`` gets evaluated to a new line. To escape ``\`` when using -single-quote, use ``\\``. This is necessary if you want your regex to be something like -``\d\n`` to match a number and adjacent new-line. It must be written as ``\\d\n``. - -If you want to regex-search multiple source variables to create a logical OR effect, you can -create an override variable that contains the concatenation of them, and search that with regex. -For example, creating the override variable ``"title_and_description": "{title} {description}"`` -and using ``title_and_description`` can regex match/exclude from either ``title`` or -``description``. - -:Usage: - -.. code-block:: yaml - - regex: - # By default, if any match fails and has no defaults, the entry will - # be skipped. If False, ytdl-sub will error and stop all downloads - # from proceeding. - skip_if_match_fails: True - - from: - # For each entry's `title` value... - title: - # Perform this regex match on it to act as a filter. - # This will only download videos with "[Official Video]" in it. Note that we - # double backslash to make YAML happy - match: - - '\\[Official Video\\]' - - # For each entry's `description` value... - description: - # Match with capture groups and defaults. - # This tries to scrape a date from the description and produce new - # source variables - match: - - '([0-9]{4})-([0-9]{2})-([0-9]{2})' - # Exclude any entry where the description contains #short - exclude: - - '#short' - - # Each capture group creates these new source variables, respectively, - # as well a sanitized version, i.e. `captured_upload_year_sanitized` - capture_group_names: - - "captured_upload_year" - - "captured_upload_month" - - "captured_upload_day" - - # And if the string does not match, use these as respective default - # values for the new source variables. - capture_group_defaults: - - "{upload_year}" - - "{upload_month}" - - "{upload_day}" - -``enable`` - -:expected type: Optional[OverridesFormatter] -:description: - Can typically be left undefined to always default to enable. For preset convenience, - this field can be set using an override variable to easily toggle whether this plugin - is enabled or not via Boolean. - - -``skip_if_match_fails`` - -:expected type: Optional[Boolean] -:description: - Defaults to True. If True, when any match fails and has no defaults, the entry will be - skipped. If False, ytdl-sub will error and all downloads will not proceed. - - ---------------------------------------------------------------------------------------------------- split_by_chapters diff --git a/docs/source/deprecation_notices.rst b/docs/source/deprecation_notices.rst index 29e3c5b0..fc238128 100644 --- a/docs/source/deprecation_notices.rst +++ b/docs/source/deprecation_notices.rst @@ -1,6 +1,45 @@ Deprecation Notices =================== +Sep 2024 +-------- + +regex plugin +~~~~~~~~~~~~ +Regex plugin has been removed in favor of scripting. The function +:ref:`config_reference/scripting/scripting_functions:regex_capture_many` +has been created to replicate the plugin's behavior. See the following converted example: + +.. code-block:: yaml + :caption: regex plugin + + regex: + from: + title: + match: + - ".*? - (.*)" # Captures 'Some - Song' from 'Emily Hopkins - Some - Song' + capture_group_names: + - "captured_track_title" + capture_group_defaults: + - "{title}" + overrides: + track_title: "{captured_track_title}" + +.. code-block:: yaml + :caption: scripting + + overrides: + # Captures 'Some - Song' from 'Emily Hopkins - Some - Song' + captured_track_title: >- + { + %regex_capture_many( + title, + [ ".*? - (.*)" ], + [ title ] + ) + } + track_title: "{%array_at(captured_track_title, 1)}" + Oct 2023 -------- diff --git a/src/ytdl_sub/config/plugin/plugin_mapping.py b/src/ytdl_sub/config/plugin/plugin_mapping.py index dadc4a20..316096b2 100644 --- a/src/ytdl_sub/config/plugin/plugin_mapping.py +++ b/src/ytdl_sub/config/plugin/plugin_mapping.py @@ -23,7 +23,6 @@ from ytdl_sub.plugins.match_filters import MatchFiltersPlugin from ytdl_sub.plugins.music_tags import MusicTagsPlugin from ytdl_sub.plugins.nfo_tags import NfoTagsPlugin from ytdl_sub.plugins.output_directory_nfo_tags import OutputDirectoryNfoTagsPlugin -from ytdl_sub.plugins.regex import RegexPlugin from ytdl_sub.plugins.split_by_chapters import SplitByChaptersPlugin from ytdl_sub.plugins.subtitles import SubtitlesPlugin from ytdl_sub.plugins.throttle_protection import ThrottleProtectionPlugin @@ -47,7 +46,6 @@ class PluginMapping: "video_tags": VideoTagsPlugin, "nfo_tags": NfoTagsPlugin, "output_directory_nfo_tags": OutputDirectoryNfoTagsPlugin, - "regex": RegexPlugin, "subtitles": SubtitlesPlugin, "chapters": ChaptersPlugin, "split_by_chapters": SplitByChaptersPlugin, @@ -74,7 +72,6 @@ class PluginMapping: SplitByChaptersPlugin, FilterExcludePlugin, FilterIncludePlugin, - RegexPlugin, # add all others ] diff --git a/src/ytdl_sub/plugins/regex.py b/src/ytdl_sub/plugins/regex.py deleted file mode 100644 index 8b2bbfc4..00000000 --- a/src/ytdl_sub/plugins/regex.py +++ /dev/null @@ -1,456 +0,0 @@ -from collections import defaultdict -from typing import Any -from typing import Dict -from typing import List -from typing import Optional -from typing import Set - -from ytdl_sub.config.overrides import Overrides -from ytdl_sub.config.plugin.plugin import Plugin -from ytdl_sub.config.plugin.plugin_operation import PluginOperation -from ytdl_sub.config.validators.options import ToggleableOptionsDictValidator -from ytdl_sub.entries.entry import Entry -from ytdl_sub.script.parser import parse -from ytdl_sub.script.utils.exceptions import RuntimeException -from ytdl_sub.utils.exceptions import RegexNoMatchException -from ytdl_sub.utils.logger import Logger -from ytdl_sub.validators.regex_validator import RegexListValidator -from ytdl_sub.validators.source_variable_validator import SourceVariableNameListValidator -from ytdl_sub.validators.strict_dict_validator import StrictDictValidator -from ytdl_sub.validators.string_formatter_validators import ListFormatterValidator -from ytdl_sub.validators.string_formatter_validators import StringFormatterValidator -from ytdl_sub.validators.validators import BoolValidator -from ytdl_sub.validators.validators import DictValidator -from ytdl_sub.ytdl_additions.enhanced_download_archive import EnhancedDownloadArchive - -logger = Logger.get(name="regex") - - -class VariableRegex(StrictDictValidator): - - _optional_keys = {"match", "exclude", "capture_group_defaults", "capture_group_names"} - - def __init__(self, name, value): - super().__init__(name, value) - self._match = self._validate_key_if_present(key="match", validator=RegexListValidator) - self._exclude = self._validate_key_if_present(key="exclude", validator=RegexListValidator) - self._capture_group_defaults = self._validate_key_if_present( - key="capture_group_defaults", validator=ListFormatterValidator - ) - self._capture_group_names = self._validate_key_if_present( - key="capture_group_names", validator=SourceVariableNameListValidator, default=[] - ) - - if self._match is None and self._exclude is None: - raise self._validation_exception("must specify either `match` or `exclude`") - - if self._match is None and (self._capture_group_defaults or self._capture_group_names.list): - raise self._validation_exception( - "capture group parameters requires at least one `match` to be specified" - ) - - # If defaults are to be used, ensure there are the same number of defaults as there are - # capture groups - if self._capture_group_defaults is not None and self._match.num_capture_groups != len( - self._capture_group_defaults.list - ): - raise self._validation_exception( - f"number of defaults must match number of capture groups, " - f"{len(self._capture_group_defaults.list)} != {self._match.num_capture_groups}" - ) - - # If there are capture groups, ensure there are capture group names - if self._match and (len(self._capture_group_names.list) != self._match.num_capture_groups): - raise self._validation_exception( - f"number of capture group names must match number of capture groups, " - f"{len(self._capture_group_names.list)} != {self._match.num_capture_groups}" - ) - - @property - def match(self) -> Optional[RegexListValidator]: - """ - List of regex strings to try to match against a source or override variable. Each regex - string must have the same number of capture groups. - """ - return self._match - - @property - def exclude(self) -> Optional[RegexListValidator]: - """ - List of regex strings to try to match against a source or override variable. If one of the - regex strings match, then the entry will be skipped. If both ``exclude`` and ``match`` are - specified, entries will get skipped if the regex matches against both ``exclude`` and - ``match``. - """ - return self._exclude - - @property - def capture_group_names(self) -> Optional[List[str]]: - """ - Optional (only when no capture groups are in the regex string). List of names to store the - capture group values to. These and ``_sanitized`` versions will be available to use as - source variables. The list's length must be equal to the number of match capture groups. - """ - return [validator.value for validator in self._capture_group_names.list] - - @property - def capture_group_defaults(self) -> Optional[List[StringFormatterValidator]]: - """ - Optional. List of string format validators to use for capture group defaults if a - source variable cannot be matched. The list's length must be equal to the number of match - capture groups. - """ - return self._capture_group_defaults.list if self.has_defaults else None - - @property - def has_defaults(self) -> bool: - """ - Returns - ------- - True if a validation exception should be raised if not captured. False otherwise. - """ - return self._capture_group_defaults is not None - - -class FromSourceVariablesRegex(DictValidator): - def __init__(self, name, value): - super().__init__(name, value) - self.variable_capture_dict: Dict[str, VariableRegex] = { - key: self._validate_key(key=key, validator=VariableRegex) for key in self._keys - } - - -class RegexOptions(ToggleableOptionsDictValidator): - r""" - .. attention:: - - This plugin will eventually be deprecated and replaced by scripting functions. - You can replicate the example below using the following. - - .. code-block:: yaml - - # Only includes videos with 'Official Video' - filter_include: - - >- - { %contains( %lower(title), "official video" ) } - - # Excludes videos with '#short' in its description - filter_exclude: - - >- - { %contains( %lower(description), '#short' ) } - - # Creates a capture array with defaults, and assigns - # each capture group to its own variable - overrides: - description_date_capture: >- - { - %regex_capture_many( - description, - [ "([0-9]{4})-([0-9]{2})-([0-9]{2})" ], - [ upload_year, upload_month, upload_day ] - ) - } - captured_upload_year: >- - { %array_at(description_date_capture, 1) } - captured_upload_month: >- - { %array_at(description_date_capture, 2) } - captured_upload_day: >- - { %array_at(description_date_capture, 3) } - - Performs regex matching on an entry's source or override variables. Regex can be used to filter - entries from proceeding with download or capture groups to create new source variables. - - NOTE that YAML differentiates between single-quote (``'``) and double-quote (``"``), which can - affect regex. Double-quote implies string literals, i.e. ``"\n"`` is the literal chars ``\n``, - whereas single-quote, ``'\n'`` gets evaluated to a new line. To escape ``\`` when using - single-quote, use ``\\``. This is necessary if you want your regex to be something like - ``\d\n`` to match a number and adjacent new-line. It must be written as ``\\d\n``. - - If you want to regex-search multiple source variables to create a logical OR effect, you can - create an override variable that contains the concatenation of them, and search that with regex. - For example, creating the override variable ``"title_and_description": "{title} {description}"`` - and using ``title_and_description`` can regex match/exclude from either ``title`` or - ``description``. - - :Usage: - - .. code-block:: yaml - - regex: - # By default, if any match fails and has no defaults, the entry will - # be skipped. If False, ytdl-sub will error and stop all downloads - # from proceeding. - skip_if_match_fails: True - - from: - # For each entry's `title` value... - title: - # Perform this regex match on it to act as a filter. - # This will only download videos with "[Official Video]" in it. Note that we - # double backslash to make YAML happy - match: - - '\\[Official Video\\]' - - # For each entry's `description` value... - description: - # Match with capture groups and defaults. - # This tries to scrape a date from the description and produce new - # source variables - match: - - '([0-9]{4})-([0-9]{2})-([0-9]{2})' - # Exclude any entry where the description contains #short - exclude: - - '#short' - - # Each capture group creates these new source variables, respectively, - # as well a sanitized version, i.e. `captured_upload_year_sanitized` - capture_group_names: - - "captured_upload_year" - - "captured_upload_month" - - "captured_upload_day" - - # And if the string does not match, use these as respective default - # values for the new source variables. - capture_group_defaults: - - "{upload_year}" - - "{upload_month}" - - "{upload_day}" - """ - - _required_keys = {"from"} - _optional_keys = {"enable", "skip_if_match_fails"} - - @classmethod - def partial_validate(cls, name: str, value: Any) -> None: - """ - Partially validate regex - """ - if isinstance(value, dict): - value["from"] = value.get("from", {}) - _ = cls(name, value) - - def __init__(self, name, value): - super().__init__(name, value) - self._from = self._validate_key(key="from", validator=FromSourceVariablesRegex) - self._skip_if_match_fails: bool = self._validate_key_if_present( - key="skip_if_match_fails", validator=BoolValidator, default=True - ).value - - # Variables added by the regex plugin - self._added_variable_names: Set[str] = set() - - @property - def skip_if_match_fails(self) -> Optional[bool]: - """ - :expected type: Optional[Boolean] - :description: - Defaults to True. If True, when any match fails and has no defaults, the entry will be - skipped. If False, ytdl-sub will error and all downloads will not proceed. - """ - return self._skip_if_match_fails - - @property - def source_variable_capture_dict(self) -> Dict[str, VariableRegex]: - """ - Returns - ------- - Dict of { source variable: capture options } - """ - return self._from.variable_capture_dict - - @classmethod - def _can_resolve( - cls, unresolved_variables: Set[str], input_variable_name: str, regex_options: VariableRegex - ) -> bool: - if input_variable_name in unresolved_variables: - return False - for capture_group_default in regex_options.capture_group_defaults or []: - parsed_default = parse(capture_group_default.format_string) - if parsed_default.variables and parsed_default.variables.issubset(unresolved_variables): - return False - return True - - def added_variables( - self, - resolved_variables: Set[str], - unresolved_variables: Set[str], - plugin_op: PluginOperation, - ) -> Dict[PluginOperation, Set[str]]: - """ - Returns - ------- - List of new source variables created via regex capture - """ - added_source_vars: Dict[PluginOperation, Set[str]] = { - PluginOperation.MODIFY_ENTRY_METADATA: set(), - PluginOperation.MODIFY_ENTRY: set(), - } - for input_variable_name, regex_options in self.source_variable_capture_dict.items(): - variables_to_add = set(regex_options.capture_group_names) - - if plugin_op != PluginOperation.ANY and input_variable_name not in ( - resolved_variables | unresolved_variables - ): - raise self._validation_exception( - f"cannot regex capture '{input_variable_name}' because it is not a" - " defined variable." - ) - if ( - plugin_op.value >= PluginOperation.MODIFY_ENTRY.value - and input_variable_name in unresolved_variables - ): - raise self._validation_exception( - f"cannot regex capture '{input_variable_name}' because it is not " - f"computed until later in execution." - ) - - if plugin_op == PluginOperation.ANY: - added_source_vars[PluginOperation.MODIFY_ENTRY_METADATA] |= variables_to_add - self._added_variable_names |= variables_to_add - continue - - if not self._can_resolve( - unresolved_variables=unresolved_variables, - input_variable_name=input_variable_name, - regex_options=regex_options, - ): - continue - - for capture_group_name in regex_options.capture_group_names: - if capture_group_name in (resolved_variables - self._added_variable_names): - raise self._validation_exception( - f"cannot use '{capture_group_name}' as a capture group name because it is " - f"an already defined variable." - ) - added_source_vars[plugin_op] |= set(regex_options.capture_group_names) - - return added_source_vars - - -class RegexPlugin(Plugin[RegexOptions]): - plugin_options_type = RegexOptions - - def __init__( - self, - options: RegexOptions, - overrides: Overrides, - enhanced_download_archive: EnhancedDownloadArchive, - ): - super().__init__( - options=options, - overrides=overrides, - enhanced_download_archive=enhanced_download_archive, - ) - # Lookup of entry id to processed regex variables - self._processed_regex_vars: Dict[str, Set[str]] = defaultdict(set) - - def _try_skip_entry(self, entry: Entry, variable_name: str) -> None: - # Skip the entry if toggled - if self.plugin_options.skip_if_match_fails: - logger.info( - "Regex failed to match '%s' from '%s', skipping.", - variable_name, - entry.title, - ) - return None - - # Otherwise, error - raise RegexNoMatchException(f"Regex failed to match '{variable_name}' from '{entry.title}'") - - @classmethod - def _can_process_at_metadata_stage(cls, entry: Entry, variable_name: str) -> bool: - # Try to see if it can resolve - try: - _ = entry.script.get(variable_name) - return True - # If it can not from missing variables (from post-metadata stage), return False - except RuntimeException: - return False - - def _modify_entry_metadata(self, entry: Entry, is_metadata_stage: bool) -> Optional[Entry]: - """ - Parameters - ---------- - entry - Entry to add source variables to - is_metadata_stage - Whether this is called at the metadata stage or modify stage - - Returns - ------- - Entry with regex capture variables added to its source variables - - Raises - ------ - ValidationException - If no capture and no defaults - """ - # Iterate each source var to capture and add to the entry - for ( - variable_name, - regex_options, - ) in self.plugin_options.source_variable_capture_dict.items(): - - # Record which regex source variables are processed, to - # process as many variables as possible in the metadata stage, then the rest - # after the media file has been downloaded. - if variable_name in self._processed_regex_vars[entry.ytdl_uid()]: - continue - - # If it's the metadata stage, and it can't be processed, skip until post-metadata - if is_metadata_stage and not self._can_process_at_metadata_stage( - entry=entry, variable_name=variable_name - ): - continue - - self._processed_regex_vars[entry.ytdl_uid()].add(variable_name) - regex_input_str = str(entry.script.get(variable_name)) - - if ( - regex_options.exclude is not None - and regex_options.exclude.match_any(input_str=regex_input_str) is not None - ): - return self._try_skip_entry(entry=entry, variable_name=variable_name) - - # If match is present - if regex_options.match is not None: - maybe_capture = regex_options.match.match_any(input_str=regex_input_str) - - # And nothing matched - if maybe_capture is None: - # and no defaults - if not regex_options.has_defaults: - return self._try_skip_entry(entry=entry, variable_name=variable_name) - - # add both the default... - entry.add( - { - regex_options.capture_group_names[i]: self.overrides.apply_formatter( - formatter=default, entry=entry - ) - for i, default in enumerate(regex_options.capture_group_defaults) - } - ) - # There is a capture, add the source variables to the entry as - # {source_var}_capture_1, {source_var}_capture_2, ... - else: - entry.add( - { - regex_options.capture_group_names[i]: capture - for i, capture in enumerate(maybe_capture) - } - ) - - return entry - - def modify_entry_metadata(self, entry: Entry) -> Optional[Entry]: - """ - Perform regex at the metadata stage - """ - return self._modify_entry_metadata(entry, is_metadata_stage=True) - - def modify_entry(self, entry: Entry) -> Optional[Entry]: - """ - Perform regex at the metadata stage - """ - return self._modify_entry_metadata(entry, is_metadata_stage=False) diff --git a/src/ytdl_sub/prebuilt_presets/music/albums_from_playlists.yaml b/src/ytdl_sub/prebuilt_presets/music/albums_from_playlists.yaml index ab0955a0..d5f2cdad 100644 --- a/src/ytdl_sub/prebuilt_presets/music/albums_from_playlists.yaml +++ b/src/ytdl_sub/prebuilt_presets/music/albums_from_playlists.yaml @@ -23,14 +23,14 @@ presets: "Bandcamp": preset: - "_albums_from_playlists" - regex: - from: - title: - match: - - ".*? - (.*)" # Captures 'Some - Song' from 'Emily Hopkins - Some - Song' - capture_group_names: - - "captured_track_title" - capture_group_defaults: - - "{title}" overrides: - track_title: "{captured_track_title}" \ No newline at end of file + # Captures 'Some - Song' from 'Emily Hopkins - Some - Song' + captured_track_title: >- + { + %regex_capture_many( + title, + [ ".*? - (.*)" ], + [ title ] + ) + } + track_title: "{%array_at(captured_track_title, 1)}" diff --git a/tests/e2e/plugins/test_regex.py b/tests/e2e/plugins/test_regex.py deleted file mode 100644 index 63b4be53..00000000 --- a/tests/e2e/plugins/test_regex.py +++ /dev/null @@ -1,402 +0,0 @@ -import copy -import re -from typing import Any -from typing import Dict - -import mergedeep -import pytest -from expected_transaction_log import assert_transaction_log_matches - -from ytdl_sub.config.config_file import ConfigFile -from ytdl_sub.subscriptions.subscription import Subscription -from ytdl_sub.utils.exceptions import RegexNoMatchException -from ytdl_sub.utils.exceptions import ValidationException - - -@pytest.fixture -def regex_subscription_dict_base(output_directory): - return { - "preset": "Jellyfin Music Videos", - # override the output directory with our fixture-generated dir - "output_options": {"output_directory": output_directory}, - "format": "best[height<=480]", # download the worst format so it is fast - "regex": { - # tests that skip_if_match_fails defaults to True - "from": { - "description": { - "match": [".*http:\\/\\/(.+).com.*"], - "capture_group_names": ["description_website"], - }, - "upload_date_standardized": { - "match": ["([0-9]+)-([0-9]+)-27"], - "capture_group_names": [ - "upload_captured_year", - "upload_captured_month", - ], - "capture_group_defaults": [ - "First", - "Second containing {in_regex_default}", - ], - }, - }, - }, - "overrides": { - "in_regex_default": "in regex default", - "url": "https://youtube.com/playlist?list=PL5BC0FC26BECA5A35", - }, - } - - -@pytest.fixture -def regex_subscription_dict(regex_subscription_dict_base, output_directory): - return mergedeep.merge( - regex_subscription_dict_base, - { - "regex": { - # tests that skip_if_match_fails defaults to True - "from": { - "title": { - "match": [ - "should not cap (.+) - (.+)", - ".*\\[(.+) - (Feb.+)]", # should filter out march video - ], - "capture_group_names": ["title_type", "title_date"], - }, - }, - }, - "nfo_tags": { - "tags": { - "title_cap_1": "{title_type}", - "title_cap_1_sanitized": "{title_type_sanitized}", - "title_cap_2": "{title_date}", - "desc_cap": "{description_website}", - "upload_date_both_caps": "{upload_captured_year} and {upload_captured_month}", - "override_with_capture_variable": "{contains_regex_default}", - "override_with_capture_variable_sanitized": "{contains_regex_sanitized_default}", - } - }, - "overrides": { - "title_capture_list": f"""{{ - %regex_capture_many_with_defaults( - title, - [ - "should not cap (.+) - (.+)", - ".*\\[(.+) - (Feb.+)]" - ], - [ - "ack", - "ack" - ] - ) - }}""", - "title_capture_list_1": "{%array_at(title_capture_list, 1)}", - "title_capture_list_2": "{%array_at(title_capture_list, 2)}", - "contains_regex_default": "contains {title_type}", - "contains_regex_sanitized_default": "contains {title_type_sanitized}", - }, - }, - ) - - -@pytest.fixture -def regex_subscription_dict_exclude(regex_subscription_dict_base, output_directory): - return mergedeep.merge( - regex_subscription_dict_base, - { - "regex": { - # tests that skip_if_match_fails defaults to True - "from": { - "title": { - "exclude": [ - "should not cap", - ".*Feb.*", # should filter out march video - ], - }, - }, - }, - }, - ) - - -@pytest.fixture -def regex_subscription_dict_match_and_exclude(regex_subscription_dict_base, output_directory): - return mergedeep.merge( - regex_subscription_dict_base, - { - "regex": { - # tests that skip_if_match_fails defaults to True - "from": { - "title": { - "match": [ - "should not cap (.+) - (.+)", - ".*\\[(.+) - (Feb.+)]", # should filter out march video - ], - "capture_group_names": ["title_type", "title_date"], - "exclude": [ - "should not cap", - ".*27.*", # should filter out Feb 27th video - ], - }, - }, - }, - "nfo_tags": { - "tags": { - "title_cap_1": "{title_type}", - "title_cap_1_sanitized": "{title_type_sanitized}", - "title_cap_2": "{title_date}", - "desc_cap": "{description_website}", - "upload_date_both_caps": "{upload_captured_year} and {upload_captured_month}", - "override_with_capture_variable": "{contains_regex_default}", - "override_with_capture_variable_sanitized": "{contains_regex_sanitized_default}", - } - }, - "overrides": { - "contains_regex_default": "contains {title_type}", - "contains_regex_sanitized_default": "contains {title_type_sanitized}", - }, - }, - ) - - -@pytest.fixture -def regex_subscription_dict_match_and_exclude_override_variable( - regex_subscription_dict_base, output_directory -): - return mergedeep.merge( - regex_subscription_dict_base, - { - "regex": { - # tests that skip_if_match_fails defaults to True - "from": { - "override_title": { - "exclude": [ - "should not cap", - ".*Feb.*", # should filter out march video - ], - }, - "override_description": { - "match": [".*http:\\/\\/(.+).com.*"], - "capture_group_names": ["override_description_website"], - }, - }, - }, - "overrides": {"override_title": "{title}", "override_description": "{description}"}, - }, - ) - - -@pytest.fixture -def playlist_subscription(default_config, regex_subscription_dict): - return Subscription.from_dict( - config=default_config, - preset_name="regex_capture_playlist_test", - preset_dict=regex_subscription_dict, - ) - - -@pytest.fixture -def playlist_subscription_no_match_fails( - default_config: ConfigFile, regex_subscription_dict: Dict[str, Any] -): - regex_subscription_dict["regex"]["skip_if_match_fails"] = False - - return Subscription.from_dict( - config=default_config, - preset_name="regex_capture_playlist_test", - preset_dict=regex_subscription_dict, - ) - - -@pytest.fixture -def playlist_subscription_exclude( - default_config: ConfigFile, regex_subscription_dict_exclude: Dict[str, Any] -) -> Subscription: - return Subscription.from_dict( - config=default_config, - preset_name="regex_exclude_playlist_test", - preset_dict=regex_subscription_dict_exclude, - ) - - -@pytest.fixture -def playlist_subscription_overrides( - default_config: ConfigFile, - regex_subscription_dict_match_and_exclude_override_variable: Dict[str, Any], -) -> Subscription: - return Subscription.from_dict( - config=default_config, - preset_name="regex_using_overrides_test", - preset_dict=regex_subscription_dict_match_and_exclude_override_variable, - ) - - -@pytest.fixture -def playlist_subscription_match_and_exclude( - default_config: ConfigFile, regex_subscription_dict_match_and_exclude: Dict[str, Any] -) -> Subscription: - return Subscription.from_dict( - config=default_config, - preset_name="regex_match_and_exclude_playlist_test", - preset_dict=regex_subscription_dict_match_and_exclude, - ) - - -class TestRegex: - def test_regex_success(self, playlist_subscription, output_directory): - # Only dry run is needed to see if capture variables are created - transaction_log = playlist_subscription.download(dry_run=True) - assert_transaction_log_matches( - output_directory=output_directory, - transaction_log=transaction_log, - transaction_log_summary_file_name="plugins/test_regex.txt", - ) - - def test_regex_excludes_success(self, playlist_subscription_exclude, output_directory): - # Should only contain the march video - transaction_log = playlist_subscription_exclude.download(dry_run=True) - assert_transaction_log_matches( - output_directory=output_directory, - transaction_log=transaction_log, - transaction_log_summary_file_name="plugins/test_regex_exclude.txt", - ) - - def test_regex_match_and_excludes_success( - self, playlist_subscription_match_and_exclude, output_directory - ): - # Should only contain the Feb 1st video - transaction_log = playlist_subscription_match_and_exclude.download(dry_run=True) - assert_transaction_log_matches( - output_directory=output_directory, - transaction_log=transaction_log, - transaction_log_summary_file_name="plugins/test_regex_match_and_exclude.txt", - ) - - def test_regex_using_overrides_success(self, playlist_subscription_overrides, output_directory): - # Should only contain the march video - transaction_log = playlist_subscription_overrides.download(dry_run=True) - assert_transaction_log_matches( - output_directory=output_directory, - transaction_log=transaction_log, - transaction_log_summary_file_name="plugins/test_regex_overrides.txt", - ) - - def test_regex_fails_no_match(self, playlist_subscription_no_match_fails, output_directory): - with pytest.raises( - RegexNoMatchException, - match=re.escape( - "Regex failed to match 'title' from 'Jesse's Minecraft Server [Trailer - Mar.21]'" - ), - ): - _ = playlist_subscription_no_match_fails.download(dry_run=True) - - def test_regex_fails_capture_group_with_only_excludes( - self, regex_subscription_dict_exclude, default_config - ): - regex_subscription_dict_exclude["regex"]["from"]["title"]["capture_group_names"] = ["uid"] - with pytest.raises( - ValidationException, - match=re.escape( - "capture group parameters requires at least one `match` to be specified" - ), - ): - _ = Subscription.from_dict( - config=default_config, - preset_name="test_regex_fails_capture_group_is_source_variable", - preset_dict=regex_subscription_dict_exclude, - ) - - def test_regex_fails_no_match_or_exclude(self, regex_subscription_dict, default_config): - del regex_subscription_dict["regex"]["from"]["title"]["match"] - with pytest.raises( - ValidationException, - match=re.escape("must specify either `match` or `exclude`"), - ): - _ = Subscription.from_dict( - config=default_config, - preset_name="test_regex_fails_capture_group_is_source_variable", - preset_dict=regex_subscription_dict, - ) - - def test_regex_fails_capture_group_is_entry_variable( - self, regex_subscription_dict, default_config - ): - regex_subscription_dict["regex"]["from"]["playlist_uid"] = { - "match": [".*http:\\/\\/(.+).com.*"], - "capture_group_names": ["uid"], - } - - with pytest.raises( - ValidationException, - match=re.escape( - "Cannot use the variable name uid because it exists as a built-in " - "ytdl-sub variable name." - ), - ): - _ = Subscription.from_dict( - config=default_config, - preset_name="test_regex_fails_capture_group_is_entry_variable", - preset_dict=regex_subscription_dict, - ) - - def test_regex_fails_capture_group_is_override_variable( - self, regex_subscription_dict, default_config - ): - regex_subscription_dict["regex"]["from"]["playlist_uid"] = { - "match": [".*http:\\/\\/(.+).com.*"], - "capture_group_names": ["contains_regex_default"], - } - - with pytest.raises( - ValidationException, - match=re.escape( - "Override variable with name contains_regex_default cannot be used since it is " - "added by a plugin." - ), - ): - _ = Subscription.from_dict( - config=default_config, - preset_name="test_regex_fails_capture_group_is_override_variable", - preset_dict=regex_subscription_dict, - ) - - def test_regex_fails_source_variable_does_not_exist( - self, regex_subscription_dict, default_config - ): - regex_subscription_dict["regex"]["from"]["dne"] = copy.deepcopy( - regex_subscription_dict["regex"]["from"]["title"] - ) - with pytest.raises( - ValidationException, - match=re.escape("cannot regex capture 'dne' because it is not a defined variable"), - ): - _ = Subscription.from_dict( - config=default_config, - preset_name="test_regex_fails_source_variable_does_not_exist", - preset_dict=regex_subscription_dict, - ) - - def test_regex_fails_unequal_defaults(self, regex_subscription_dict, default_config): - regex_subscription_dict["regex"]["from"]["title"]["capture_group_defaults"] = ["1 != 2"] - with pytest.raises( - ValidationException, - match=re.escape("number of defaults must match number of capture groups, 1 != 2"), - ): - _ = Subscription.from_dict( - config=default_config, - preset_name="test_regex_fails_unequal_defaults", - preset_dict=regex_subscription_dict, - ) - - def test_regex_fails_unequal_capture_group_names(self, regex_subscription_dict, default_config): - regex_subscription_dict["regex"]["from"]["title"]["capture_group_names"].append("unequal") - with pytest.raises( - ValidationException, - match=re.escape( - "number of capture group names must match number of capture groups, 3 != 2" - ), - ): - _ = Subscription.from_dict( - config=default_config, - preset_name="test_regex_fails_unequal_capture_group_names", - preset_dict=regex_subscription_dict, - ) diff --git a/tests/e2e/plugins/test_split_by_chapters.py b/tests/e2e/plugins/test_split_by_chapters.py index cb02715e..bfa06823 100644 --- a/tests/e2e/plugins/test_split_by_chapters.py +++ b/tests/e2e/plugins/test_split_by_chapters.py @@ -30,35 +30,27 @@ def yt_album_as_chapters_with_regex_preset_dict(yt_album_as_chapters_preset_dict mergedeep.merge( yt_album_as_chapters_preset_dict, { - "regex": { - "from": { - # Ensure regex can handle override variables that come from the - # post-metadata stage - "chapter_title": { - "match": r"\d+\. (.+)", - "capture_group_names": "captured_track_title", - "capture_group_defaults": "{chapter_title}", - }, - "title": { - "match": [ - "(.+) - (.+) [-[\\(\\{].+", - "(.+) - (.+)", - ], - "capture_group_names": [ - "captured_track_artist", - "captured_track_album", - ], - "capture_group_defaults": [ - "{channel}", - "{title}", - ], - }, - } - }, "overrides": { - "track_title": "{captured_track_title}", - "track_album": "{captured_track_album}", - "track_artist": "{captured_track_artist}", + "title_capture": """{ + %regex_capture_many( + title, + [ + "(.+) - (.+) [-[\\(\\{].+", + "(.+) - (.+)" + ], + [ channel, title ] + ) + }""", + "chapter_title_capture": r"""{ + %regex_capture_many( + chapter_title, + [ "\d+\. (.+)" ], + [ chapter_title ] + ) + }""", + "track_title": "{%array_at(chapter_title_capture, 1)}", + "track_artist": "{%array_at(title_capture, 1)}", + "track_album": "{%array_at(title_capture, 2)}", }, }, )