From 308c76a8f2c6d29e13a3dd9a7d150ee4b8d72fab Mon Sep 17 00:00:00 2001 From: Jesse Bannon Date: Wed, 20 Dec 2023 09:52:12 -0800 Subject: [PATCH] [FEATURE] Filter plugins, regex helper script functions (#848) Adds generic `filter_include` and `filter_exclude` plugins in anticipation for function script usage. Will detail it more with official docs at a later time! --- src/ytdl_sub/config/plugin/plugin_mapping.py | 8 + .../config/validators/variable_validation.py | 16 + .../entries/script/function_scripts.py | 91 ++++-- src/ytdl_sub/plugins/filter_exclude.py | 70 ++++ src/ytdl_sub/plugins/filter_include.py | 79 +++++ .../script/functions/array_functions.py | 51 +++ .../script/functions/boolean_functions.py | 8 + .../script/functions/error_functions.py | 36 +++ .../script/functions/regex_functions.py | 8 + .../script/functions/string_functions.py | 8 + src/ytdl_sub/script/types/resolvable.py | 4 + tests/e2e/plugins/test_filter.py | 299 ++++++++++++++++++ tests/e2e/plugins/test_regex.py | 15 + tests/unit/config/test_config_file.py | 2 + tests/unit/config/test_preset.py | 1 - .../script/functions/test_array_functions.py | 47 +++ .../functions/test_boolean_functions.py | 14 + .../functions/test_conditional_functions.py | 10 +- .../script/functions/test_error_functions.py | 80 +++++ .../script/functions/test_string_functions.py | 7 + 20 files changed, 827 insertions(+), 27 deletions(-) create mode 100644 src/ytdl_sub/plugins/filter_exclude.py create mode 100644 src/ytdl_sub/plugins/filter_include.py create mode 100644 tests/e2e/plugins/test_filter.py diff --git a/src/ytdl_sub/config/plugin/plugin_mapping.py b/src/ytdl_sub/config/plugin/plugin_mapping.py index 2918c746..dadc4a20 100644 --- a/src/ytdl_sub/config/plugin/plugin_mapping.py +++ b/src/ytdl_sub/config/plugin/plugin_mapping.py @@ -15,6 +15,8 @@ from ytdl_sub.plugins.chapters import ChaptersPlugin from ytdl_sub.plugins.date_range import DateRangePlugin from ytdl_sub.plugins.embed_thumbnail import EmbedThumbnailPlugin from ytdl_sub.plugins.file_convert import FileConvertPlugin +from ytdl_sub.plugins.filter_exclude import FilterExcludePlugin +from ytdl_sub.plugins.filter_include import FilterIncludePlugin from ytdl_sub.plugins.format import FormatPlugin from ytdl_sub.plugins.internal.view import ViewPlugin from ytdl_sub.plugins.match_filters import MatchFiltersPlugin @@ -50,6 +52,8 @@ class PluginMapping: "chapters": ChaptersPlugin, "split_by_chapters": SplitByChaptersPlugin, "throttle_protection": ThrottleProtectionPlugin, + "filter_include": FilterIncludePlugin, + "filter_exclude": FilterExcludePlugin, } # All other plugins are added after the defined ordered ones @@ -57,6 +61,8 @@ class PluginMapping: ThrottleProtectionPlugin, UrlDownloaderCollectionVariablePlugin, SubtitlesPlugin, + FilterExcludePlugin, + FilterIncludePlugin, # add all others ] @@ -66,6 +72,8 @@ class PluginMapping: FileConvertPlugin, ChaptersPlugin, SplitByChaptersPlugin, + FilterExcludePlugin, + FilterIncludePlugin, RegexPlugin, # add all others ] diff --git a/src/ytdl_sub/config/validators/variable_validation.py b/src/ytdl_sub/config/validators/variable_validation.py index a0aa180d..d52489a7 100644 --- a/src/ytdl_sub/config/validators/variable_validation.py +++ b/src/ytdl_sub/config/validators/variable_validation.py @@ -27,6 +27,18 @@ def _add_dummy_variables(variables: Iterable[str]) -> Dict[str, str]: return dummy_variables +def _add_dummy_overrides(overrides: Overrides) -> Dict[str, str]: + # Have the dummy override variable contain all variable deps that it uses in the string + dummy_overrides: Dict[str, str] = {} + for override_name in _override_variables(overrides): + dummy_overrides[override_name] = "" + # pylint: disable=protected-access + for variable_dependency in overrides.script._variables[override_name].variables: + dummy_overrides[override_name] += f"{{ {variable_dependency.name } }}" + # pylint: enable=protected-access + return dummy_overrides + + def _get_added_and_modified_variables( plugins: PresetPlugins, downloader_options: MultiUrlValidator ) -> Iterable[Tuple[OptionsValidator, Set[str], Set[str]]]: @@ -119,6 +131,10 @@ class VariableValidation: # copy the script and mock entry variables self.script = copy.deepcopy(overrides.script).add(_add_dummy_variables(entry_variables)) + self.script.add( + variables=_add_dummy_overrides(overrides=overrides), + unresolvable=self.unresolved_variables, + ) return self diff --git a/src/ytdl_sub/entries/script/function_scripts.py b/src/ytdl_sub/entries/script/function_scripts.py index 85622187..68019a00 100644 --- a/src/ytdl_sub/entries/script/function_scripts.py +++ b/src/ytdl_sub/entries/script/function_scripts.py @@ -6,32 +6,81 @@ from ytdl_sub.entries.script.variable_definitions import VariableDefinitions v: VariableDefinitions = VARIABLES CUSTOM_FUNCTION_SCRIPTS: Dict[str, str] = { - "%extract_field_from_metadata_array_getter": """{ - %map_get( %map(%array_at($0, 0)), %array_at($0, 1) ) - }""", - "%extract_field_from_metadata_array": """{ - %if( - %bool($0), - %array_extend( - %array_apply( - %array_product( - %array($0), - [ %string($1) ] - ), - %extract_field_from_metadata_array_getter - ) - ), - [] - ) - }""", + ############################################################################################# + # SIBLING GETTER "%extract_field_from_siblings": f"""{{ %if( %bool({v.sibling_metadata.variable_name}), - %extract_field_from_metadata_array( + %array_apply_fixed( {v.sibling_metadata.variable_name}, - $0 - ), + %string($0), + %map_get + ) [] ) }}""", + ############################################################################################# + # REGEX PLUGIN + # $0 - input variable + # $1 - regex array + # $2 - defaults + # $3 - error message + "%regex_capture_inner": """{ + %assert_then( + %array_reduce( + %array_apply_fixed( + %array_apply( + $1, + %regex_capture_groups + ), + %array_size($2), + %lte + ), + %and + ), + %array_overlay( + %array( + %array_first( + %array_apply_fixed( + %array($1), + %string($0), + %regex_search + ), + [] + ) + ), + %array_extend( ['using all defaults'], $2 ), + True + ), + $3 + ) + }""", + "%regex_capture_many_required": """{ + %assert_ne( + %regex_capture_inner( + $0, $1, ['', '', '', '', '', '', '', '', '', ''], + 'When using %regex_capture_many, number of regex capture groups must be less than or equal to the number of defaults' + ), + ['using all defaults', '', '', '', '', '', '', '', '', '', ''], + 'When running %regex_capture_many_required, no regex strings were captured' + ) + }""", + "%regex_capture_many_with_defaults": """{ + %regex_capture_inner( + $0, $1, $2, + 'When using %regex_capture_with_defaults, number of regex capture groups must be less than or equal to the number of defaults' + ) + }""", + "%regex_search_any": """{ + %ne( + %array_at( + %regex_capture_inner( + $0, $1, [], + 'When using %regex_search_many, all regex strings must contain no capture groups' + ), + 0 + ), + 'using all defaults' + ) + }""", } diff --git a/src/ytdl_sub/plugins/filter_exclude.py b/src/ytdl_sub/plugins/filter_exclude.py new file mode 100644 index 00000000..cceb4700 --- /dev/null +++ b/src/ytdl_sub/plugins/filter_exclude.py @@ -0,0 +1,70 @@ +import json +from typing import Dict +from typing import Optional + +from ytdl_sub.config.overrides import Overrides +from ytdl_sub.config.plugin.plugin import Plugin +from ytdl_sub.config.validators.options import OptionsValidator +from ytdl_sub.entries.entry import Entry +from ytdl_sub.utils.exceptions import StringFormattingException +from ytdl_sub.utils.logger import Logger +from ytdl_sub.validators.string_formatter_validators import ListFormatterValidator +from ytdl_sub.ytdl_additions.enhanced_download_archive import EnhancedDownloadArchive + +logger = Logger.get("conditional") + + +class FilterExcludeOptions(ListFormatterValidator, OptionsValidator): + """ + Applies a conditional OR on any number of filters comprised of either variables or scripts. + If any filter evaluates to True, the entry will be excluded. + + Usage: + + .. code-block:: yaml + + presets: + my_example_preset: + filter_exclude: + - { %contains( %lower(title), '#short' ) } + - { %contains( %lower(description), '#short' ) } + """ + + +class FilterExcludePlugin(Plugin[FilterExcludeOptions]): + plugin_options_type = FilterExcludeOptions + + def __init__( + self, + options: FilterExcludeOptions, + overrides: Overrides, + enhanced_download_archive: EnhancedDownloadArchive, + ): + super().__init__( + options=options, + overrides=overrides, + enhanced_download_archive=enhanced_download_archive, + ) + self._evaluated_map: Dict[str, bool] = {} + + def modify_entry(self, entry: Entry) -> Optional[Entry]: + # Already evaluated in modify_entry_metadata, do not recompute + if entry.ytdl_uid() in self._evaluated_map: + return entry + + for formatter in self.plugin_options.list: + out = json.loads(self.overrides.apply_formatter(formatter=formatter, entry=entry)) + if bool(out): + return None + + return entry + + def modify_entry_metadata(self, entry: Entry) -> Optional[Entry]: + try: + output_entry = self.modify_entry(entry=entry) + except StringFormattingException: + # If filtering fails at the metadata stage, try again w/no catch in modify_entry + return entry + + self._evaluated_map[entry.ytdl_uid()] = True + return output_entry diff --git a/src/ytdl_sub/plugins/filter_include.py b/src/ytdl_sub/plugins/filter_include.py new file mode 100644 index 00000000..988426fe --- /dev/null +++ b/src/ytdl_sub/plugins/filter_include.py @@ -0,0 +1,79 @@ +import json +from typing import Dict +from typing import Optional + +from ytdl_sub.config.overrides import Overrides +from ytdl_sub.config.plugin.plugin import Plugin +from ytdl_sub.config.validators.options import OptionsValidator +from ytdl_sub.entries.entry import Entry +from ytdl_sub.utils.exceptions import StringFormattingException +from ytdl_sub.utils.logger import Logger +from ytdl_sub.validators.string_formatter_validators import ListFormatterValidator +from ytdl_sub.ytdl_additions.enhanced_download_archive import EnhancedDownloadArchive + +logger = Logger.get("conditional") + + +class FilterIncludeOptions(ListFormatterValidator, OptionsValidator): + """ + Applies a conditional AND on any number of filters comprised of either variables or scripts. + If all filters evaluate to True, the entry will be included. + + Usage: + + .. code-block:: yaml + + presets: + my_example_preset: + filter_include: + - {description} + - >- + { + %regex_search_any( + title, + [ + "Full Episode", + "FULL", + ] + ) + } + """ + + +class FilterIncludePlugin(Plugin[FilterIncludeOptions]): + plugin_options_type = FilterIncludeOptions + + def __init__( + self, + options: FilterIncludeOptions, + overrides: Overrides, + enhanced_download_archive: EnhancedDownloadArchive, + ): + super().__init__( + options=options, + overrides=overrides, + enhanced_download_archive=enhanced_download_archive, + ) + self._evaluated_map: Dict[str, bool] = {} + + def modify_entry(self, entry: Entry) -> Optional[Entry]: + # Already evaluated in modify_entry_metadata, do not recompute + if entry.ytdl_uid() in self._evaluated_map: + return entry + + for formatter in self.plugin_options.list: + out = json.loads(self.overrides.apply_formatter(formatter=formatter, entry=entry)) + if not bool(out): + return None + + return entry + + def modify_entry_metadata(self, entry: Entry) -> Optional[Entry]: + try: + output_entry = self.modify_entry(entry=entry) + except StringFormattingException: + # If filtering fails at the metadata stage, try again w/no catch in modify_entry + return entry + + self._evaluated_map[entry.ytdl_uid()] = True + return output_entry diff --git a/src/ytdl_sub/script/functions/array_functions.py b/src/ytdl_sub/script/functions/array_functions.py index ffff0da7..5dc0e653 100644 --- a/src/ytdl_sub/script/functions/array_functions.py +++ b/src/ytdl_sub/script/functions/array_functions.py @@ -50,6 +50,29 @@ class ArrayFunctions: return Array(output) + @staticmethod + def array_overlay( + array: Array, overlap: Array, only_missing: Optional[Boolean] = None + ) -> Array: + """ + Overlaps ``overlap`` onto ``array``. Can optionally only overlay missing indices. + """ + output: List[Resolvable] = [] + output.extend(array.value) + + overlap_only_missing = only_missing and only_missing.value + + for idx, overlap_value in enumerate(overlap.value): + if overlap_only_missing and idx < len(array.value): + continue + + if idx < len(array.value): + output[idx] = overlap_value + else: + output.append(overlap_value) + + return Array(output) + @staticmethod def array_at(array: Array, idx: Integer) -> AnyArgument: """ @@ -57,6 +80,18 @@ class ArrayFunctions: """ return array.value[idx.value] + @staticmethod + def array_first(array: Array, fallback: AnyArgument) -> AnyArgument: + """ + Returns the first element whose boolean conversion is True. Returns fallback + if all elements evaluate to False. + """ + for val in array.value: + if bool(val.value): + return val + + return fallback + @staticmethod def array_contains(array: Array, value: AnyArgument) -> Boolean: """ @@ -130,6 +165,22 @@ class ArrayFunctions: """ return Array([Array([val]) for val in array.value]) + @staticmethod + def array_apply_fixed( + array: Array, + fixed_argument: AnyArgument, + lambda2_function: LambdaTwo, + reverse_args: Optional[Boolean] = None, + ) -> Array: + """ + Apply a lambda function on every element in the Array, with ``fixed_argument`` + passed as a second argument to every invocation. + """ + if reverse_args and reverse_args.value: + return Array([Array([fixed_argument, val]) for val in array.value]) + + return Array([Array([val, fixed_argument]) for val in array.value]) + @staticmethod def array_enumerate(array: Array, lambda_function: LambdaTwo) -> Array: """ diff --git a/src/ytdl_sub/script/functions/boolean_functions.py b/src/ytdl_sub/script/functions/boolean_functions.py index 0b45a547..f1bb9a35 100644 --- a/src/ytdl_sub/script/functions/boolean_functions.py +++ b/src/ytdl_sub/script/functions/boolean_functions.py @@ -1,5 +1,6 @@ from ytdl_sub.script.types.resolvable import AnyArgument from ytdl_sub.script.types.resolvable import Boolean +from ytdl_sub.script.types.resolvable import String # pylint: disable=invalid-name @@ -87,3 +88,10 @@ class BooleanFunctions: ``not`` operator. Returns the opposite of value. """ return Boolean(not value.value) + + @staticmethod + def is_null(value: AnyArgument) -> Boolean: + """ + Returns True if a value is null (i.e. an empty string). False otherwise. + """ + return Boolean(isinstance(value, String) and value.value == "") diff --git a/src/ytdl_sub/script/functions/error_functions.py b/src/ytdl_sub/script/functions/error_functions.py index 7eb1420f..517c6141 100644 --- a/src/ytdl_sub/script/functions/error_functions.py +++ b/src/ytdl_sub/script/functions/error_functions.py @@ -21,3 +21,39 @@ class ErrorFunctions: if not bool(value.value): raise UserThrownRuntimeError(assert_message) return value + + @staticmethod + def assert_then( + value: AnyArgument, ret: ReturnableArgument, assert_message: String + ) -> ReturnableArgument: + """ + Explicitly throw an error with the provided assert message if ``value`` evaluates to False. + If it evaluates to True, it will return ``ret``. + """ + if not bool(value.value): + raise UserThrownRuntimeError(assert_message) + return ret + + @staticmethod + def assert_eq( + value: ReturnableArgument, equals: AnyArgument, assert_message: String + ) -> ReturnableArgument: + """ + Explicitly throw an error with the provided assert message if ``value`` does not equal + ``equals``. If they do equal, then return ``value``. + """ + if not value.value == equals.value: + raise UserThrownRuntimeError(assert_message) + return value + + @staticmethod + def assert_ne( + value: ReturnableArgument, equals: AnyArgument, assert_message: String + ) -> ReturnableArgument: + """ + Explicitly throw an error with the provided assert message if ``value`` equals + ``equals``. If they do equal, then return ``value``. + """ + if value.value == equals.value: + raise UserThrownRuntimeError(assert_message) + return value diff --git a/src/ytdl_sub/script/functions/regex_functions.py b/src/ytdl_sub/script/functions/regex_functions.py index 2abca1ec..e38b3139 100644 --- a/src/ytdl_sub/script/functions/regex_functions.py +++ b/src/ytdl_sub/script/functions/regex_functions.py @@ -3,6 +3,7 @@ from typing import AnyStr from typing import Match from ytdl_sub.script.types.array import Array +from ytdl_sub.script.types.resolvable import Integer from ytdl_sub.script.types.resolvable import String @@ -40,3 +41,10 @@ class RegexFunctions: group as a subsequent element in the Array. """ return _re_output_to_array(re.fullmatch(regex.value, string.value)) + + @staticmethod + def regex_capture_groups(regex: String) -> Integer: + """ + Returns number of capture groups in regex + """ + return Integer(re.compile(regex.value).groups) diff --git a/src/ytdl_sub/script/functions/string_functions.py b/src/ytdl_sub/script/functions/string_functions.py index be3b52c2..807a5dd4 100644 --- a/src/ytdl_sub/script/functions/string_functions.py +++ b/src/ytdl_sub/script/functions/string_functions.py @@ -1,6 +1,7 @@ from typing import Optional from ytdl_sub.script.types.resolvable import AnyArgument +from ytdl_sub.script.types.resolvable import Boolean from ytdl_sub.script.types.resolvable import Integer from ytdl_sub.script.types.resolvable import Numeric from ytdl_sub.script.types.resolvable import String @@ -14,6 +15,13 @@ class StringFunctions: """ return String(value=str(value.value)) + @staticmethod + def contains(string: String, contains: String) -> Boolean: + """ + Returns True if ``contains`` is in ``string``. False otherwise. + """ + return Boolean(contains.value in string.value) + @staticmethod def slice(string: String, start: Integer, end: Optional[Integer] = None) -> String: """ diff --git a/src/ytdl_sub/script/types/resolvable.py b/src/ytdl_sub/script/types/resolvable.py index 829c0e78..c042b1f5 100644 --- a/src/ytdl_sub/script/types/resolvable.py +++ b/src/ytdl_sub/script/types/resolvable.py @@ -170,6 +170,10 @@ class Boolean(ResolvableT[bool], Argument): Resolved bool type """ + def __str__(self): + # makes it JSON friendly + return str(self.value).lower() + @dataclass(frozen=True) class String(ResolvableT[str], Argument): diff --git a/tests/e2e/plugins/test_filter.py b/tests/e2e/plugins/test_filter.py new file mode 100644 index 00000000..487c3269 --- /dev/null +++ b/tests/e2e/plugins/test_filter.py @@ -0,0 +1,299 @@ +import re +from typing import Any +from typing import Dict + +import mergedeep +import pytest +from expected_transaction_log import assert_transaction_log_matches + +from ytdl_sub.config.config_file import ConfigFile +from ytdl_sub.script.utils.exceptions import UserThrownRuntimeError +from ytdl_sub.subscriptions.subscription import Subscription + + +@pytest.fixture +def regex_subscription_dict_base(output_directory): + return { + "preset": "Jellyfin Music Videos", + # override the output directory with our fixture-generated dir + "output_options": {"output_directory": output_directory}, + "format": "best[height<=480]", # download the worst format so it is fast + "filter_include": ["{%not(%is_null(description_website))}"], + "overrides": { + "in_regex_default": "in regex default", + "url": "https://youtube.com/playlist?list=PL5BC0FC26BECA5A35", + "upload_capture": """{ + %regex_capture_many_with_defaults( + upload_date_standardized, + ["([0-9]+)-([0-9]+)-27"], + ["First", %concat("Second containing ", in_regex_default)] + ) + }""", + "upload_captured_year": "{%array_at(upload_capture, 1)}", + "upload_captured_month": "{%array_at(upload_capture, 2)}", + "description_capture": """{ + %regex_capture_many_with_defaults( + description, + [".*http:\\/\\/(.+).com.*"], + [null] + ) + }""", + "description_website": "{%array_at(description_capture, 1)}", + }, + } + + +@pytest.fixture +def regex_subscription_dict(regex_subscription_dict_base, output_directory): + return mergedeep.merge( + regex_subscription_dict_base, + { + "nfo_tags": { + "tags": { + "title_cap_1": "{title_type}", + "title_cap_1_sanitized": "{title_type_sanitized}", + "title_cap_2": "{title_date}", + "desc_cap": "{description_website}", + "upload_date_both_caps": "{upload_captured_year} and {upload_captured_month}", + "override_with_capture_variable": "{contains_regex_default}", + "override_with_capture_variable_sanitized": "{contains_regex_sanitized_default}", + } + }, + "filter_exclude": ["{%is_null(title_type)}"], + "overrides": { + "title_capture_list": """{ + %regex_capture_many_with_defaults( + title, + [ "should not cap (.+) - (.+)", ".*\\[(.+) - (Feb.+)]" ], + [ null, null ] + ) + }""", + "title_type": "{%array_at(title_capture_list, 1)}", + "title_date": "{%array_at(title_capture_list, 2)}", + "contains_regex_default": "contains {title_type}", + "contains_regex_sanitized_default": "contains {title_type_sanitized}", + }, + }, + ) + + +@pytest.fixture +def regex_subscription_dict_exclude(regex_subscription_dict_base, output_directory): + return mergedeep.merge( + regex_subscription_dict_base, + { + "filter_exclude": [ + """{ + %regex_search_any( + title, + [ "should not cap", ".*Feb.*" ] + ) + }""" + ] + }, + ) + + +@pytest.fixture +def regex_subscription_dict_match_and_exclude(regex_subscription_dict_base, output_directory): + return mergedeep.merge( + regex_subscription_dict_base, + { + "regex": { + # tests that skip_if_match_fails defaults to True + "from": { + "title": { + "match": [ + "should not cap (.+) - (.+)", + ".*\\[(.+) - (Feb.+)]", # should filter out march video + ], + "capture_group_names": ["title_type", "title_date"], + "exclude": [ + "should not cap", + ".*27.*", # should filter out Feb 27th video + ], + }, + }, + }, + "nfo_tags": { + "tags": { + "title_cap_1": "{title_type}", + "title_cap_1_sanitized": "{title_type_sanitized}", + "title_cap_2": "{title_date}", + "desc_cap": "{description_website}", + "upload_date_both_caps": "{upload_captured_year} and {upload_captured_month}", + "override_with_capture_variable": "{contains_regex_default}", + "override_with_capture_variable_sanitized": "{contains_regex_sanitized_default}", + } + }, + "overrides": { + "contains_regex_default": "contains {title_type}", + "contains_regex_sanitized_default": "contains {title_type_sanitized}", + }, + }, + ) + + +@pytest.fixture +def regex_subscription_dict_match_and_exclude_override_variable( + regex_subscription_dict_base, output_directory +): + return mergedeep.merge( + regex_subscription_dict_base, + { + "regex": { + # tests that skip_if_match_fails defaults to True + "from": { + "override_title": { + "exclude": [ + "should not cap", + ".*Feb.*", # should filter out march video + ], + }, + "override_description": { + "match": [".*http:\\/\\/(.+).com.*"], + "capture_group_names": ["override_description_website"], + }, + }, + }, + "overrides": {"override_title": "{title}", "override_description": "{description}"}, + }, + ) + + +@pytest.fixture +def playlist_subscription(default_config, regex_subscription_dict): + return Subscription.from_dict( + config=default_config, + preset_name="regex_capture_playlist_test", + preset_dict=regex_subscription_dict, + ) + + +@pytest.fixture +def playlist_subscription_no_match_fails( + default_config: ConfigFile, regex_subscription_dict: Dict[str, Any] +): + regex_subscription_dict["overrides"][ + "title_capture_list" + ] = """{ + %regex_capture_many_required( + title, + [ "should not cap (.+) - (.+)", ".*\\[(.+) - (Feb.+)]" ] + ) + }""" + + return Subscription.from_dict( + config=default_config, + preset_name="regex_capture_playlist_test", + preset_dict=regex_subscription_dict, + ) + + +@pytest.fixture +def playlist_subscription_exclude( + default_config: ConfigFile, regex_subscription_dict_exclude: Dict[str, Any] +) -> Subscription: + return Subscription.from_dict( + config=default_config, + preset_name="regex_exclude_playlist_test", + preset_dict=regex_subscription_dict_exclude, + ) + + +@pytest.fixture +def playlist_subscription_overrides( + default_config: ConfigFile, + regex_subscription_dict_match_and_exclude_override_variable: Dict[str, Any], +) -> Subscription: + return Subscription.from_dict( + config=default_config, + preset_name="regex_using_overrides_test", + preset_dict=regex_subscription_dict_match_and_exclude_override_variable, + ) + + +@pytest.fixture +def playlist_subscription_match_and_exclude( + default_config: ConfigFile, regex_subscription_dict_match_and_exclude: Dict[str, Any] +) -> Subscription: + return Subscription.from_dict( + config=default_config, + preset_name="regex_match_and_exclude_playlist_test", + preset_dict=regex_subscription_dict_match_and_exclude, + ) + + +class TestFilter: + def test_regex_success(self, playlist_subscription, output_directory): + # Only dry run is needed to see if capture variables are created + transaction_log = playlist_subscription.download(dry_run=True) + assert_transaction_log_matches( + output_directory=output_directory, + transaction_log=transaction_log, + transaction_log_summary_file_name="plugins/test_regex.txt", + ) + + def test_regex_excludes_success(self, playlist_subscription_exclude, output_directory): + # Should only contain the march video + transaction_log = playlist_subscription_exclude.download(dry_run=True) + assert_transaction_log_matches( + output_directory=output_directory, + transaction_log=transaction_log, + transaction_log_summary_file_name="plugins/test_regex_exclude.txt", + ) + + def test_regex_match_and_excludes_success( + self, playlist_subscription_match_and_exclude, output_directory + ): + # Should only contain the Feb 1st video + transaction_log = playlist_subscription_match_and_exclude.download(dry_run=True) + assert_transaction_log_matches( + output_directory=output_directory, + transaction_log=transaction_log, + transaction_log_summary_file_name="plugins/test_regex_match_and_exclude.txt", + ) + + def test_regex_using_overrides_success(self, playlist_subscription_overrides, output_directory): + # Should only contain the march video + transaction_log = playlist_subscription_overrides.download(dry_run=True) + assert_transaction_log_matches( + output_directory=output_directory, + transaction_log=transaction_log, + transaction_log_summary_file_name="plugins/test_regex_overrides.txt", + ) + + def test_regex_fails_no_match(self, playlist_subscription_no_match_fails, output_directory): + with pytest.raises( + UserThrownRuntimeError, + match=re.escape( + "When running %regex_capture_many_required, no regex strings were captured" + ), + ): + _ = playlist_subscription_no_match_fails.download(dry_run=True) + + def test_regex_fails_unequal_defaults(self, regex_subscription_dict, default_config): + regex_subscription_dict["overrides"][ + "title_capture_list" + ] = """{ + %regex_capture_many_with_defaults( + title, + [ "should not cap (.+) - (.+)", ".*\\[(.+) - (Feb.+)]" ], + [ "one default, expects >= 2" ] + ) + }""" + + subscription = Subscription.from_dict( + config=default_config, + preset_name="regex_capture_playlist_test", + preset_dict=regex_subscription_dict, + ) + + with pytest.raises( + UserThrownRuntimeError, + match=re.escape( + "When using %regex_capture_with_defaults, number of regex capture groups must " + "be less than or equal to the number of defaults" + ), + ): + _ = subscription.download(dry_run=True) diff --git a/tests/e2e/plugins/test_regex.py b/tests/e2e/plugins/test_regex.py index 58abb325..63b4be53 100644 --- a/tests/e2e/plugins/test_regex.py +++ b/tests/e2e/plugins/test_regex.py @@ -76,6 +76,21 @@ def regex_subscription_dict(regex_subscription_dict_base, output_directory): } }, "overrides": { + "title_capture_list": f"""{{ + %regex_capture_many_with_defaults( + title, + [ + "should not cap (.+) - (.+)", + ".*\\[(.+) - (Feb.+)]" + ], + [ + "ack", + "ack" + ] + ) + }}""", + "title_capture_list_1": "{%array_at(title_capture_list, 1)}", + "title_capture_list_2": "{%array_at(title_capture_list, 2)}", "contains_regex_default": "contains {title_type}", "contains_regex_sanitized_default": "contains {title_type_sanitized}", }, diff --git a/tests/unit/config/test_config_file.py b/tests/unit/config/test_config_file.py index 56b97498..1baab94f 100644 --- a/tests/unit/config/test_config_file.py +++ b/tests/unit/config/test_config_file.py @@ -54,6 +54,8 @@ class TestConfigFilePartiallyValidatesPresets: excluded_plugins = [ "embed_thumbnail", # value is bool, not dict "format", # value is string, not dict + "filter_include", # is list + "filter_exclude", # is list ] if plugin not in excluded_plugins: self._partial_validate({plugin: {}}) diff --git a/tests/unit/config/test_preset.py b/tests/unit/config/test_preset.py index 20c64214..dff51a50 100644 --- a/tests/unit/config/test_preset.py +++ b/tests/unit/config/test_preset.py @@ -264,7 +264,6 @@ class TestPreset: "function_name", [ "%extract_field_from_siblings", - "%extract_field_from_metadata_array", "%sanitize", "%array", ], diff --git a/tests/unit/script/functions/test_array_functions.py b/tests/unit/script/functions/test_array_functions.py index 7ad81b5d..d9515d2d 100644 --- a/tests/unit/script/functions/test_array_functions.py +++ b/tests/unit/script/functions/test_array_functions.py @@ -96,3 +96,50 @@ class TestArrayFunctions: FunctionRuntimeException, match="Tried and failed to cast Integer as an Array" ): single_variable_output("{%array(1)}") + + def test_array_overlay(self): + output = single_variable_output("{%array_overlay([1, 2, 3], [4, 5])}") + assert output == [4, 5, 3] + + output = single_variable_output("{%array_overlay([1, 2, 3], [4, 5, 6, 7, 8])}") + assert output == [4, 5, 6, 7, 8] + + output = single_variable_output("{%array_overlay([1, 2, 3], [4, 5, 6, 7, 8], True)}") + assert output == [1, 2, 3, 7, 8] + + def test_array_first(self): + output = single_variable_output( + "{%array_first(['', false, null, [], {}, 0, 'hi', 'no'], 'fallback')}" + ) + assert output == "hi" + + output = single_variable_output("{%array_first(['', false, null, [], {}, 0], 'fallback')}") + assert output == "fallback" + + def test_array_apply_fixed(self): + output = ( + Script( + { + "map_test": "{ {'key1': 7, 'key2': 8, 'key3': 9} }", + "output": """{ + %array_apply_fixed( ['key1', 'key2', 'key3'], map_test, %map_get, True) + }""", + } + ) + .resolve(update=True) + .get("output") + .native + ) + assert output == [7, 8, 9] + + output = ( + Script( + { + "output": "{%array_apply_fixed( ['key1', 'key2', 'key3'], '3', %contains)}", + } + ) + .resolve(update=True) + .get("output") + .native + ) + assert output == [False, False, True] diff --git a/tests/unit/script/functions/test_boolean_functions.py b/tests/unit/script/functions/test_boolean_functions.py index aab6afdf..dfdd79c6 100644 --- a/tests/unit/script/functions/test_boolean_functions.py +++ b/tests/unit/script/functions/test_boolean_functions.py @@ -126,3 +126,17 @@ class TestBooleanFunctions: def test_not(self, value: str, expected_output: bool): output = single_variable_output(f"{{%not({value})}}") assert output == expected_output + + @pytest.mark.parametrize( + "value, expected_output", + [ + ("null", True), + ("''", True), + ("0", False), + ("{}", False), + ("'h'", False), + ], + ) + def test_is_null(self, value: str, expected_output: bool): + output = single_variable_output(f"{{%is_null({value})}}") + assert output == expected_output diff --git a/tests/unit/script/functions/test_conditional_functions.py b/tests/unit/script/functions/test_conditional_functions.py index 259c0a33..f7217915 100644 --- a/tests/unit/script/functions/test_conditional_functions.py +++ b/tests/unit/script/functions/test_conditional_functions.py @@ -1,6 +1,5 @@ import pytest - -from ytdl_sub.script.script import Script +from unit.script.conftest import single_variable_output class TestConditionalFunction: @@ -12,11 +11,12 @@ class TestConditionalFunction: ], ) def test_if_function(self, function_str: str, expected_output: bool): - output = Script({"output": function_str}).resolve(update=True).get("output").native + output = single_variable_output(function_str) assert output == expected_output def test_nested_if_function(self): - function_str = """{ + output = single_variable_output( + """{ %if( True, %if( @@ -31,5 +31,5 @@ class TestConditionalFunction: True ) }""" - output = Script({"output": function_str}).resolve(update=True).get("output").native + ) assert output == "winner" diff --git a/tests/unit/script/functions/test_error_functions.py b/tests/unit/script/functions/test_error_functions.py index 9f1092b4..be2d90c2 100644 --- a/tests/unit/script/functions/test_error_functions.py +++ b/tests/unit/script/functions/test_error_functions.py @@ -23,3 +23,83 @@ class TestErrorFunctions: def test_user_assert_passthrough_as_arg(self): output = single_variable_output("{%int(%assert('123', 'test this error message'))}") assert output == 123 + + def test_user_assert_eq(self): + output = single_variable_output( + """{ + %int( + %assert_eq( + '123', + %array_at(['123'], 0), + 'test this error message' + ) + ) + }""" + ) + assert output == 123 + + def test_user_assert_eq_raises(self): + with pytest.raises(UserThrownRuntimeError, match=re.escape("test this error message")): + single_variable_output( + """{ + %int( + %assert_eq( + '123', + %array_at(['no'], 0), + 'test this error message' + ) + ) + }""" + ) + + def test_user_assert_ne(self): + output = single_variable_output( + """{ + %int( + %assert_ne( + '123', + %array_at(['nope'], 0), + 'test this error message' + ) + ) + }""" + ) + assert output == 123 + + def test_user_assert_ne_raises(self): + with pytest.raises(UserThrownRuntimeError, match=re.escape("test this error message")): + single_variable_output( + """{ + %int( + %assert_ne( + '123', + %array_at(['123'], 0), + 'test this error message' + ) + ) + }""" + ) + + def test_user_assert_then(self): + output = single_variable_output( + """{ + %assert_then( + '123', + %array_at(['nope'], 0), + 'test this error message' + ) + }""" + ) + assert output == "nope" + + def test_user_assert_then_raises(self): + with pytest.raises(UserThrownRuntimeError, match=re.escape("test this error message")): + single_variable_output( + """{ + %assert_then( + {}, + %array_at(['nope'], 0), + 'test this error message' + ) + }""" + ) diff --git a/tests/unit/script/functions/test_string_functions.py b/tests/unit/script/functions/test_string_functions.py index 4533d5bf..9a48079a 100644 --- a/tests/unit/script/functions/test_string_functions.py +++ b/tests/unit/script/functions/test_string_functions.py @@ -107,3 +107,10 @@ class TestNumericFunctions: def test_slice(self, values, expected_output): output = single_variable_output(f"{{%slice({values})}}") assert output == expected_output + + @pytest.mark.parametrize( + "value, expected_output", [("a", True), ("nope", False), ("dog", True)] + ) + def test_contains(self, value, expected_output): + output = single_variable_output(f"{{%contains('a brown dog', '{value}')}}") + assert output == expected_output