diff --git a/src/ytdl_sub/config/plugin/plugin_mapping.py b/src/ytdl_sub/config/plugin/plugin_mapping.py index 2918c746..dadc4a20 100644 --- a/src/ytdl_sub/config/plugin/plugin_mapping.py +++ b/src/ytdl_sub/config/plugin/plugin_mapping.py @@ -15,6 +15,8 @@ from ytdl_sub.plugins.chapters import ChaptersPlugin from ytdl_sub.plugins.date_range import DateRangePlugin from ytdl_sub.plugins.embed_thumbnail import EmbedThumbnailPlugin from ytdl_sub.plugins.file_convert import FileConvertPlugin +from ytdl_sub.plugins.filter_exclude import FilterExcludePlugin +from ytdl_sub.plugins.filter_include import FilterIncludePlugin from ytdl_sub.plugins.format import FormatPlugin from ytdl_sub.plugins.internal.view import ViewPlugin from ytdl_sub.plugins.match_filters import MatchFiltersPlugin @@ -50,6 +52,8 @@ class PluginMapping: "chapters": ChaptersPlugin, "split_by_chapters": SplitByChaptersPlugin, "throttle_protection": ThrottleProtectionPlugin, + "filter_include": FilterIncludePlugin, + "filter_exclude": FilterExcludePlugin, } # All other plugins are added after the defined ordered ones @@ -57,6 +61,8 @@ class PluginMapping: ThrottleProtectionPlugin, UrlDownloaderCollectionVariablePlugin, SubtitlesPlugin, + FilterExcludePlugin, + FilterIncludePlugin, # add all others ] @@ -66,6 +72,8 @@ class PluginMapping: FileConvertPlugin, ChaptersPlugin, SplitByChaptersPlugin, + FilterExcludePlugin, + FilterIncludePlugin, RegexPlugin, # add all others ] diff --git a/src/ytdl_sub/config/validators/variable_validation.py b/src/ytdl_sub/config/validators/variable_validation.py index a0aa180d..d52489a7 100644 --- a/src/ytdl_sub/config/validators/variable_validation.py +++ b/src/ytdl_sub/config/validators/variable_validation.py @@ -27,6 +27,18 @@ def _add_dummy_variables(variables: Iterable[str]) -> Dict[str, str]: return dummy_variables +def _add_dummy_overrides(overrides: Overrides) -> Dict[str, str]: + # Have the dummy override variable contain all variable deps that it uses in the string + dummy_overrides: Dict[str, str] = {} + for override_name in _override_variables(overrides): + dummy_overrides[override_name] = "" + # pylint: disable=protected-access + for variable_dependency in overrides.script._variables[override_name].variables: + dummy_overrides[override_name] += f"{{ {variable_dependency.name } }}" + # pylint: enable=protected-access + return dummy_overrides + + def _get_added_and_modified_variables( plugins: PresetPlugins, downloader_options: MultiUrlValidator ) -> Iterable[Tuple[OptionsValidator, Set[str], Set[str]]]: @@ -119,6 +131,10 @@ class VariableValidation: # copy the script and mock entry variables self.script = copy.deepcopy(overrides.script).add(_add_dummy_variables(entry_variables)) + self.script.add( + variables=_add_dummy_overrides(overrides=overrides), + unresolvable=self.unresolved_variables, + ) return self diff --git a/src/ytdl_sub/entries/script/function_scripts.py b/src/ytdl_sub/entries/script/function_scripts.py index 3a111766..b0d82223 100644 --- a/src/ytdl_sub/entries/script/function_scripts.py +++ b/src/ytdl_sub/entries/script/function_scripts.py @@ -56,10 +56,15 @@ CUSTOM_FUNCTION_SCRIPTS: Dict[str, str] = { %and ), %array_overlay( - %array_apply_fixed( - %array($1), - %string($0), - %regex_search + %array( + %array_first( + %array_apply_fixed( + %array($1), + %string($0), + %regex_search + ), + [] + ) ), %array_extend( ['using all defaults'], $2 ), True @@ -74,7 +79,7 @@ CUSTOM_FUNCTION_SCRIPTS: Dict[str, str] = { 'When using %regex_capture_many, number of regex capture groups must be less than or equal to the number of defaults' ), ['using all defaults', '', '', '', '', '', '', '', '', '', ''], - 'When running %regex_capture_many_required, no regex strings captured' + 'When running %regex_capture_many_required, no regex strings were captured' ) }""", "%regex_capture_many_with_defaults": """{ diff --git a/src/ytdl_sub/plugins/filter_exclude.py b/src/ytdl_sub/plugins/filter_exclude.py new file mode 100644 index 00000000..cceb4700 --- /dev/null +++ b/src/ytdl_sub/plugins/filter_exclude.py @@ -0,0 +1,70 @@ +import json +from typing import Dict +from typing import Optional + +from ytdl_sub.config.overrides import Overrides +from ytdl_sub.config.plugin.plugin import Plugin +from ytdl_sub.config.validators.options import OptionsValidator +from ytdl_sub.entries.entry import Entry +from ytdl_sub.utils.exceptions import StringFormattingException +from ytdl_sub.utils.logger import Logger +from ytdl_sub.validators.string_formatter_validators import ListFormatterValidator +from ytdl_sub.ytdl_additions.enhanced_download_archive import EnhancedDownloadArchive + +logger = Logger.get("conditional") + + +class FilterExcludeOptions(ListFormatterValidator, OptionsValidator): + """ + Applies a conditional OR on any number of filters comprised of either variables or scripts. + If any filter evaluates to True, the entry will be excluded. + + Usage: + + .. code-block:: yaml + + presets: + my_example_preset: + filter_exclude: + - { %contains( %lower(title), '#short' ) } + - { %contains( %lower(description), '#short' ) } + """ + + +class FilterExcludePlugin(Plugin[FilterExcludeOptions]): + plugin_options_type = FilterExcludeOptions + + def __init__( + self, + options: FilterExcludeOptions, + overrides: Overrides, + enhanced_download_archive: EnhancedDownloadArchive, + ): + super().__init__( + options=options, + overrides=overrides, + enhanced_download_archive=enhanced_download_archive, + ) + self._evaluated_map: Dict[str, bool] = {} + + def modify_entry(self, entry: Entry) -> Optional[Entry]: + # Already evaluated in modify_entry_metadata, do not recompute + if entry.ytdl_uid() in self._evaluated_map: + return entry + + for formatter in self.plugin_options.list: + out = json.loads(self.overrides.apply_formatter(formatter=formatter, entry=entry)) + if bool(out): + return None + + return entry + + def modify_entry_metadata(self, entry: Entry) -> Optional[Entry]: + try: + output_entry = self.modify_entry(entry=entry) + except StringFormattingException: + # If filtering fails at the metadata stage, try again w/no catch in modify_entry + return entry + + self._evaluated_map[entry.ytdl_uid()] = True + return output_entry diff --git a/src/ytdl_sub/plugins/filter_include.py b/src/ytdl_sub/plugins/filter_include.py new file mode 100644 index 00000000..988426fe --- /dev/null +++ b/src/ytdl_sub/plugins/filter_include.py @@ -0,0 +1,79 @@ +import json +from typing import Dict +from typing import Optional + +from ytdl_sub.config.overrides import Overrides +from ytdl_sub.config.plugin.plugin import Plugin +from ytdl_sub.config.validators.options import OptionsValidator +from ytdl_sub.entries.entry import Entry +from ytdl_sub.utils.exceptions import StringFormattingException +from ytdl_sub.utils.logger import Logger +from ytdl_sub.validators.string_formatter_validators import ListFormatterValidator +from ytdl_sub.ytdl_additions.enhanced_download_archive import EnhancedDownloadArchive + +logger = Logger.get("conditional") + + +class FilterIncludeOptions(ListFormatterValidator, OptionsValidator): + """ + Applies a conditional AND on any number of filters comprised of either variables or scripts. + If all filters evaluate to True, the entry will be included. + + Usage: + + .. code-block:: yaml + + presets: + my_example_preset: + filter_include: + - {description} + - >- + { + %regex_search_any( + title, + [ + "Full Episode", + "FULL", + ] + ) + } + """ + + +class FilterIncludePlugin(Plugin[FilterIncludeOptions]): + plugin_options_type = FilterIncludeOptions + + def __init__( + self, + options: FilterIncludeOptions, + overrides: Overrides, + enhanced_download_archive: EnhancedDownloadArchive, + ): + super().__init__( + options=options, + overrides=overrides, + enhanced_download_archive=enhanced_download_archive, + ) + self._evaluated_map: Dict[str, bool] = {} + + def modify_entry(self, entry: Entry) -> Optional[Entry]: + # Already evaluated in modify_entry_metadata, do not recompute + if entry.ytdl_uid() in self._evaluated_map: + return entry + + for formatter in self.plugin_options.list: + out = json.loads(self.overrides.apply_formatter(formatter=formatter, entry=entry)) + if not bool(out): + return None + + return entry + + def modify_entry_metadata(self, entry: Entry) -> Optional[Entry]: + try: + output_entry = self.modify_entry(entry=entry) + except StringFormattingException: + # If filtering fails at the metadata stage, try again w/no catch in modify_entry + return entry + + self._evaluated_map[entry.ytdl_uid()] = True + return output_entry diff --git a/src/ytdl_sub/script/functions/boolean_functions.py b/src/ytdl_sub/script/functions/boolean_functions.py index 0b45a547..f1bb9a35 100644 --- a/src/ytdl_sub/script/functions/boolean_functions.py +++ b/src/ytdl_sub/script/functions/boolean_functions.py @@ -1,5 +1,6 @@ from ytdl_sub.script.types.resolvable import AnyArgument from ytdl_sub.script.types.resolvable import Boolean +from ytdl_sub.script.types.resolvable import String # pylint: disable=invalid-name @@ -87,3 +88,10 @@ class BooleanFunctions: ``not`` operator. Returns the opposite of value. """ return Boolean(not value.value) + + @staticmethod + def is_null(value: AnyArgument) -> Boolean: + """ + Returns True if a value is null (i.e. an empty string). False otherwise. + """ + return Boolean(isinstance(value, String) and value.value == "") diff --git a/src/ytdl_sub/script/functions/error_functions.py b/src/ytdl_sub/script/functions/error_functions.py index 5be2eb3b..517c6141 100644 --- a/src/ytdl_sub/script/functions/error_functions.py +++ b/src/ytdl_sub/script/functions/error_functions.py @@ -56,4 +56,4 @@ class ErrorFunctions: """ if value.value == equals.value: raise UserThrownRuntimeError(assert_message) - return value \ No newline at end of file + return value diff --git a/src/ytdl_sub/script/functions/string_functions.py b/src/ytdl_sub/script/functions/string_functions.py index be3b52c2..807a5dd4 100644 --- a/src/ytdl_sub/script/functions/string_functions.py +++ b/src/ytdl_sub/script/functions/string_functions.py @@ -1,6 +1,7 @@ from typing import Optional from ytdl_sub.script.types.resolvable import AnyArgument +from ytdl_sub.script.types.resolvable import Boolean from ytdl_sub.script.types.resolvable import Integer from ytdl_sub.script.types.resolvable import Numeric from ytdl_sub.script.types.resolvable import String @@ -14,6 +15,13 @@ class StringFunctions: """ return String(value=str(value.value)) + @staticmethod + def contains(string: String, contains: String) -> Boolean: + """ + Returns True if ``contains`` is in ``string``. False otherwise. + """ + return Boolean(contains.value in string.value) + @staticmethod def slice(string: String, start: Integer, end: Optional[Integer] = None) -> String: """ diff --git a/src/ytdl_sub/script/types/resolvable.py b/src/ytdl_sub/script/types/resolvable.py index 829c0e78..c042b1f5 100644 --- a/src/ytdl_sub/script/types/resolvable.py +++ b/src/ytdl_sub/script/types/resolvable.py @@ -170,6 +170,10 @@ class Boolean(ResolvableT[bool], Argument): Resolved bool type """ + def __str__(self): + # makes it JSON friendly + return str(self.value).lower() + @dataclass(frozen=True) class String(ResolvableT[str], Argument): diff --git a/tests/e2e/plugins/test_filter.py b/tests/e2e/plugins/test_filter.py new file mode 100644 index 00000000..e971c51b --- /dev/null +++ b/tests/e2e/plugins/test_filter.py @@ -0,0 +1,360 @@ +import copy +import re +from typing import Any +from typing import Dict + +import mergedeep +import pytest +from expected_transaction_log import assert_transaction_log_matches + +from ytdl_sub.config.config_file import ConfigFile +from ytdl_sub.script.utils.exceptions import UserThrownRuntimeError +from ytdl_sub.subscriptions.subscription import Subscription +from ytdl_sub.utils.exceptions import RegexNoMatchException +from ytdl_sub.utils.exceptions import ValidationException + + +@pytest.fixture +def regex_subscription_dict_base(output_directory): + return { + "preset": "Jellyfin Music Videos", + # override the output directory with our fixture-generated dir + "output_options": {"output_directory": output_directory}, + "format": "best[height<=480]", # download the worst format so it is fast + "filter_exclude": ["{%is_null(description_website)}"], + "overrides": { + "in_regex_default": "in regex default", + "url": "https://youtube.com/playlist?list=PL5BC0FC26BECA5A35", + "upload_capture": """{ + %regex_capture_many_with_defaults( + upload_date_standardized, + ["([0-9]+)-([0-9]+)-27"], + ["First", %concat("Second containing ", in_regex_default)] + ) + }""", + "upload_captured_year": "{%array_at(upload_capture, 1)}", + "upload_captured_month": "{%array_at(upload_capture, 2)}", + "description_capture": """{ + %regex_capture_many_with_defaults( + description, + [".*http:\\/\\/(.+).com.*"], + [null] + ) + }""", + "description_website": "{%array_at(description_capture, 1)}", + }, + } + + +@pytest.fixture +def regex_subscription_dict(regex_subscription_dict_base, output_directory): + return mergedeep.merge( + regex_subscription_dict_base, + { + "nfo_tags": { + "tags": { + "title_cap_1": "{title_type}", + "title_cap_1_sanitized": "{title_type_sanitized}", + "title_cap_2": "{title_date}", + "desc_cap": "{description_website}", + "upload_date_both_caps": "{upload_captured_year} and {upload_captured_month}", + "override_with_capture_variable": "{contains_regex_default}", + "override_with_capture_variable_sanitized": "{contains_regex_sanitized_default}", + } + }, + "filter_exclude": ["{%is_null(title_type)}"], + "overrides": { + "title_capture_list": """{ + %regex_capture_many_with_defaults( + title, + [ "should not cap (.+) - (.+)", ".*\\[(.+) - (Feb.+)]" ], + [ null, null ] + ) + }""", + "title_type": "{%array_at(title_capture_list, 1)}", + "title_date": "{%array_at(title_capture_list, 2)}", + "contains_regex_default": "contains {title_type}", + "contains_regex_sanitized_default": "contains {title_type_sanitized}", + }, + }, + ) + + +@pytest.fixture +def regex_subscription_dict_exclude(regex_subscription_dict_base, output_directory): + return mergedeep.merge( + regex_subscription_dict_base, + { + "filter_exclude": [ + """{ + %regex_search_any( + title, + [ "should not cap", ".*Feb.*" ] + ) + }""" + ] + }, + ) + + +@pytest.fixture +def regex_subscription_dict_match_and_exclude(regex_subscription_dict_base, output_directory): + return mergedeep.merge( + regex_subscription_dict_base, + { + "regex": { + # tests that skip_if_match_fails defaults to True + "from": { + "title": { + "match": [ + "should not cap (.+) - (.+)", + ".*\\[(.+) - (Feb.+)]", # should filter out march video + ], + "capture_group_names": ["title_type", "title_date"], + "exclude": [ + "should not cap", + ".*27.*", # should filter out Feb 27th video + ], + }, + }, + }, + "nfo_tags": { + "tags": { + "title_cap_1": "{title_type}", + "title_cap_1_sanitized": "{title_type_sanitized}", + "title_cap_2": "{title_date}", + "desc_cap": "{description_website}", + "upload_date_both_caps": "{upload_captured_year} and {upload_captured_month}", + "override_with_capture_variable": "{contains_regex_default}", + "override_with_capture_variable_sanitized": "{contains_regex_sanitized_default}", + } + }, + "overrides": { + "contains_regex_default": "contains {title_type}", + "contains_regex_sanitized_default": "contains {title_type_sanitized}", + }, + }, + ) + + +@pytest.fixture +def regex_subscription_dict_match_and_exclude_override_variable( + regex_subscription_dict_base, output_directory +): + return mergedeep.merge( + regex_subscription_dict_base, + { + "regex": { + # tests that skip_if_match_fails defaults to True + "from": { + "override_title": { + "exclude": [ + "should not cap", + ".*Feb.*", # should filter out march video + ], + }, + "override_description": { + "match": [".*http:\\/\\/(.+).com.*"], + "capture_group_names": ["override_description_website"], + }, + }, + }, + "overrides": {"override_title": "{title}", "override_description": "{description}"}, + }, + ) + + +@pytest.fixture +def playlist_subscription(default_config, regex_subscription_dict): + return Subscription.from_dict( + config=default_config, + preset_name="regex_capture_playlist_test", + preset_dict=regex_subscription_dict, + ) + + +@pytest.fixture +def playlist_subscription_no_match_fails( + default_config: ConfigFile, regex_subscription_dict: Dict[str, Any] +): + regex_subscription_dict["overrides"][ + "title_capture_list" + ] = """{ + %regex_capture_many_required( + title, + [ "should not cap (.+) - (.+)", ".*\\[(.+) - (Feb.+)]" ] + ) + }""" + + return Subscription.from_dict( + config=default_config, + preset_name="regex_capture_playlist_test", + preset_dict=regex_subscription_dict, + ) + + +@pytest.fixture +def playlist_subscription_exclude( + default_config: ConfigFile, regex_subscription_dict_exclude: Dict[str, Any] +) -> Subscription: + return Subscription.from_dict( + config=default_config, + preset_name="regex_exclude_playlist_test", + preset_dict=regex_subscription_dict_exclude, + ) + + +@pytest.fixture +def playlist_subscription_overrides( + default_config: ConfigFile, + regex_subscription_dict_match_and_exclude_override_variable: Dict[str, Any], +) -> Subscription: + return Subscription.from_dict( + config=default_config, + preset_name="regex_using_overrides_test", + preset_dict=regex_subscription_dict_match_and_exclude_override_variable, + ) + + +@pytest.fixture +def playlist_subscription_match_and_exclude( + default_config: ConfigFile, regex_subscription_dict_match_and_exclude: Dict[str, Any] +) -> Subscription: + return Subscription.from_dict( + config=default_config, + preset_name="regex_match_and_exclude_playlist_test", + preset_dict=regex_subscription_dict_match_and_exclude, + ) + + +class TestRegex: + def test_regex_success(self, playlist_subscription, output_directory): + # Only dry run is needed to see if capture variables are created + transaction_log = playlist_subscription.download(dry_run=True) + assert_transaction_log_matches( + output_directory=output_directory, + transaction_log=transaction_log, + transaction_log_summary_file_name="plugins/test_regex.txt", + ) + + def test_regex_excludes_success(self, playlist_subscription_exclude, output_directory): + # Should only contain the march video + transaction_log = playlist_subscription_exclude.download(dry_run=True) + assert_transaction_log_matches( + output_directory=output_directory, + transaction_log=transaction_log, + transaction_log_summary_file_name="plugins/test_regex_exclude.txt", + ) + + def test_regex_match_and_excludes_success( + self, playlist_subscription_match_and_exclude, output_directory + ): + # Should only contain the Feb 1st video + transaction_log = playlist_subscription_match_and_exclude.download(dry_run=True) + assert_transaction_log_matches( + output_directory=output_directory, + transaction_log=transaction_log, + transaction_log_summary_file_name="plugins/test_regex_match_and_exclude.txt", + ) + + def test_regex_using_overrides_success(self, playlist_subscription_overrides, output_directory): + # Should only contain the march video + transaction_log = playlist_subscription_overrides.download(dry_run=True) + assert_transaction_log_matches( + output_directory=output_directory, + transaction_log=transaction_log, + transaction_log_summary_file_name="plugins/test_regex_overrides.txt", + ) + + def test_regex_fails_no_match(self, playlist_subscription_no_match_fails, output_directory): + with pytest.raises( + UserThrownRuntimeError, + match=re.escape( + "When running %regex_capture_many_required, no regex strings were captured" + ), + ): + _ = playlist_subscription_no_match_fails.download(dry_run=True) + + def test_regex_fails_capture_group_is_entry_variable( + self, regex_subscription_dict, default_config + ): + regex_subscription_dict["regex"]["from"]["playlist_uid"] = { + "match": [".*http:\\/\\/(.+).com.*"], + "capture_group_names": ["uid"], + } + + with pytest.raises( + ValidationException, + match=re.escape( + "Cannot use the variable name uid because it exists as a built-in " + "ytdl-sub variable name." + ), + ): + _ = Subscription.from_dict( + config=default_config, + preset_name="test_regex_fails_capture_group_is_entry_variable", + preset_dict=regex_subscription_dict, + ) + + def test_regex_fails_capture_group_is_override_variable( + self, regex_subscription_dict, default_config + ): + regex_subscription_dict["regex"]["from"]["playlist_uid"] = { + "match": [".*http:\\/\\/(.+).com.*"], + "capture_group_names": ["contains_regex_default"], + } + + with pytest.raises( + ValidationException, + match=re.escape( + "Override variable with name contains_regex_default cannot be used since it is " + "added by a plugin." + ), + ): + _ = Subscription.from_dict( + config=default_config, + preset_name="test_regex_fails_capture_group_is_override_variable", + preset_dict=regex_subscription_dict, + ) + + def test_regex_fails_source_variable_does_not_exist( + self, regex_subscription_dict, default_config + ): + regex_subscription_dict["regex"]["from"]["dne"] = copy.deepcopy( + regex_subscription_dict["regex"]["from"]["title"] + ) + with pytest.raises( + ValidationException, + match=re.escape("cannot regex capture 'dne' because it is not a defined variable"), + ): + _ = Subscription.from_dict( + config=default_config, + preset_name="test_regex_fails_source_variable_does_not_exist", + preset_dict=regex_subscription_dict, + ) + + def test_regex_fails_unequal_defaults(self, regex_subscription_dict, default_config): + regex_subscription_dict["regex"]["from"]["title"]["capture_group_defaults"] = ["1 != 2"] + with pytest.raises( + ValidationException, + match=re.escape("number of defaults must match number of capture groups, 1 != 2"), + ): + _ = Subscription.from_dict( + config=default_config, + preset_name="test_regex_fails_unequal_defaults", + preset_dict=regex_subscription_dict, + ) + + def test_regex_fails_unequal_capture_group_names(self, regex_subscription_dict, default_config): + regex_subscription_dict["regex"]["from"]["title"]["capture_group_names"].append("unequal") + with pytest.raises( + ValidationException, + match=re.escape( + "number of capture group names must match number of capture groups, 3 != 2" + ), + ): + _ = Subscription.from_dict( + config=default_config, + preset_name="test_regex_fails_unequal_capture_group_names", + preset_dict=regex_subscription_dict, + )