[FEATURE] Filter plugins, regex helper script functions (#848)

Adds generic `filter_include` and `filter_exclude` plugins in anticipation for function script usage. Will detail it more with official docs at a later time!

src/ytdl_sub/config/plugin/plugin_mapping.py

@@ -15,6 +15,8 @@ from ytdl_sub.plugins.chapters import ChaptersPlugin
 from ytdl_sub.plugins.date_range import DateRangePlugin
 from ytdl_sub.plugins.embed_thumbnail import EmbedThumbnailPlugin
 from ytdl_sub.plugins.file_convert import FileConvertPlugin
+from ytdl_sub.plugins.filter_exclude import FilterExcludePlugin
+from ytdl_sub.plugins.filter_include import FilterIncludePlugin
 from ytdl_sub.plugins.format import FormatPlugin
 from ytdl_sub.plugins.internal.view import ViewPlugin
 from ytdl_sub.plugins.match_filters import MatchFiltersPlugin
@@ -50,6 +52,8 @@ class PluginMapping:
         "chapters": ChaptersPlugin,
         "split_by_chapters": SplitByChaptersPlugin,
         "throttle_protection": ThrottleProtectionPlugin,
+        "filter_include": FilterIncludePlugin,
+        "filter_exclude": FilterExcludePlugin,
     }
 
     # All other plugins are added after the defined ordered ones
@@ -57,6 +61,8 @@ class PluginMapping:
         ThrottleProtectionPlugin,
         UrlDownloaderCollectionVariablePlugin,
         SubtitlesPlugin,
+        FilterExcludePlugin,
+        FilterIncludePlugin,
         # add all others
     ]
 
@@ -66,6 +72,8 @@ class PluginMapping:
         FileConvertPlugin,
         ChaptersPlugin,
         SplitByChaptersPlugin,
+        FilterExcludePlugin,
+        FilterIncludePlugin,
         RegexPlugin,
         # add all others
     ]

src/ytdl_sub/config/validators/variable_validation.py

@@ -27,6 +27,18 @@ def _add_dummy_variables(variables: Iterable[str]) -> Dict[str, str]:
     return dummy_variables
 
 
+def _add_dummy_overrides(overrides: Overrides) -> Dict[str, str]:
+    # Have the dummy override variable contain all variable deps that it uses in the string
+    dummy_overrides: Dict[str, str] = {}
+    for override_name in _override_variables(overrides):
+        dummy_overrides[override_name] = ""
+        # pylint: disable=protected-access
+        for variable_dependency in overrides.script._variables[override_name].variables:
+            dummy_overrides[override_name] += f"{{ {variable_dependency.name } }}"
+        # pylint: enable=protected-access
+    return dummy_overrides
+
+
 def _get_added_and_modified_variables(
     plugins: PresetPlugins, downloader_options: MultiUrlValidator
 ) -> Iterable[Tuple[OptionsValidator, Set[str], Set[str]]]:
@@ -119,6 +131,10 @@ class VariableValidation:
 
         # copy the script and mock entry variables
         self.script = copy.deepcopy(overrides.script).add(_add_dummy_variables(entry_variables))
+        self.script.add(
+            variables=_add_dummy_overrides(overrides=overrides),
+            unresolvable=self.unresolved_variables,
+        )
 
         return self
 

src/ytdl_sub/entries/script/function_scripts.py

@@ -6,32 +6,81 @@ from ytdl_sub.entries.script.variable_definitions import VariableDefinitions
 v: VariableDefinitions = VARIABLES
 
 CUSTOM_FUNCTION_SCRIPTS: Dict[str, str] = {
-    "%extract_field_from_metadata_array_getter": """{
+    #############################################################################################
-        %map_get( %map(%array_at($0, 0)), %array_at($0, 1) )
+    # SIBLING GETTER
-    }""",
-    "%extract_field_from_metadata_array": """{
-        %if(
-            %bool($0),
-            %array_extend(
-                %array_apply(
-                    %array_product(
-                        %array($0),
-                        [ %string($1) ]
-                    ),
-                    %extract_field_from_metadata_array_getter
-                )
-            ),
-            []
-        )   
-    }""",
     "%extract_field_from_siblings": f"""{{
         %if(
             %bool({v.sibling_metadata.variable_name}),
-            %extract_field_from_metadata_array(
+            %array_apply_fixed(
                 {v.sibling_metadata.variable_name},
-                $0
+                %string($0),
-            ),
+                %map_get
+            )
             []
         )
     }}""",
+    #############################################################################################
+    # REGEX PLUGIN
+    #   $0 - input variable
+    #   $1 - regex array
+    #   $2 - defaults
+    #   $3 - error message
+    "%regex_capture_inner": """{
+        %assert_then(
+            %array_reduce(
+                %array_apply_fixed(
+                    %array_apply(
+                        $1,
+                        %regex_capture_groups
+                    ),
+                    %array_size($2),
+                    %lte
+                ),
+                %and
+            ),
+            %array_overlay(
+                %array(
+                    %array_first(
+                        %array_apply_fixed(
+                            %array($1),
+                            %string($0),
+                            %regex_search
+                        ),
+                        []
+                    )
+                ),
+                %array_extend( ['using all defaults'], $2 ),
+                True
+            ),
+            $3
+        )
+    }""",
+    "%regex_capture_many_required": """{
+        %assert_ne(
+            %regex_capture_inner(
+                $0, $1, ['', '', '', '', '', '', '', '', '', ''],
+                'When using %regex_capture_many, number of regex capture groups must be less than or equal to the number of defaults'
+            ),
+            ['using all defaults', '', '', '', '', '', '', '', '', '', ''],
+            'When running %regex_capture_many_required, no regex strings were captured'
+        )
+    }""",
+    "%regex_capture_many_with_defaults": """{
+        %regex_capture_inner(
+            $0, $1, $2,
+            'When using %regex_capture_with_defaults, number of regex capture groups must be less than or equal to the number of defaults'
+        )
+    }""",
+    "%regex_search_any": """{
+        %ne(
+            %array_at(
+                %regex_capture_inner(
+                    $0, $1, [],
+                    'When using %regex_search_many, all regex strings must contain no capture groups'
+                ),
+                0
+            ),
+            'using all defaults'
+        )
+    }""",
 }

src/ytdl_sub/plugins/filter_exclude.py

@@ -0,0 +1,70 @@
+import json
+from typing import Dict
+from typing import Optional
+
+from ytdl_sub.config.overrides import Overrides
+from ytdl_sub.config.plugin.plugin import Plugin
+from ytdl_sub.config.validators.options import OptionsValidator
+from ytdl_sub.entries.entry import Entry
+from ytdl_sub.utils.exceptions import StringFormattingException
+from ytdl_sub.utils.logger import Logger
+from ytdl_sub.validators.string_formatter_validators import ListFormatterValidator
+from ytdl_sub.ytdl_additions.enhanced_download_archive import EnhancedDownloadArchive
+
+logger = Logger.get("conditional")
+
+
+class FilterExcludeOptions(ListFormatterValidator, OptionsValidator):
+    """
+    Applies a conditional OR on any number of filters comprised of either variables or scripts.
+    If any filter evaluates to True, the entry will be excluded.
+
+    Usage:
+
+    .. code-block:: yaml
+
+       presets:
+         my_example_preset:
+           filter_exclude:
+             - { %contains( %lower(title), '#short' ) }
+             - { %contains( %lower(description), '#short' ) }
+    """
+
+
+class FilterExcludePlugin(Plugin[FilterExcludeOptions]):
+    plugin_options_type = FilterExcludeOptions
+
+    def __init__(
+        self,
+        options: FilterExcludeOptions,
+        overrides: Overrides,
+        enhanced_download_archive: EnhancedDownloadArchive,
+    ):
+        super().__init__(
+            options=options,
+            overrides=overrides,
+            enhanced_download_archive=enhanced_download_archive,
+        )
+        self._evaluated_map: Dict[str, bool] = {}
+
+    def modify_entry(self, entry: Entry) -> Optional[Entry]:
+        # Already evaluated in modify_entry_metadata, do not recompute
+        if entry.ytdl_uid() in self._evaluated_map:
+            return entry
+
+        for formatter in self.plugin_options.list:
+            out = json.loads(self.overrides.apply_formatter(formatter=formatter, entry=entry))
+            if bool(out):
+                return None
+
+        return entry
+
+    def modify_entry_metadata(self, entry: Entry) -> Optional[Entry]:
+        try:
+            output_entry = self.modify_entry(entry=entry)
+        except StringFormattingException:
+            # If filtering fails at the metadata stage, try again w/no catch in modify_entry
+            return entry
+
+        self._evaluated_map[entry.ytdl_uid()] = True
+        return output_entry

src/ytdl_sub/plugins/filter_include.py

@@ -0,0 +1,79 @@
+import json
+from typing import Dict
+from typing import Optional
+
+from ytdl_sub.config.overrides import Overrides
+from ytdl_sub.config.plugin.plugin import Plugin
+from ytdl_sub.config.validators.options import OptionsValidator
+from ytdl_sub.entries.entry import Entry
+from ytdl_sub.utils.exceptions import StringFormattingException
+from ytdl_sub.utils.logger import Logger
+from ytdl_sub.validators.string_formatter_validators import ListFormatterValidator
+from ytdl_sub.ytdl_additions.enhanced_download_archive import EnhancedDownloadArchive
+
+logger = Logger.get("conditional")
+
+
+class FilterIncludeOptions(ListFormatterValidator, OptionsValidator):
+    """
+    Applies a conditional AND on any number of filters comprised of either variables or scripts.
+    If all filters evaluate to True, the entry will be included.
+
+    Usage:
+
+    .. code-block:: yaml
+
+       presets:
+         my_example_preset:
+           filter_include:
+             - {description}
+             - >-
+               {
+                 %regex_search_any(
+                    title,
+                    [
+                        "Full Episode",
+                        "FULL",
+                    ]
+                 )
+               }
+    """
+
+
+class FilterIncludePlugin(Plugin[FilterIncludeOptions]):
+    plugin_options_type = FilterIncludeOptions
+
+    def __init__(
+        self,
+        options: FilterIncludeOptions,
+        overrides: Overrides,
+        enhanced_download_archive: EnhancedDownloadArchive,
+    ):
+        super().__init__(
+            options=options,
+            overrides=overrides,
+            enhanced_download_archive=enhanced_download_archive,
+        )
+        self._evaluated_map: Dict[str, bool] = {}
+
+    def modify_entry(self, entry: Entry) -> Optional[Entry]:
+        # Already evaluated in modify_entry_metadata, do not recompute
+        if entry.ytdl_uid() in self._evaluated_map:
+            return entry
+
+        for formatter in self.plugin_options.list:
+            out = json.loads(self.overrides.apply_formatter(formatter=formatter, entry=entry))
+            if not bool(out):
+                return None
+
+        return entry
+
+    def modify_entry_metadata(self, entry: Entry) -> Optional[Entry]:
+        try:
+            output_entry = self.modify_entry(entry=entry)
+        except StringFormattingException:
+            # If filtering fails at the metadata stage, try again w/no catch in modify_entry
+            return entry
+
+        self._evaluated_map[entry.ytdl_uid()] = True
+        return output_entry

src/ytdl_sub/script/functions/array_functions.py

@@ -50,6 +50,29 @@ class ArrayFunctions:
 
         return Array(output)
 
+    @staticmethod
+    def array_overlay(
+        array: Array, overlap: Array, only_missing: Optional[Boolean] = None
+    ) -> Array:
+        """
+        Overlaps ``overlap`` onto ``array``. Can optionally only overlay missing indices.
+        """
+        output: List[Resolvable] = []
+        output.extend(array.value)
+
+        overlap_only_missing = only_missing and only_missing.value
+
+        for idx, overlap_value in enumerate(overlap.value):
+            if overlap_only_missing and idx < len(array.value):
+                continue
+
+            if idx < len(array.value):
+                output[idx] = overlap_value
+            else:
+                output.append(overlap_value)
+
+        return Array(output)
+
     @staticmethod
     def array_at(array: Array, idx: Integer) -> AnyArgument:
         """
@@ -57,6 +80,18 @@ class ArrayFunctions:
         """
         return array.value[idx.value]
 
+    @staticmethod
+    def array_first(array: Array, fallback: AnyArgument) -> AnyArgument:
+        """
+        Returns the first element whose boolean conversion is True. Returns fallback
+        if all elements evaluate to False.
+        """
+        for val in array.value:
+            if bool(val.value):
+                return val
+
+        return fallback
+
     @staticmethod
     def array_contains(array: Array, value: AnyArgument) -> Boolean:
         """
@@ -130,6 +165,22 @@ class ArrayFunctions:
         """
         return Array([Array([val]) for val in array.value])
 
+    @staticmethod
+    def array_apply_fixed(
+        array: Array,
+        fixed_argument: AnyArgument,
+        lambda2_function: LambdaTwo,
+        reverse_args: Optional[Boolean] = None,
+    ) -> Array:
+        """
+        Apply a lambda function on every element in the Array, with ``fixed_argument``
+        passed as a second argument to every invocation.
+        """
+        if reverse_args and reverse_args.value:
+            return Array([Array([fixed_argument, val]) for val in array.value])
+
+        return Array([Array([val, fixed_argument]) for val in array.value])
+
     @staticmethod
     def array_enumerate(array: Array, lambda_function: LambdaTwo) -> Array:
         """

src/ytdl_sub/script/functions/boolean_functions.py

@@ -1,5 +1,6 @@
 from ytdl_sub.script.types.resolvable import AnyArgument
 from ytdl_sub.script.types.resolvable import Boolean
+from ytdl_sub.script.types.resolvable import String
 
 # pylint: disable=invalid-name
 
@@ -87,3 +88,10 @@ class BooleanFunctions:
         ``not`` operator. Returns the opposite of value.
         """
         return Boolean(not value.value)
+
+    @staticmethod
+    def is_null(value: AnyArgument) -> Boolean:
+        """
+        Returns True if a value is null (i.e. an empty string). False otherwise.
+        """
+        return Boolean(isinstance(value, String) and value.value == "")

src/ytdl_sub/script/functions/error_functions.py

@@ -21,3 +21,39 @@ class ErrorFunctions:
         if not bool(value.value):
             raise UserThrownRuntimeError(assert_message)
         return value
+
+    @staticmethod
+    def assert_then(
+        value: AnyArgument, ret: ReturnableArgument, assert_message: String
+    ) -> ReturnableArgument:
+        """
+        Explicitly throw an error with the provided assert message if ``value`` evaluates to False.
+        If it evaluates to True, it will return ``ret``.
+        """
+        if not bool(value.value):
+            raise UserThrownRuntimeError(assert_message)
+        return ret
+
+    @staticmethod
+    def assert_eq(
+        value: ReturnableArgument, equals: AnyArgument, assert_message: String
+    ) -> ReturnableArgument:
+        """
+        Explicitly throw an error with the provided assert message if ``value`` does not equal
+        ``equals``. If they do equal, then return ``value``.
+        """
+        if not value.value == equals.value:
+            raise UserThrownRuntimeError(assert_message)
+        return value
+
+    @staticmethod
+    def assert_ne(
+        value: ReturnableArgument, equals: AnyArgument, assert_message: String
+    ) -> ReturnableArgument:
+        """
+        Explicitly throw an error with the provided assert message if ``value`` equals
+        ``equals``. If they do equal, then return ``value``.
+        """
+        if value.value == equals.value:
+            raise UserThrownRuntimeError(assert_message)
+        return value

src/ytdl_sub/script/functions/regex_functions.py

@@ -3,6 +3,7 @@ from typing import AnyStr
 from typing import Match
 
 from ytdl_sub.script.types.array import Array
+from ytdl_sub.script.types.resolvable import Integer
 from ytdl_sub.script.types.resolvable import String
 
 
@@ -40,3 +41,10 @@ class RegexFunctions:
         group as a subsequent element in the Array.
         """
         return _re_output_to_array(re.fullmatch(regex.value, string.value))
+
+    @staticmethod
+    def regex_capture_groups(regex: String) -> Integer:
+        """
+        Returns number of capture groups in regex
+        """
+        return Integer(re.compile(regex.value).groups)

src/ytdl_sub/script/functions/string_functions.py

@@ -1,6 +1,7 @@
 from typing import Optional
 
 from ytdl_sub.script.types.resolvable import AnyArgument
+from ytdl_sub.script.types.resolvable import Boolean
 from ytdl_sub.script.types.resolvable import Integer
 from ytdl_sub.script.types.resolvable import Numeric
 from ytdl_sub.script.types.resolvable import String
@@ -14,6 +15,13 @@ class StringFunctions:
         """
         return String(value=str(value.value))
 
+    @staticmethod
+    def contains(string: String, contains: String) -> Boolean:
+        """
+        Returns True if ``contains`` is in ``string``. False otherwise.
+        """
+        return Boolean(contains.value in string.value)
+
     @staticmethod
     def slice(string: String, start: Integer, end: Optional[Integer] = None) -> String:
         """

src/ytdl_sub/script/types/resolvable.py

@@ -170,6 +170,10 @@ class Boolean(ResolvableT[bool], Argument):
     Resolved bool type
     """
 
+    def __str__(self):
+        # makes it JSON friendly
+        return str(self.value).lower()
+
 
 @dataclass(frozen=True)
 class String(ResolvableT[str], Argument):

tests/e2e/plugins/test_filter.py

@@ -0,0 +1,299 @@
+import re
+from typing import Any
+from typing import Dict
+
+import mergedeep
+import pytest
+from expected_transaction_log import assert_transaction_log_matches
+
+from ytdl_sub.config.config_file import ConfigFile
+from ytdl_sub.script.utils.exceptions import UserThrownRuntimeError
+from ytdl_sub.subscriptions.subscription import Subscription
+
+
+@pytest.fixture
+def regex_subscription_dict_base(output_directory):
+    return {
+        "preset": "Jellyfin Music Videos",
+        # override the output directory with our fixture-generated dir
+        "output_options": {"output_directory": output_directory},
+        "format": "best[height<=480]",  # download the worst format so it is fast
+        "filter_include": ["{%not(%is_null(description_website))}"],
+        "overrides": {
+            "in_regex_default": "in regex default",
+            "url": "https://youtube.com/playlist?list=PL5BC0FC26BECA5A35",
+            "upload_capture": """{
+                %regex_capture_many_with_defaults(
+                    upload_date_standardized,
+                    ["([0-9]+)-([0-9]+)-27"],
+                    ["First", %concat("Second containing ", in_regex_default)]
+                )
+            }""",
+            "upload_captured_year": "{%array_at(upload_capture, 1)}",
+            "upload_captured_month": "{%array_at(upload_capture, 2)}",
+            "description_capture": """{
+                %regex_capture_many_with_defaults(
+                    description,
+                    [".*http:\\/\\/(.+).com.*"],
+                    [null]
+                )
+            }""",
+            "description_website": "{%array_at(description_capture, 1)}",
+        },
+    }
+
+
+@pytest.fixture
+def regex_subscription_dict(regex_subscription_dict_base, output_directory):
+    return mergedeep.merge(
+        regex_subscription_dict_base,
+        {
+            "nfo_tags": {
+                "tags": {
+                    "title_cap_1": "{title_type}",
+                    "title_cap_1_sanitized": "{title_type_sanitized}",
+                    "title_cap_2": "{title_date}",
+                    "desc_cap": "{description_website}",
+                    "upload_date_both_caps": "{upload_captured_year} and {upload_captured_month}",
+                    "override_with_capture_variable": "{contains_regex_default}",
+                    "override_with_capture_variable_sanitized": "{contains_regex_sanitized_default}",
+                }
+            },
+            "filter_exclude": ["{%is_null(title_type)}"],
+            "overrides": {
+                "title_capture_list": """{
+                    %regex_capture_many_with_defaults(
+                        title,
+                        [ "should not cap (.+) - (.+)", ".*\\[(.+) - (Feb.+)]" ],
+                        [ null, null ]
+                    )
+                }""",
+                "title_type": "{%array_at(title_capture_list, 1)}",
+                "title_date": "{%array_at(title_capture_list, 2)}",
+                "contains_regex_default": "contains {title_type}",
+                "contains_regex_sanitized_default": "contains {title_type_sanitized}",
+            },
+        },
+    )
+
+
+@pytest.fixture
+def regex_subscription_dict_exclude(regex_subscription_dict_base, output_directory):
+    return mergedeep.merge(
+        regex_subscription_dict_base,
+        {
+            "filter_exclude": [
+                """{
+                    %regex_search_any(
+                        title,
+                        [ "should not cap", ".*Feb.*" ]
+                    )
+                }"""
+            ]
+        },
+    )
+
+
+@pytest.fixture
+def regex_subscription_dict_match_and_exclude(regex_subscription_dict_base, output_directory):
+    return mergedeep.merge(
+        regex_subscription_dict_base,
+        {
+            "regex": {
+                # tests that skip_if_match_fails defaults to True
+                "from": {
+                    "title": {
+                        "match": [
+                            "should not cap (.+) - (.+)",
+                            ".*\\[(.+) - (Feb.+)]",  # should filter out march video
+                        ],
+                        "capture_group_names": ["title_type", "title_date"],
+                        "exclude": [
+                            "should not cap",
+                            ".*27.*",  # should filter out Feb 27th video
+                        ],
+                    },
+                },
+            },
+            "nfo_tags": {
+                "tags": {
+                    "title_cap_1": "{title_type}",
+                    "title_cap_1_sanitized": "{title_type_sanitized}",
+                    "title_cap_2": "{title_date}",
+                    "desc_cap": "{description_website}",
+                    "upload_date_both_caps": "{upload_captured_year} and {upload_captured_month}",
+                    "override_with_capture_variable": "{contains_regex_default}",
+                    "override_with_capture_variable_sanitized": "{contains_regex_sanitized_default}",
+                }
+            },
+            "overrides": {
+                "contains_regex_default": "contains {title_type}",
+                "contains_regex_sanitized_default": "contains {title_type_sanitized}",
+            },
+        },
+    )
+
+
+@pytest.fixture
+def regex_subscription_dict_match_and_exclude_override_variable(
+    regex_subscription_dict_base, output_directory
+):
+    return mergedeep.merge(
+        regex_subscription_dict_base,
+        {
+            "regex": {
+                # tests that skip_if_match_fails defaults to True
+                "from": {
+                    "override_title": {
+                        "exclude": [
+                            "should not cap",
+                            ".*Feb.*",  # should filter out march video
+                        ],
+                    },
+                    "override_description": {
+                        "match": [".*http:\\/\\/(.+).com.*"],
+                        "capture_group_names": ["override_description_website"],
+                    },
+                },
+            },
+            "overrides": {"override_title": "{title}", "override_description": "{description}"},
+        },
+    )
+
+
+@pytest.fixture
+def playlist_subscription(default_config, regex_subscription_dict):
+    return Subscription.from_dict(
+        config=default_config,
+        preset_name="regex_capture_playlist_test",
+        preset_dict=regex_subscription_dict,
+    )
+
+
+@pytest.fixture
+def playlist_subscription_no_match_fails(
+    default_config: ConfigFile, regex_subscription_dict: Dict[str, Any]
+):
+    regex_subscription_dict["overrides"][
+        "title_capture_list"
+    ] = """{
+        %regex_capture_many_required(
+            title,
+            [ "should not cap (.+) - (.+)", ".*\\[(.+) - (Feb.+)]" ]
+        )
+    }"""
+
+    return Subscription.from_dict(
+        config=default_config,
+        preset_name="regex_capture_playlist_test",
+        preset_dict=regex_subscription_dict,
+    )
+
+
+@pytest.fixture
+def playlist_subscription_exclude(
+    default_config: ConfigFile, regex_subscription_dict_exclude: Dict[str, Any]
+) -> Subscription:
+    return Subscription.from_dict(
+        config=default_config,
+        preset_name="regex_exclude_playlist_test",
+        preset_dict=regex_subscription_dict_exclude,
+    )
+
+
+@pytest.fixture
+def playlist_subscription_overrides(
+    default_config: ConfigFile,
+    regex_subscription_dict_match_and_exclude_override_variable: Dict[str, Any],
+) -> Subscription:
+    return Subscription.from_dict(
+        config=default_config,
+        preset_name="regex_using_overrides_test",
+        preset_dict=regex_subscription_dict_match_and_exclude_override_variable,
+    )
+
+
+@pytest.fixture
+def playlist_subscription_match_and_exclude(
+    default_config: ConfigFile, regex_subscription_dict_match_and_exclude: Dict[str, Any]
+) -> Subscription:
+    return Subscription.from_dict(
+        config=default_config,
+        preset_name="regex_match_and_exclude_playlist_test",
+        preset_dict=regex_subscription_dict_match_and_exclude,
+    )
+
+
+class TestFilter:
+    def test_regex_success(self, playlist_subscription, output_directory):
+        # Only dry run is needed to see if capture variables are created
+        transaction_log = playlist_subscription.download(dry_run=True)
+        assert_transaction_log_matches(
+            output_directory=output_directory,
+            transaction_log=transaction_log,
+            transaction_log_summary_file_name="plugins/test_regex.txt",
+        )
+
+    def test_regex_excludes_success(self, playlist_subscription_exclude, output_directory):
+        # Should only contain the march video
+        transaction_log = playlist_subscription_exclude.download(dry_run=True)
+        assert_transaction_log_matches(
+            output_directory=output_directory,
+            transaction_log=transaction_log,
+            transaction_log_summary_file_name="plugins/test_regex_exclude.txt",
+        )
+
+    def test_regex_match_and_excludes_success(
+        self, playlist_subscription_match_and_exclude, output_directory
+    ):
+        # Should only contain the Feb 1st video
+        transaction_log = playlist_subscription_match_and_exclude.download(dry_run=True)
+        assert_transaction_log_matches(
+            output_directory=output_directory,
+            transaction_log=transaction_log,
+            transaction_log_summary_file_name="plugins/test_regex_match_and_exclude.txt",
+        )
+
+    def test_regex_using_overrides_success(self, playlist_subscription_overrides, output_directory):
+        # Should only contain the march video
+        transaction_log = playlist_subscription_overrides.download(dry_run=True)
+        assert_transaction_log_matches(
+            output_directory=output_directory,
+            transaction_log=transaction_log,
+            transaction_log_summary_file_name="plugins/test_regex_overrides.txt",
+        )
+
+    def test_regex_fails_no_match(self, playlist_subscription_no_match_fails, output_directory):
+        with pytest.raises(
+            UserThrownRuntimeError,
+            match=re.escape(
+                "When running %regex_capture_many_required, no regex strings were captured"
+            ),
+        ):
+            _ = playlist_subscription_no_match_fails.download(dry_run=True)
+
+    def test_regex_fails_unequal_defaults(self, regex_subscription_dict, default_config):
+        regex_subscription_dict["overrides"][
+            "title_capture_list"
+        ] = """{
+            %regex_capture_many_with_defaults(
+                title,
+                [ "should not cap (.+) - (.+)", ".*\\[(.+) - (Feb.+)]" ],
+                [ "one default, expects >= 2" ]
+            )
+        }"""
+
+        subscription = Subscription.from_dict(
+            config=default_config,
+            preset_name="regex_capture_playlist_test",
+            preset_dict=regex_subscription_dict,
+        )
+
+        with pytest.raises(
+            UserThrownRuntimeError,
+            match=re.escape(
+                "When using %regex_capture_with_defaults, number of regex capture groups must "
+                "be less than or equal to the number of defaults"
+            ),
+        ):
+            _ = subscription.download(dry_run=True)

tests/e2e/plugins/test_regex.py

@@ -76,6 +76,21 @@ def regex_subscription_dict(regex_subscription_dict_base, output_directory):
                 }
             },
             "overrides": {
+                "title_capture_list": f"""{{
+                    %regex_capture_many_with_defaults(
+                        title,
+                        [
+                            "should not cap (.+) - (.+)",
+                            ".*\\[(.+) - (Feb.+)]"
+                        ],
+                        [
+                            "ack",
+                            "ack"
+                        ]
+                    )
+                }}""",
+                "title_capture_list_1": "{%array_at(title_capture_list, 1)}",
+                "title_capture_list_2": "{%array_at(title_capture_list, 2)}",
                 "contains_regex_default": "contains {title_type}",
                 "contains_regex_sanitized_default": "contains {title_type_sanitized}",
             },

tests/unit/config/test_config_file.py

@@ -54,6 +54,8 @@ class TestConfigFilePartiallyValidatesPresets:
         excluded_plugins = [
             "embed_thumbnail",  # value is bool, not dict
             "format",  # value is string, not dict
+            "filter_include",  # is list
+            "filter_exclude",  # is list
         ]
         if plugin not in excluded_plugins:
             self._partial_validate({plugin: {}})

tests/unit/config/test_preset.py

@@ -264,7 +264,6 @@ class TestPreset:
         "function_name",
         [
             "%extract_field_from_siblings",
-            "%extract_field_from_metadata_array",
             "%sanitize",
             "%array",
         ],

tests/unit/script/functions/test_array_functions.py

@@ -96,3 +96,50 @@ class TestArrayFunctions:
             FunctionRuntimeException, match="Tried and failed to cast Integer as an Array"
         ):
             single_variable_output("{%array(1)}")
+
+    def test_array_overlay(self):
+        output = single_variable_output("{%array_overlay([1, 2, 3], [4, 5])}")
+        assert output == [4, 5, 3]
+
+        output = single_variable_output("{%array_overlay([1, 2, 3], [4, 5, 6, 7, 8])}")
+        assert output == [4, 5, 6, 7, 8]
+
+        output = single_variable_output("{%array_overlay([1, 2, 3], [4, 5, 6, 7, 8], True)}")
+        assert output == [1, 2, 3, 7, 8]
+
+    def test_array_first(self):
+        output = single_variable_output(
+            "{%array_first(['', false, null, [], {}, 0, 'hi', 'no'], 'fallback')}"
+        )
+        assert output == "hi"
+
+        output = single_variable_output("{%array_first(['', false, null, [], {}, 0], 'fallback')}")
+        assert output == "fallback"
+
+    def test_array_apply_fixed(self):
+        output = (
+            Script(
+                {
+                    "map_test": "{ {'key1': 7, 'key2': 8, 'key3': 9} }",
+                    "output": """{
+                        %array_apply_fixed( ['key1', 'key2', 'key3'], map_test, %map_get, True)
+                    }""",
+                }
+            )
+            .resolve(update=True)
+            .get("output")
+            .native
+        )
+        assert output == [7, 8, 9]
+
+        output = (
+            Script(
+                {
+                    "output": "{%array_apply_fixed( ['key1', 'key2', 'key3'], '3', %contains)}",
+                }
+            )
+            .resolve(update=True)
+            .get("output")
+            .native
+        )
+        assert output == [False, False, True]

tests/unit/script/functions/test_boolean_functions.py

@@ -126,3 +126,17 @@ class TestBooleanFunctions:
     def test_not(self, value: str, expected_output: bool):
         output = single_variable_output(f"{{%not({value})}}")
         assert output == expected_output
+
+    @pytest.mark.parametrize(
+        "value, expected_output",
+        [
+            ("null", True),
+            ("''", True),
+            ("0", False),
+            ("{}", False),
+            ("'h'", False),
+        ],
+    )
+    def test_is_null(self, value: str, expected_output: bool):
+        output = single_variable_output(f"{{%is_null({value})}}")
+        assert output == expected_output

tests/unit/script/functions/test_conditional_functions.py

@@ -1,6 +1,5 @@
 import pytest
-
+from unit.script.conftest import single_variable_output
-from ytdl_sub.script.script import Script
 
 
 class TestConditionalFunction:
@@ -12,11 +11,12 @@ class TestConditionalFunction:
         ],
     )
     def test_if_function(self, function_str: str, expected_output: bool):
-        output = Script({"output": function_str}).resolve(update=True).get("output").native
+        output = single_variable_output(function_str)
         assert output == expected_output
 
     def test_nested_if_function(self):
-        function_str = """{
+        output = single_variable_output(
+            """{
             %if(
                 True,
                 %if(
@@ -31,5 +31,5 @@ class TestConditionalFunction:
                 True
             )
         }"""
-        output = Script({"output": function_str}).resolve(update=True).get("output").native
+        )
         assert output == "winner"

tests/unit/script/functions/test_error_functions.py

@@ -23,3 +23,83 @@ class TestErrorFunctions:
     def test_user_assert_passthrough_as_arg(self):
         output = single_variable_output("{%int(%assert('123', 'test this error message'))}")
         assert output == 123
+
+    def test_user_assert_eq(self):
+        output = single_variable_output(
+            """{
+            %int(
+                %assert_eq(
+                    '123',
+                    %array_at(['123'], 0),
+                    'test this error message'
+                )
+            )
+        }"""
+        )
+        assert output == 123
+
+    def test_user_assert_eq_raises(self):
+        with pytest.raises(UserThrownRuntimeError, match=re.escape("test this error message")):
+            single_variable_output(
+                """{
+                %int(
+                    %assert_eq(
+                        '123',
+                        %array_at(['no'], 0),
+                        'test this error message'
+                    )
+                )
+            }"""
+            )
+
+    def test_user_assert_ne(self):
+        output = single_variable_output(
+            """{
+            %int(
+                %assert_ne(
+                    '123',
+                    %array_at(['nope'], 0),
+                    'test this error message'
+                )
+            )
+        }"""
+        )
+        assert output == 123
+
+    def test_user_assert_ne_raises(self):
+        with pytest.raises(UserThrownRuntimeError, match=re.escape("test this error message")):
+            single_variable_output(
+                """{
+                %int(
+                    %assert_ne(
+                        '123',
+                        %array_at(['123'], 0),
+                        'test this error message'
+                    )
+                )
+            }"""
+            )
+
+    def test_user_assert_then(self):
+        output = single_variable_output(
+            """{
+            %assert_then(
+                '123',
+                %array_at(['nope'], 0),
+                'test this error message'
+            )
+        }"""
+        )
+        assert output == "nope"
+
+    def test_user_assert_then_raises(self):
+        with pytest.raises(UserThrownRuntimeError, match=re.escape("test this error message")):
+            single_variable_output(
+                """{
+                %assert_then(
+                    {},
+                    %array_at(['nope'], 0),
+                    'test this error message'
+                )
+            }"""
+            )

tests/unit/script/functions/test_string_functions.py

@@ -107,3 +107,10 @@ class TestNumericFunctions:
     def test_slice(self, values, expected_output):
         output = single_variable_output(f"{{%slice({values})}}")
         assert output == expected_output
+
+    @pytest.mark.parametrize(
+        "value, expected_output", [("a", True), ("nope", False), ("dog", True)]
+    )
+    def test_contains(self, value, expected_output):
+        output = single_variable_output(f"{{%contains('a brown dog', '{value}')}}")
+        assert output == expected_output