asadfasdf

src/ytdl_sub/config/plugin/plugin_mapping.py

@@ -15,6 +15,8 @@ from ytdl_sub.plugins.chapters import ChaptersPlugin
 from ytdl_sub.plugins.date_range import DateRangePlugin
 from ytdl_sub.plugins.embed_thumbnail import EmbedThumbnailPlugin
 from ytdl_sub.plugins.file_convert import FileConvertPlugin
+from ytdl_sub.plugins.filter_exclude import FilterExcludePlugin
+from ytdl_sub.plugins.filter_include import FilterIncludePlugin
 from ytdl_sub.plugins.format import FormatPlugin
 from ytdl_sub.plugins.internal.view import ViewPlugin
 from ytdl_sub.plugins.match_filters import MatchFiltersPlugin
@@ -50,6 +52,8 @@ class PluginMapping:
         "chapters": ChaptersPlugin,
         "split_by_chapters": SplitByChaptersPlugin,
         "throttle_protection": ThrottleProtectionPlugin,
+        "filter_include": FilterIncludePlugin,
+        "filter_exclude": FilterExcludePlugin,
     }
 
     # All other plugins are added after the defined ordered ones
@@ -57,6 +61,8 @@ class PluginMapping:
         ThrottleProtectionPlugin,
         UrlDownloaderCollectionVariablePlugin,
         SubtitlesPlugin,
+        FilterExcludePlugin,
+        FilterIncludePlugin,
         # add all others
     ]
 
@@ -66,6 +72,8 @@ class PluginMapping:
         FileConvertPlugin,
         ChaptersPlugin,
         SplitByChaptersPlugin,
+        FilterExcludePlugin,
+        FilterIncludePlugin,
         RegexPlugin,
         # add all others
     ]

src/ytdl_sub/config/validators/variable_validation.py

@@ -27,6 +27,18 @@ def _add_dummy_variables(variables: Iterable[str]) -> Dict[str, str]:
     return dummy_variables
 
 
+def _add_dummy_overrides(overrides: Overrides) -> Dict[str, str]:
+    # Have the dummy override variable contain all variable deps that it uses in the string
+    dummy_overrides: Dict[str, str] = {}
+    for override_name in _override_variables(overrides):
+        dummy_overrides[override_name] = ""
+        # pylint: disable=protected-access
+        for variable_dependency in overrides.script._variables[override_name].variables:
+            dummy_overrides[override_name] += f"{{ {variable_dependency.name } }}"
+        # pylint: enable=protected-access
+    return dummy_overrides
+
+
 def _get_added_and_modified_variables(
     plugins: PresetPlugins, downloader_options: MultiUrlValidator
 ) -> Iterable[Tuple[OptionsValidator, Set[str], Set[str]]]:
@@ -119,6 +131,10 @@ class VariableValidation:
 
         # copy the script and mock entry variables
         self.script = copy.deepcopy(overrides.script).add(_add_dummy_variables(entry_variables))
+        self.script.add(
+            variables=_add_dummy_overrides(overrides=overrides),
+            unresolvable=self.unresolved_variables,
+        )
 
         return self
 

src/ytdl_sub/entries/script/function_scripts.py

@@ -56,10 +56,15 @@ CUSTOM_FUNCTION_SCRIPTS: Dict[str, str] = {
                 %and
             ),
             %array_overlay(
-                %array_apply_fixed(
+                %array(
-                    %array($1),
+                    %array_first(
-                    %string($0),
+                        %array_apply_fixed(
-                    %regex_search
+                            %array($1),
+                            %string($0),
+                            %regex_search
+                        ),
+                        []
+                    )
                 ),
                 %array_extend( ['using all defaults'], $2 ),
                 True
@@ -74,7 +79,7 @@ CUSTOM_FUNCTION_SCRIPTS: Dict[str, str] = {
                 'When using %regex_capture_many, number of regex capture groups must be less than or equal to the number of defaults'
             ),
             ['using all defaults', '', '', '', '', '', '', '', '', '', ''],
-            'When running %regex_capture_many_required, no regex strings captured'
+            'When running %regex_capture_many_required, no regex strings were captured'
         )
     }""",
     "%regex_capture_many_with_defaults": """{

src/ytdl_sub/plugins/filter_exclude.py

@@ -0,0 +1,70 @@
+import json
+from typing import Dict
+from typing import Optional
+
+from ytdl_sub.config.overrides import Overrides
+from ytdl_sub.config.plugin.plugin import Plugin
+from ytdl_sub.config.validators.options import OptionsValidator
+from ytdl_sub.entries.entry import Entry
+from ytdl_sub.utils.exceptions import StringFormattingException
+from ytdl_sub.utils.logger import Logger
+from ytdl_sub.validators.string_formatter_validators import ListFormatterValidator
+from ytdl_sub.ytdl_additions.enhanced_download_archive import EnhancedDownloadArchive
+
+logger = Logger.get("conditional")
+
+
+class FilterExcludeOptions(ListFormatterValidator, OptionsValidator):
+    """
+    Applies a conditional OR on any number of filters comprised of either variables or scripts.
+    If any filter evaluates to True, the entry will be excluded.
+
+    Usage:
+
+    .. code-block:: yaml
+
+       presets:
+         my_example_preset:
+           filter_exclude:
+             - { %contains( %lower(title), '#short' ) }
+             - { %contains( %lower(description), '#short' ) }
+    """
+
+
+class FilterExcludePlugin(Plugin[FilterExcludeOptions]):
+    plugin_options_type = FilterExcludeOptions
+
+    def __init__(
+        self,
+        options: FilterExcludeOptions,
+        overrides: Overrides,
+        enhanced_download_archive: EnhancedDownloadArchive,
+    ):
+        super().__init__(
+            options=options,
+            overrides=overrides,
+            enhanced_download_archive=enhanced_download_archive,
+        )
+        self._evaluated_map: Dict[str, bool] = {}
+
+    def modify_entry(self, entry: Entry) -> Optional[Entry]:
+        # Already evaluated in modify_entry_metadata, do not recompute
+        if entry.ytdl_uid() in self._evaluated_map:
+            return entry
+
+        for formatter in self.plugin_options.list:
+            out = json.loads(self.overrides.apply_formatter(formatter=formatter, entry=entry))
+            if bool(out):
+                return None
+
+        return entry
+
+    def modify_entry_metadata(self, entry: Entry) -> Optional[Entry]:
+        try:
+            output_entry = self.modify_entry(entry=entry)
+        except StringFormattingException:
+            # If filtering fails at the metadata stage, try again w/no catch in modify_entry
+            return entry
+
+        self._evaluated_map[entry.ytdl_uid()] = True
+        return output_entry

src/ytdl_sub/plugins/filter_include.py

@@ -0,0 +1,79 @@
+import json
+from typing import Dict
+from typing import Optional
+
+from ytdl_sub.config.overrides import Overrides
+from ytdl_sub.config.plugin.plugin import Plugin
+from ytdl_sub.config.validators.options import OptionsValidator
+from ytdl_sub.entries.entry import Entry
+from ytdl_sub.utils.exceptions import StringFormattingException
+from ytdl_sub.utils.logger import Logger
+from ytdl_sub.validators.string_formatter_validators import ListFormatterValidator
+from ytdl_sub.ytdl_additions.enhanced_download_archive import EnhancedDownloadArchive
+
+logger = Logger.get("conditional")
+
+
+class FilterIncludeOptions(ListFormatterValidator, OptionsValidator):
+    """
+    Applies a conditional AND on any number of filters comprised of either variables or scripts.
+    If all filters evaluate to True, the entry will be included.
+
+    Usage:
+
+    .. code-block:: yaml
+
+       presets:
+         my_example_preset:
+           filter_include:
+             - {description}
+             - >-
+               {
+                 %regex_search_any(
+                    title,
+                    [
+                        "Full Episode",
+                        "FULL",
+                    ]
+                 )
+               }
+    """
+
+
+class FilterIncludePlugin(Plugin[FilterIncludeOptions]):
+    plugin_options_type = FilterIncludeOptions
+
+    def __init__(
+        self,
+        options: FilterIncludeOptions,
+        overrides: Overrides,
+        enhanced_download_archive: EnhancedDownloadArchive,
+    ):
+        super().__init__(
+            options=options,
+            overrides=overrides,
+            enhanced_download_archive=enhanced_download_archive,
+        )
+        self._evaluated_map: Dict[str, bool] = {}
+
+    def modify_entry(self, entry: Entry) -> Optional[Entry]:
+        # Already evaluated in modify_entry_metadata, do not recompute
+        if entry.ytdl_uid() in self._evaluated_map:
+            return entry
+
+        for formatter in self.plugin_options.list:
+            out = json.loads(self.overrides.apply_formatter(formatter=formatter, entry=entry))
+            if not bool(out):
+                return None
+
+        return entry
+
+    def modify_entry_metadata(self, entry: Entry) -> Optional[Entry]:
+        try:
+            output_entry = self.modify_entry(entry=entry)
+        except StringFormattingException:
+            # If filtering fails at the metadata stage, try again w/no catch in modify_entry
+            return entry
+
+        self._evaluated_map[entry.ytdl_uid()] = True
+        return output_entry

src/ytdl_sub/script/functions/boolean_functions.py

@@ -1,5 +1,6 @@
 from ytdl_sub.script.types.resolvable import AnyArgument
 from ytdl_sub.script.types.resolvable import Boolean
+from ytdl_sub.script.types.resolvable import String
 
 # pylint: disable=invalid-name
 
@@ -87,3 +88,10 @@ class BooleanFunctions:
         ``not`` operator. Returns the opposite of value.
         """
         return Boolean(not value.value)
+
+    @staticmethod
+    def is_null(value: AnyArgument) -> Boolean:
+        """
+        Returns True if a value is null (i.e. an empty string). False otherwise.
+        """
+        return Boolean(isinstance(value, String) and value.value == "")

src/ytdl_sub/script/functions/string_functions.py

@@ -1,6 +1,7 @@
 from typing import Optional
 
 from ytdl_sub.script.types.resolvable import AnyArgument
+from ytdl_sub.script.types.resolvable import Boolean
 from ytdl_sub.script.types.resolvable import Integer
 from ytdl_sub.script.types.resolvable import Numeric
 from ytdl_sub.script.types.resolvable import String
@@ -14,6 +15,13 @@ class StringFunctions:
         """
         return String(value=str(value.value))
 
+    @staticmethod
+    def contains(string: String, contains: String) -> Boolean:
+        """
+        Returns True if ``contains`` is in ``string``. False otherwise.
+        """
+        return Boolean(contains.value in string.value)
+
     @staticmethod
     def slice(string: String, start: Integer, end: Optional[Integer] = None) -> String:
         """

src/ytdl_sub/script/types/resolvable.py

@@ -170,6 +170,10 @@ class Boolean(ResolvableT[bool], Argument):
     Resolved bool type
     """
 
+    def __str__(self):
+        # makes it JSON friendly
+        return str(self.value).lower()
+
 
 @dataclass(frozen=True)
 class String(ResolvableT[str], Argument):

tests/e2e/plugins/test_filter.py

@@ -0,0 +1,360 @@
+import copy
+import re
+from typing import Any
+from typing import Dict
+
+import mergedeep
+import pytest
+from expected_transaction_log import assert_transaction_log_matches
+
+from ytdl_sub.config.config_file import ConfigFile
+from ytdl_sub.script.utils.exceptions import UserThrownRuntimeError
+from ytdl_sub.subscriptions.subscription import Subscription
+from ytdl_sub.utils.exceptions import RegexNoMatchException
+from ytdl_sub.utils.exceptions import ValidationException
+
+
+@pytest.fixture
+def regex_subscription_dict_base(output_directory):
+    return {
+        "preset": "Jellyfin Music Videos",
+        # override the output directory with our fixture-generated dir
+        "output_options": {"output_directory": output_directory},
+        "format": "best[height<=480]",  # download the worst format so it is fast
+        "filter_exclude": ["{%is_null(description_website)}"],
+        "overrides": {
+            "in_regex_default": "in regex default",
+            "url": "https://youtube.com/playlist?list=PL5BC0FC26BECA5A35",
+            "upload_capture": """{
+                %regex_capture_many_with_defaults(
+                    upload_date_standardized,
+                    ["([0-9]+)-([0-9]+)-27"],
+                    ["First", %concat("Second containing ", in_regex_default)]
+                )
+            }""",
+            "upload_captured_year": "{%array_at(upload_capture, 1)}",
+            "upload_captured_month": "{%array_at(upload_capture, 2)}",
+            "description_capture": """{
+                %regex_capture_many_with_defaults(
+                    description,
+                    [".*http:\\/\\/(.+).com.*"],
+                    [null]
+                )
+            }""",
+            "description_website": "{%array_at(description_capture, 1)}",
+        },
+    }
+
+
+@pytest.fixture
+def regex_subscription_dict(regex_subscription_dict_base, output_directory):
+    return mergedeep.merge(
+        regex_subscription_dict_base,
+        {
+            "nfo_tags": {
+                "tags": {
+                    "title_cap_1": "{title_type}",
+                    "title_cap_1_sanitized": "{title_type_sanitized}",
+                    "title_cap_2": "{title_date}",
+                    "desc_cap": "{description_website}",
+                    "upload_date_both_caps": "{upload_captured_year} and {upload_captured_month}",
+                    "override_with_capture_variable": "{contains_regex_default}",
+                    "override_with_capture_variable_sanitized": "{contains_regex_sanitized_default}",
+                }
+            },
+            "filter_exclude": ["{%is_null(title_type)}"],
+            "overrides": {
+                "title_capture_list": """{
+                    %regex_capture_many_with_defaults(
+                        title,
+                        [ "should not cap (.+) - (.+)", ".*\\[(.+) - (Feb.+)]" ],
+                        [ null, null ]
+                    )
+                }""",
+                "title_type": "{%array_at(title_capture_list, 1)}",
+                "title_date": "{%array_at(title_capture_list, 2)}",
+                "contains_regex_default": "contains {title_type}",
+                "contains_regex_sanitized_default": "contains {title_type_sanitized}",
+            },
+        },
+    )
+
+
+@pytest.fixture
+def regex_subscription_dict_exclude(regex_subscription_dict_base, output_directory):
+    return mergedeep.merge(
+        regex_subscription_dict_base,
+        {
+            "filter_exclude": [
+                """{
+                    %regex_search_any(
+                        title,
+                        [ "should not cap", ".*Feb.*" ]
+                    )
+                }"""
+            ]
+        },
+    )
+
+
+@pytest.fixture
+def regex_subscription_dict_match_and_exclude(regex_subscription_dict_base, output_directory):
+    return mergedeep.merge(
+        regex_subscription_dict_base,
+        {
+            "regex": {
+                # tests that skip_if_match_fails defaults to True
+                "from": {
+                    "title": {
+                        "match": [
+                            "should not cap (.+) - (.+)",
+                            ".*\\[(.+) - (Feb.+)]",  # should filter out march video
+                        ],
+                        "capture_group_names": ["title_type", "title_date"],
+                        "exclude": [
+                            "should not cap",
+                            ".*27.*",  # should filter out Feb 27th video
+                        ],
+                    },
+                },
+            },
+            "nfo_tags": {
+                "tags": {
+                    "title_cap_1": "{title_type}",
+                    "title_cap_1_sanitized": "{title_type_sanitized}",
+                    "title_cap_2": "{title_date}",
+                    "desc_cap": "{description_website}",
+                    "upload_date_both_caps": "{upload_captured_year} and {upload_captured_month}",
+                    "override_with_capture_variable": "{contains_regex_default}",
+                    "override_with_capture_variable_sanitized": "{contains_regex_sanitized_default}",
+                }
+            },
+            "overrides": {
+                "contains_regex_default": "contains {title_type}",
+                "contains_regex_sanitized_default": "contains {title_type_sanitized}",
+            },
+        },
+    )
+
+
+@pytest.fixture
+def regex_subscription_dict_match_and_exclude_override_variable(
+    regex_subscription_dict_base, output_directory
+):
+    return mergedeep.merge(
+        regex_subscription_dict_base,
+        {
+            "regex": {
+                # tests that skip_if_match_fails defaults to True
+                "from": {
+                    "override_title": {
+                        "exclude": [
+                            "should not cap",
+                            ".*Feb.*",  # should filter out march video
+                        ],
+                    },
+                    "override_description": {
+                        "match": [".*http:\\/\\/(.+).com.*"],
+                        "capture_group_names": ["override_description_website"],
+                    },
+                },
+            },
+            "overrides": {"override_title": "{title}", "override_description": "{description}"},
+        },
+    )
+
+
+@pytest.fixture
+def playlist_subscription(default_config, regex_subscription_dict):
+    return Subscription.from_dict(
+        config=default_config,
+        preset_name="regex_capture_playlist_test",
+        preset_dict=regex_subscription_dict,
+    )
+
+
+@pytest.fixture
+def playlist_subscription_no_match_fails(
+    default_config: ConfigFile, regex_subscription_dict: Dict[str, Any]
+):
+    regex_subscription_dict["overrides"][
+        "title_capture_list"
+    ] = """{
+        %regex_capture_many_required(
+            title,
+            [ "should not cap (.+) - (.+)", ".*\\[(.+) - (Feb.+)]" ]
+        )
+    }"""
+
+    return Subscription.from_dict(
+        config=default_config,
+        preset_name="regex_capture_playlist_test",
+        preset_dict=regex_subscription_dict,
+    )
+
+
+@pytest.fixture
+def playlist_subscription_exclude(
+    default_config: ConfigFile, regex_subscription_dict_exclude: Dict[str, Any]
+) -> Subscription:
+    return Subscription.from_dict(
+        config=default_config,
+        preset_name="regex_exclude_playlist_test",
+        preset_dict=regex_subscription_dict_exclude,
+    )
+
+
+@pytest.fixture
+def playlist_subscription_overrides(
+    default_config: ConfigFile,
+    regex_subscription_dict_match_and_exclude_override_variable: Dict[str, Any],
+) -> Subscription:
+    return Subscription.from_dict(
+        config=default_config,
+        preset_name="regex_using_overrides_test",
+        preset_dict=regex_subscription_dict_match_and_exclude_override_variable,
+    )
+
+
+@pytest.fixture
+def playlist_subscription_match_and_exclude(
+    default_config: ConfigFile, regex_subscription_dict_match_and_exclude: Dict[str, Any]
+) -> Subscription:
+    return Subscription.from_dict(
+        config=default_config,
+        preset_name="regex_match_and_exclude_playlist_test",
+        preset_dict=regex_subscription_dict_match_and_exclude,
+    )
+
+
+class TestRegex:
+    def test_regex_success(self, playlist_subscription, output_directory):
+        # Only dry run is needed to see if capture variables are created
+        transaction_log = playlist_subscription.download(dry_run=True)
+        assert_transaction_log_matches(
+            output_directory=output_directory,
+            transaction_log=transaction_log,
+            transaction_log_summary_file_name="plugins/test_regex.txt",
+        )
+
+    def test_regex_excludes_success(self, playlist_subscription_exclude, output_directory):
+        # Should only contain the march video
+        transaction_log = playlist_subscription_exclude.download(dry_run=True)
+        assert_transaction_log_matches(
+            output_directory=output_directory,
+            transaction_log=transaction_log,
+            transaction_log_summary_file_name="plugins/test_regex_exclude.txt",
+        )
+
+    def test_regex_match_and_excludes_success(
+        self, playlist_subscription_match_and_exclude, output_directory
+    ):
+        # Should only contain the Feb 1st video
+        transaction_log = playlist_subscription_match_and_exclude.download(dry_run=True)
+        assert_transaction_log_matches(
+            output_directory=output_directory,
+            transaction_log=transaction_log,
+            transaction_log_summary_file_name="plugins/test_regex_match_and_exclude.txt",
+        )
+
+    def test_regex_using_overrides_success(self, playlist_subscription_overrides, output_directory):
+        # Should only contain the march video
+        transaction_log = playlist_subscription_overrides.download(dry_run=True)
+        assert_transaction_log_matches(
+            output_directory=output_directory,
+            transaction_log=transaction_log,
+            transaction_log_summary_file_name="plugins/test_regex_overrides.txt",
+        )
+
+    def test_regex_fails_no_match(self, playlist_subscription_no_match_fails, output_directory):
+        with pytest.raises(
+            UserThrownRuntimeError,
+            match=re.escape(
+                "When running %regex_capture_many_required, no regex strings were captured"
+            ),
+        ):
+            _ = playlist_subscription_no_match_fails.download(dry_run=True)
+
+    def test_regex_fails_capture_group_is_entry_variable(
+        self, regex_subscription_dict, default_config
+    ):
+        regex_subscription_dict["regex"]["from"]["playlist_uid"] = {
+            "match": [".*http:\\/\\/(.+).com.*"],
+            "capture_group_names": ["uid"],
+        }
+
+        with pytest.raises(
+            ValidationException,
+            match=re.escape(
+                "Cannot use the variable name uid because it exists as a built-in "
+                "ytdl-sub variable name."
+            ),
+        ):
+            _ = Subscription.from_dict(
+                config=default_config,
+                preset_name="test_regex_fails_capture_group_is_entry_variable",
+                preset_dict=regex_subscription_dict,
+            )
+
+    def test_regex_fails_capture_group_is_override_variable(
+        self, regex_subscription_dict, default_config
+    ):
+        regex_subscription_dict["regex"]["from"]["playlist_uid"] = {
+            "match": [".*http:\\/\\/(.+).com.*"],
+            "capture_group_names": ["contains_regex_default"],
+        }
+
+        with pytest.raises(
+            ValidationException,
+            match=re.escape(
+                "Override variable with name contains_regex_default cannot be used since it is "
+                "added by a plugin."
+            ),
+        ):
+            _ = Subscription.from_dict(
+                config=default_config,
+                preset_name="test_regex_fails_capture_group_is_override_variable",
+                preset_dict=regex_subscription_dict,
+            )
+
+    def test_regex_fails_source_variable_does_not_exist(
+        self, regex_subscription_dict, default_config
+    ):
+        regex_subscription_dict["regex"]["from"]["dne"] = copy.deepcopy(
+            regex_subscription_dict["regex"]["from"]["title"]
+        )
+        with pytest.raises(
+            ValidationException,
+            match=re.escape("cannot regex capture 'dne' because it is not a defined variable"),
+        ):
+            _ = Subscription.from_dict(
+                config=default_config,
+                preset_name="test_regex_fails_source_variable_does_not_exist",
+                preset_dict=regex_subscription_dict,
+            )
+
+    def test_regex_fails_unequal_defaults(self, regex_subscription_dict, default_config):
+        regex_subscription_dict["regex"]["from"]["title"]["capture_group_defaults"] = ["1 != 2"]
+        with pytest.raises(
+            ValidationException,
+            match=re.escape("number of defaults must match number of capture groups, 1 != 2"),
+        ):
+            _ = Subscription.from_dict(
+                config=default_config,
+                preset_name="test_regex_fails_unequal_defaults",
+                preset_dict=regex_subscription_dict,
+            )
+
+    def test_regex_fails_unequal_capture_group_names(self, regex_subscription_dict, default_config):
+        regex_subscription_dict["regex"]["from"]["title"]["capture_group_names"].append("unequal")
+        with pytest.raises(
+            ValidationException,
+            match=re.escape(
+                "number of capture group names must match number of capture groups, 3 != 2"
+            ),
+        ):
+            _ = Subscription.from_dict(
+                config=default_config,
+                preset_name="test_regex_fails_unequal_capture_group_names",
+                preset_dict=regex_subscription_dict,
+            )