[BUGFIX] Fix match-filter edge case and date_range.before (#750)

Fixes yet another bug that is match-filter related. The `date_range.before` value would get ignored if you specified any other match-filters. That is fixed by making them `&` behind-the-scenes.

src/ytdl_sub/config/plugin.py

@@ -72,15 +72,6 @@ class Plugin(BasePlugin[TOptionsValidator], Generic[TOptionsValidator], ABC):
     Class to define the new plugin functionality
     """
 
-    @classmethod
-    def default_ytdl_options(cls) -> Dict:
-        """
-        Returns
-        -------
-        ytdl options to enable if the plugin is not specified in the download
-        """
-        return {}
-
     def ytdl_options_match_filters(self) -> Tuple[List[str], List[str]]:
         """
         Returns

src/ytdl_sub/plugins/match_filters.py

@@ -1,11 +1,8 @@
+import copy
 from typing import Any
-from typing import Dict
 from typing import List
-from typing import Optional
 from typing import Tuple
 
-from yt_dlp import match_filter_func
-
 from ytdl_sub.config.plugin import Plugin
 from ytdl_sub.config.preset_options import OptionsDictValidator
 from ytdl_sub.utils.logger import Logger
@@ -13,7 +10,50 @@ from ytdl_sub.validators.validators import StringListValidator
 
 logger = Logger.get("match_filters")
 
-_DEFAULT_DOWNLOAD_MATCH_FILTERS: List[str] = ["!is_live & !is_upcoming & !post_live"]
+
+def default_filters() -> Tuple[List[str], List[str]]:
+    """
+    Returns
+    -------
+    Default filters and breaking filters to always use
+    """
+    return ["!is_live & !is_upcoming & !post_live"], []
+
+
+def combine_filters(filters: List[str], to_combine: List[str]) -> List[str]:
+    """
+    Parameters
+    ----------
+    filters
+        User-defined match-filters
+    to_combine
+        Filters that need to be combined via AND to the original filters.
+        These are derived from plugins
+
+    Returns
+    -------
+    merged filters
+
+    Raises
+    ------
+    ValueError
+        Only supports combining 1 filter at this time. Should never be hit by users
+    """
+    if len(to_combine) == 0:
+        return filters
+    if not filters:
+        return copy.deepcopy(to_combine)
+
+    if len(to_combine) > 1:
+        raise ValueError("Match-filters to combine only supports 1 at this time")
+
+    output_filters: List[str] = []
+    filter_to_combine: str = to_combine[0]
+
+    for match_filter in filters:
+        output_filters.append(f"{match_filter} & {filter_to_combine}")
+
+    return output_filters
 
 
 class MatchFiltersOptions(OptionsDictValidator):
@@ -46,24 +86,18 @@ class MatchFiltersOptions(OptionsDictValidator):
                # - "availability=?public"
     """
 
-    _optional_keys = {"filters", "download_filters"}
+    _optional_keys = {"filters"}
 
     @classmethod
     def partial_validate(cls, name: str, value: Any) -> None:
         """Ensure filters looks right"""
         if isinstance(value, dict):
             value["filters"] = value.get("filters", [""])
-            value["download_filters"] = value.get("download_filters", [""])
         _ = cls(name, value)
 
     def __init__(self, name, value):
         super().__init__(name, value)
         self._filters = self._validate_key_if_present(key="filters", validator=StringListValidator)
-        self._download_filters = self._validate_key(
-            key="download_filters",
-            validator=StringListValidator,
-            default=_DEFAULT_DOWNLOAD_MATCH_FILTERS,
-        )
 
     @property
     def filters(self) -> List[str]:
@@ -74,35 +108,10 @@ class MatchFiltersOptions(OptionsDictValidator):
         """
         return [validator.value for validator in self._filters.list] if self._filters else []
 
-    @property
-    def download_filters(self) -> List[str]:
-        """
-        Filters to apply during the download stage. This can be useful when building presets
-        that contain match-filters that you do not want to conflict with metadata-based
-        match-filters since they act as logical OR's.
-
-        By default, if no download_filters are present, then the filter
-        ``"!is_live & !is_upcoming & !post_live"`` is added.
-        """
-        return [validator.value for validator in self._download_filters.list]
-
 
 class MatchFiltersPlugin(Plugin[MatchFiltersOptions]):
     plugin_options_type = MatchFiltersOptions
 
-    @classmethod
-    def default_ytdl_options(cls) -> Dict:
-        """
-        Returns
-        -------
-        match-filter to filter out live + upcoming videos when downloading
-        """
-        return {
-            "match_filter": match_filter_func(
-                filters=[], breaking_filters=_DEFAULT_DOWNLOAD_MATCH_FILTERS
-            ),
-        }
-
     def ytdl_options_match_filters(self) -> Tuple[List[str], List[str]]:
         """
         Returns
@@ -110,15 +119,3 @@ class MatchFiltersPlugin(Plugin[MatchFiltersOptions]):
         match_filters to apply at the metadata stage
         """
         return self.plugin_options.filters, []
-
-    def ytdl_options(self) -> Optional[Dict]:
-        """
-        Returns
-        -------
-        match_filters to apply at the download stage
-        """
-        return {
-            "match_filter": match_filter_func(
-                filters=[], breaking_filters=self.plugin_options.download_filters
-            ),
-        }

src/ytdl_sub/subscriptions/subscription_ytdl_options.py

@@ -15,12 +15,17 @@ from ytdl_sub.plugins.chapters import ChaptersPlugin
 from ytdl_sub.plugins.file_convert import FileConvertPlugin
 from ytdl_sub.plugins.format import FormatPlugin
 from ytdl_sub.plugins.match_filters import MatchFiltersPlugin
+from ytdl_sub.plugins.match_filters import combine_filters
+from ytdl_sub.plugins.match_filters import default_filters
 from ytdl_sub.plugins.subtitles import SubtitlesPlugin
 from ytdl_sub.utils.ffmpeg import FFMPEG
+from ytdl_sub.utils.logger import Logger
 from ytdl_sub.ytdl_additions.enhanced_download_archive import EnhancedDownloadArchive
 
 PluginT = TypeVar("PluginT", bound=Plugin)
 
+logger = Logger.get("ytdl-options")
+
 
 class SubscriptionYTDLOptions:
     def __init__(
@@ -89,10 +94,10 @@ class SubscriptionYTDLOptions:
         return ytdl_options
 
     def _plugin_ytdl_options(self, plugin: Type[PluginT]) -> Dict:
-        if not (plugin_obj := self._get_plugin(plugin)):
+        if plugin_obj := self._get_plugin(plugin):
-            return plugin.default_ytdl_options()
+            return plugin_obj.ytdl_options()
 
-        return plugin_obj.ytdl_options()
+        return {}
 
     @property
     def _user_ytdl_options(self) -> Dict:
@@ -100,14 +105,50 @@ class SubscriptionYTDLOptions:
 
     @property
     def _plugin_match_filters(self) -> Dict:
-        match_filters: List[str] = []
+        """
-        breaking_match_filters: List[str] = []
+        All match-filters from every plugin to fetch metadata.
+        In order for other plugins to not collide with user-defined match-filters, do
+
+        match_filters = user_match_filters or {}
+        for plugin in plugins:
+          for filter in match_filters:
+            AND plugin's match filters onto the existing filters
+
+        Otherwise, the filters separately act as an OR
+        """
+        match_filters, breaking_match_filters = default_filters()
+
+        match_filters_plugin = self._get_plugin(MatchFiltersPlugin)
+        if match_filters_plugin:
+            (
+                user_match_filters,
+                user_breaking_match_filters,
+            ) = match_filters_plugin.ytdl_options_match_filters()
+            match_filters = combine_filters(filters=user_match_filters, to_combine=match_filters)
+            breaking_match_filters = combine_filters(
+                filters=user_breaking_match_filters, to_combine=breaking_match_filters
+            )
+
         for plugin in self._plugins:
+            # Do not re-add original match-filters plugin
+            if isinstance(plugin, MatchFiltersPlugin):
+                continue
+
             pl_match_filters, pl_breaking_match_filters = plugin.ytdl_options_match_filters()
 
-            match_filters.extend(pl_match_filters)
+            match_filters = combine_filters(filters=match_filters, to_combine=pl_match_filters)
-            breaking_match_filters.extend(pl_breaking_match_filters)
+            breaking_match_filters = combine_filters(
+                filters=breaking_match_filters, to_combine=pl_breaking_match_filters
+            )
 
+        logger.debug(
+            "Setting match-filters: %s",
+            "\n - ".join([""] + match_filters) if match_filters else "[]",
+        )
+        logger.debug(
+            "Setting breaking-match-filters: %s",
+            "\n - ".join([""] + breaking_match_filters) if breaking_match_filters else "[]",
+        )
         return {
             "match_filter": match_filter_func(
                 filters=match_filters, breaking_filters=breaking_match_filters
@@ -145,7 +186,6 @@ class SubscriptionYTDLOptions:
             self._plugin_ytdl_options(ChaptersPlugin),
             self._plugin_ytdl_options(AudioExtractPlugin),
             self._plugin_ytdl_options(FormatPlugin),
-            self._plugin_ytdl_options(MatchFiltersPlugin),
             self._user_ytdl_options,  # user ytdl options...
             self._download_only_options,  # then download_only options
         )