[BACKEND] Perform match-filters at metadata stage, use match-filters for date_range plugin (#725)

Solved Issue: https://github.com/jmbannon/ytdl-sub/issues/706

From our discussion with yt-dlp folks (https://github.com/yt-dlp/yt-dlp/issues/8108), we learned that best practices for date--range filtering is done using match-filters. We now set the `date_range` plugin values to be a 'breaking' match filter, and any custom ones specified from users in the `match_filters` plugin as non-breaking. This means match-filters can be performed at the metadata stage, which should yield a significant speedup

Relevant docs:
- date_range: https://ytdl-sub.readthedocs.io/en/latest/config.html#date-range
- match_filters: https://ytdl-sub.readthedocs.io/en/latest/config.html#match-filters

examples/tv_show_config.yaml

@@ -69,10 +69,6 @@ presets:
     date_range:
       after: "today-{download_range}"
 
-    # Stops fetching metadata if the video is out of range (saves time + bandwidth)
-    ytdl_options:
-      break_on_reject: True
-
     # Deletes any videos older than download_range. WARNING: do not use
     # "season_by_year__episode_by_download_index" if you plan to delete older videos
     output_options:

src/ytdl_sub/config/plugin.py

@@ -72,6 +72,14 @@ class Plugin(BasePlugin[TOptionsValidator], Generic[TOptionsValidator], ABC):
     Class to define the new plugin functionality
     """
 
+    def ytdl_options_match_filters(self) -> Tuple[List[str], List[str]]:
+        """
+        Returns
+        -------
+        Tuple of match-filters to apply, first one being non-breaking, second breaking
+        """
+        return [], []
+
     def ytdl_options(self) -> Optional[Dict]:
         """
         Returns

src/ytdl_sub/downloaders/url/downloader.py

@@ -9,8 +9,6 @@ from typing import Optional
 from typing import Set
 from typing import Tuple
 
-from yt_dlp.utils import RejectedVideoReached
-
 from ytdl_sub.config.plugin import PluginPriority
 from ytdl_sub.config.preset_options import Overrides
 from ytdl_sub.downloaders.source_plugin import SourcePlugin
@@ -30,7 +28,6 @@ from ytdl_sub.entries.variables.kwargs import REQUESTED_SUBTITLES
 from ytdl_sub.entries.variables.kwargs import SOURCE_ENTRY
 from ytdl_sub.entries.variables.kwargs import SPONSORBLOCK_CHAPTERS
 from ytdl_sub.entries.variables.kwargs import UPLOAD_DATE_INDEX
-from ytdl_sub.entries.variables.kwargs import YTDL_SUB_MATCH_FILTER_REJECT
 from ytdl_sub.utils.file_handler import FileHandler
 from ytdl_sub.utils.logger import Logger
 from ytdl_sub.utils.thumbnail import ThumbnailTypes
@@ -485,16 +482,7 @@ class MultiUrlDownloader(SourcePlugin[MultiUrlValidator]):
             entry.title,
         )
 
-        # Match-filters are applied at the download stage (not metadata stage).
+        download_entry = self._extract_entry_info_with_retry(entry=entry)
-        # If the download is rejected, and match_filter is present in the ytdl options,
-        # then filter downstream in the match_filter plugin
-        try:
-            download_entry = self._extract_entry_info_with_retry(entry=entry)
-        except RejectedVideoReached:
-            if "match_filter" in self.download_ytdl_options:
-                entry.add_kwargs({YTDL_SUB_MATCH_FILTER_REJECT: True})
-                return entry
-            raise
 
         upload_date_idx = self._enhanced_download_archive.mapping.get_num_entries_with_upload_date(
             upload_date_standardized=entry.upload_date_standardized

src/ytdl_sub/entries/variables/kwargs.py

@@ -47,7 +47,6 @@ REQUESTED_SUBTITLES = _("requested_subtitles", backend=True)
 CHAPTERS = _("chapters", backend=True)
 YTDL_SUB_CUSTOM_CHAPTERS = _("ytdl_sub_custom_chapters", backend=True)
 YTDL_SUB_REGEX_SOURCE_VARS = _("ytdl_sub_regex_source_vars", backend=True)
-YTDL_SUB_MATCH_FILTER_REJECT = _("ytdl_sub_match_filter_reject", backend=True)
 SPONSORBLOCK_CHAPTERS = _("sponsorblock_chapters", backend=True)
 SPLIT_BY_CHAPTERS_PARENT_ENTRY = _("split_by_chapters_parent_entry", backend=True)
 COMMENTS = _("comments", backend=True)

src/ytdl_sub/plugins/date_range.py

@@ -1,10 +1,9 @@
-from typing import Dict
+from typing import List
 from typing import Optional
+from typing import Tuple
 
 from ytdl_sub.config.plugin import Plugin
 from ytdl_sub.config.preset_options import OptionsDictValidator
-from ytdl_sub.downloaders.ytdl_options_builder import YTDLOptionsBuilder
-from ytdl_sub.utils.datetime import to_date_range
 from ytdl_sub.validators.string_datetime import StringDatetimeValidator
 
 
@@ -48,25 +47,21 @@ class DateRangeOptions(OptionsDictValidator):
 class DateRangePlugin(Plugin[DateRangeOptions]):
     plugin_options_type = DateRangeOptions
 
-    def ytdl_options(self) -> Optional[Dict]:
+    def ytdl_options_match_filters(self) -> Tuple[List[str], List[str]]:
         """
         Returns
         -------
-        YTDL options for setting a date range
+        match filters for before (non-breaking) and after (breaking)
         """
-        ytdl_options_builder = YTDLOptionsBuilder()
+        match_filters: List[str] = []
+        breaking_match_filters: List[str] = []
 
-        source_date_range = to_date_range(
+        if self.plugin_options.before:
-            before=self.plugin_options.before,
+            before_str = self.overrides.apply_formatter(formatter=self.plugin_options.before)
-            after=self.plugin_options.after,
+            match_filters.append(f"upload_date < {before_str}")
-            overrides=self.overrides,
-        )
-        if source_date_range:
-            ytdl_options_builder.add({"daterange": source_date_range})
 
-            # Only add break_on_reject if after is specified, but not before.
+        if self.plugin_options.after:
-            # Otherwise, it can break on first metadata pull if it's after the 'before'
+            after_str = self.overrides.apply_formatter(formatter=self.plugin_options.after)
-            if self.plugin_options.after and not self.plugin_options.before:
+            breaking_match_filters.append(f"upload_date >= {after_str}")
-                ytdl_options_builder.add({"break_on_reject": True})
 
-        return ytdl_options_builder.to_dict()
+        return match_filters, breaking_match_filters

src/ytdl_sub/plugins/match_filters.py

@@ -1,15 +1,9 @@
 from typing import Any
-from typing import Dict
 from typing import List
-from typing import Optional
+from typing import Tuple
-
-from yt_dlp import match_filter_func
 
 from ytdl_sub.config.plugin import Plugin
-from ytdl_sub.config.plugin import PluginPriority
 from ytdl_sub.config.preset_options import OptionsDictValidator
-from ytdl_sub.entries.entry import Entry
-from ytdl_sub.entries.variables.kwargs import YTDL_SUB_MATCH_FILTER_REJECT
 from ytdl_sub.utils.logger import Logger
 from ytdl_sub.validators.validators import StringListValidator
 
@@ -71,9 +65,8 @@ class MatchFiltersOptions(OptionsDictValidator):
 
 class MatchFiltersPlugin(Plugin[MatchFiltersOptions]):
     plugin_options_type = MatchFiltersOptions
-    priority = PluginPriority(modify_entry=PluginPriority.MODIFY_ENTRY_FIRST)
 
-    def ytdl_options(self) -> Optional[Dict]:
+    def ytdl_options_match_filters(self) -> Tuple[List[str], List[str]]:
         """
         Returns
         -------
@@ -81,20 +74,6 @@ class MatchFiltersPlugin(Plugin[MatchFiltersOptions]):
         """
         match_filters: List[str] = []
         for filter_str in self.plugin_options.filters:
-            logger.debug("Adding match-filter %s", filter_str)
             match_filters.append(filter_str)
 
-        return {
+        return match_filters, []
-            "match_filter": match_filter_func(match_filters),
-        }
-
-    def modify_entry(self, entry: Entry) -> Optional[Entry]:
-        """
-        If an entry is marked as not being downloaded due to a match_filter reject,
-        do not propagate the entry further (especially since there is no entry file!)
-        """
-        if entry.kwargs_get(YTDL_SUB_MATCH_FILTER_REJECT, False):
-            logger.info("Entry rejected by match-filter, skipping ..")
-            return None
-
-        return entry

src/ytdl_sub/subscriptions/subscription_ytdl_options.py

@@ -5,15 +5,15 @@ from typing import Optional
 from typing import Type
 from typing import TypeVar
 
+from yt_dlp import match_filter_func
+
 from ytdl_sub.config.plugin import Plugin
 from ytdl_sub.config.preset import Preset
 from ytdl_sub.downloaders.ytdl_options_builder import YTDLOptionsBuilder
 from ytdl_sub.plugins.audio_extract import AudioExtractPlugin
 from ytdl_sub.plugins.chapters import ChaptersPlugin
-from ytdl_sub.plugins.date_range import DateRangePlugin
 from ytdl_sub.plugins.file_convert import FileConvertPlugin
 from ytdl_sub.plugins.format import FormatPlugin
-from ytdl_sub.plugins.match_filters import MatchFiltersPlugin
 from ytdl_sub.plugins.subtitles import SubtitlesPlugin
 from ytdl_sub.utils.ffmpeg import FFMPEG
 from ytdl_sub.ytdl_additions.enhanced_download_archive import EnhancedDownloadArchive
@@ -97,6 +97,22 @@ class SubscriptionYTDLOptions:
     def _user_ytdl_options(self) -> Dict:
         return self._preset.ytdl_options.dict
 
+    @property
+    def _plugin_match_filters(self) -> Dict:
+        match_filters: List[str] = []
+        breaking_match_filters: List[str] = []
+        for plugin in self._plugins:
+            pl_match_filters, pl_breaking_match_filters = plugin.ytdl_options_match_filters()
+
+            match_filters.extend(pl_match_filters)
+            breaking_match_filters.extend(pl_breaking_match_filters)
+
+        return {
+            "match_filter": match_filter_func(
+                filters=match_filters, breaking_filters=breaking_match_filters
+            )
+        }
+
     def metadata_builder(self) -> YTDLOptionsBuilder:
         """
         Returns
@@ -107,7 +123,7 @@ class SubscriptionYTDLOptions:
         return YTDLOptionsBuilder().add(
             self._global_options,
             self._output_options,
-            self._plugin_ytdl_options(DateRangePlugin),
+            self._plugin_match_filters,
             self._plugin_ytdl_options(FormatPlugin),
             self._user_ytdl_options,  # user ytdl options...
             self._info_json_only_options,  # then info_json_only options
@@ -123,7 +139,6 @@ class SubscriptionYTDLOptions:
         ytdl_options_builder = YTDLOptionsBuilder().add(
             self._global_options,
             self._output_options,
-            self._plugin_ytdl_options(MatchFiltersPlugin),
             self._plugin_ytdl_options(FileConvertPlugin),
             self._plugin_ytdl_options(SubtitlesPlugin),
             self._plugin_ytdl_options(ChaptersPlugin),