[BACKEND] Always filter out live, post_live, and upcoming videos on download (#742)

Closes Issue: https://github.com/jmbannon/ytdl-sub/issues/729

Add a new field to the match_filter plugin: `download_match_filters`. This set of filters will be applied at the download stage (as opposed to `filters` which get applied at the metadata stage). By default, always set `"!is_live & !is_upcoming & !post_live"` as a download filter. This prevents ytdl-sub from hanging when trying to grab live and upcoming videos.

Docs on match-filters: https://ytdl-sub.readthedocs.io/en/latest/config.html#match-filters

docs/config.rst

@@ -193,6 +193,7 @@ match_filters
 '''''''''''''
 .. autoclass:: ytdl_sub.plugins.match_filters.MatchFiltersOptions()
   :members:
+  :member-order: bysource
   :exclude-members: partial_validate
 
 -------------------------------------------------------------------------------

src/ytdl_sub/config/plugin.py

@@ -72,6 +72,15 @@ class Plugin(BasePlugin[TOptionsValidator], Generic[TOptionsValidator], ABC):
     Class to define the new plugin functionality
     """
 
+    @classmethod
+    def default_ytdl_options(cls) -> Dict:
+        """
+        Returns
+        -------
+        ytdl options to enable if the plugin is not specified in the download
+        """
+        return {}
+
     def ytdl_options_match_filters(self) -> Tuple[List[str], List[str]]:
         """
         Returns

src/ytdl_sub/downloaders/info_json/info_json_downloader.py

@@ -4,6 +4,7 @@ from pathlib import Path
 from typing import Dict
 from typing import Iterable
 from typing import List
+from typing import Optional
 
 from ytdl_sub.config.preset_options import OptionsDictValidator
 from ytdl_sub.config.preset_options import Overrides
@@ -123,7 +124,7 @@ class InfoJsonDownloader(SourcePlugin[InfoJsonDownloaderOptions]):
             ):
                 self._enhanced_download_archive.num_entries_removed += 1
 
-    def download(self, entry: Entry) -> Entry:
+    def download(self, entry: Entry) -> Optional[Entry]:
         """
         Mock the download by copying the entry file from the output directory into
         the working directory

src/ytdl_sub/downloaders/source_plugin.py

@@ -56,7 +56,7 @@ class SourcePlugin(BasePlugin[TOptionsValidator], Generic[TOptionsValidator], AB
         """Gathers metadata of all entries to download"""
 
     @abc.abstractmethod
-    def download(self, entry: Entry) -> Entry:
+    def download(self, entry: Entry) -> Optional[Entry]:
         """The function to perform the download of all media entries"""
 
     @final

src/ytdl_sub/downloaders/url/downloader.py

@@ -9,6 +9,8 @@ from typing import Optional
 from typing import Set
 from typing import Tuple
 
+from yt_dlp.utils import RejectedVideoReached
+
 from ytdl_sub.config.plugin import PluginPriority
 from ytdl_sub.config.preset_options import Overrides
 from ytdl_sub.downloaders.source_plugin import SourcePlugin
@@ -459,7 +461,7 @@ class MultiUrlDownloader(SourcePlugin[MultiUrlValidator]):
                 )
                 yield entry
 
-    def download(self, entry: Entry) -> Entry:
+    def download(self, entry: Entry) -> Optional[Entry]:
         """
         Parameters
         ----------
@@ -482,7 +484,13 @@ class MultiUrlDownloader(SourcePlugin[MultiUrlValidator]):
             entry.title,
         )
 
-        download_entry = self._extract_entry_info_with_retry(entry=entry)
+        # Match-filters can be applied at the download stage. If the download is rejected,
+        # then return None
+        try:
+            download_entry = self._extract_entry_info_with_retry(entry=entry)
+        except RejectedVideoReached:
+            download_logger.info("Entry rejected by download match-filter, skipping ..")
+            return None
 
         upload_date_idx = self._enhanced_download_archive.mapping.get_num_entries_with_upload_date(
             upload_date_standardized=entry.upload_date_standardized

src/ytdl_sub/plugins/match_filters.py

@@ -1,7 +1,11 @@
 from typing import Any
+from typing import Dict
 from typing import List
+from typing import Optional
 from typing import Tuple
 
+from yt_dlp import match_filter_func
+
 from ytdl_sub.config.plugin import Plugin
 from ytdl_sub.config.preset_options import OptionsDictValidator
 from ytdl_sub.utils.logger import Logger
@@ -9,6 +13,8 @@ from ytdl_sub.validators.validators import StringListValidator
 
 logger = Logger.get("match_filters")
 
+_DEFAULT_DOWNLOAD_MATCH_FILTERS: List[str] = ["!is_live & !is_upcoming & !post_live"]
+
 
 class MatchFiltersOptions(OptionsDictValidator):
     """
@@ -40,40 +46,79 @@ class MatchFiltersOptions(OptionsDictValidator):
                # - "availability=?public"
     """
 
-    _required_keys = {"filters"}
+    _optional_keys = {"filters", "download_filters"}
 
     @classmethod
     def partial_validate(cls, name: str, value: Any) -> None:
         """Ensure filters looks right"""
         if isinstance(value, dict):
             value["filters"] = value.get("filters", [""])
+            value["download_filters"] = value.get("download_filters", [""])
         _ = cls(name, value)
 
     def __init__(self, name, value):
         super().__init__(name, value)
-        self._filters = self._validate_key(key="filters", validator=StringListValidator).list
+        self._filters = self._validate_key_if_present(key="filters", validator=StringListValidator)
+        self._download_filters = self._validate_key(
+            key="download_filters",
+            validator=StringListValidator,
+            default=_DEFAULT_DOWNLOAD_MATCH_FILTERS,
+        )
 
     @property
     def filters(self) -> List[str]:
         """
         The filters themselves. If used multiple times, the filter matches if at least one of the
         conditions are met. For logical AND's between match filters, use the ``&`` operator in
-        a single match filter.
+        a single match filter. These are applied when gathering metadata.
         """
-        return [validator.value for validator in self._filters]
+        return [validator.value for validator in self._filters.list] if self._filters else []
+
+    @property
+    def download_filters(self) -> List[str]:
+        """
+        Filters to apply during the download stage. This can be useful when building presets
+        that contain match-filters that you do not want to conflict with metadata-based
+        match-filters since they act as logical OR's.
+
+        By default, if no download_filters are present, then the filter
+        ``"!is_live & !is_upcoming & !post_live"`` is added.
+        """
+        return [validator.value for validator in self._download_filters.list]
 
 
 class MatchFiltersPlugin(Plugin[MatchFiltersOptions]):
     plugin_options_type = MatchFiltersOptions
 
+    @classmethod
+    def default_ytdl_options(cls) -> Dict:
+        """
+        Returns
+        -------
+        match-filter to filter out live + upcoming videos when downloading
+        """
+        return {
+            "match_filter": match_filter_func(
+                filters=[], breaking_filters=_DEFAULT_DOWNLOAD_MATCH_FILTERS
+            ),
+        }
+
     def ytdl_options_match_filters(self) -> Tuple[List[str], List[str]]:
         """
         Returns
         -------
-        match_filter after calling the utility function for it
+        match_filters to apply at the metadata stage
         """
-        match_filters: List[str] = []
-        for filter_str in self.plugin_options.filters:
-            match_filters.append(filter_str)
+        return self.plugin_options.filters, []
 
-        return match_filters, []
+    def ytdl_options(self) -> Optional[Dict]:
+        """
+        Returns
+        -------
+        match_filters to apply at the download stage
+        """
+        return {
+            "match_filter": match_filter_func(
+                filters=[], breaking_filters=self.plugin_options.download_filters
+            ),
+        }

src/ytdl_sub/subscriptions/subscription_download.py

@@ -295,6 +295,9 @@ class SubscriptionDownload(BaseSubscription, ABC):
                     continue
 
                 entry = downloader.download(entry)
+                if entry is None:
+                    continue
+
                 entry_metadata = FileMetadata()
                 if isinstance(entry, tuple):
                     entry, entry_metadata = entry

src/ytdl_sub/subscriptions/subscription_ytdl_options.py

@@ -14,6 +14,7 @@ from ytdl_sub.plugins.audio_extract import AudioExtractPlugin
 from ytdl_sub.plugins.chapters import ChaptersPlugin
 from ytdl_sub.plugins.file_convert import FileConvertPlugin
 from ytdl_sub.plugins.format import FormatPlugin
+from ytdl_sub.plugins.match_filters import MatchFiltersPlugin
 from ytdl_sub.plugins.subtitles import SubtitlesPlugin
 from ytdl_sub.utils.ffmpeg import FFMPEG
 from ytdl_sub.ytdl_additions.enhanced_download_archive import EnhancedDownloadArchive
@@ -88,10 +89,10 @@ class SubscriptionYTDLOptions:
         return ytdl_options
 
     def _plugin_ytdl_options(self, plugin: Type[PluginT]) -> Dict:
-        if not (audio_extract_plugin := self._get_plugin(plugin)):
-            return {}
+        if not (plugin_obj := self._get_plugin(plugin)):
+            return plugin.default_ytdl_options()
 
-        return audio_extract_plugin.ytdl_options()
+        return plugin_obj.ytdl_options()
 
     @property
     def _user_ytdl_options(self) -> Dict:
@@ -144,6 +145,7 @@ class SubscriptionYTDLOptions:
             self._plugin_ytdl_options(ChaptersPlugin),
             self._plugin_ytdl_options(AudioExtractPlugin),
             self._plugin_ytdl_options(FormatPlugin),
+            self._plugin_ytdl_options(MatchFiltersPlugin),
             self._user_ytdl_options,  # user ytdl options...
             self._download_only_options,  # then download_only options
         )

tests/e2e/plugins/test_match_filters.py

@@ -11,7 +11,6 @@ def preset_dict(output_directory):
         "preset": "music_video",
         "download": "https://www.youtube.com/watch?v=2zYF9JLHDmA",
         "output_options": {"output_directory": output_directory},
-        # download the worst format so it is fast
         "ytdl_options": {
             "postprocessor_args": {"ffmpeg": ["-bitexact"]},  # Must add this for reproducibility
         },
@@ -25,7 +24,6 @@ def playlist_preset_dict(output_directory):
         "preset": "music_video",
         "download": "https://www.youtube.com/playlist?list=PL5BC0FC26BECA5A35",
         "output_options": {"output_directory": output_directory},
-        # download the worst format so it is fast
         "ytdl_options": {
             "postprocessor_args": {"ffmpeg": ["-bitexact"]},  # Must add this for reproducibility
         },
@@ -33,7 +31,34 @@ def playlist_preset_dict(output_directory):
     }
 
 
+@pytest.fixture
+def livestream_preset_dict(output_directory):
+    return {
+        "preset": "music_video",
+        "download": "https://www.youtube.com/watch?v=DoUOrTJbIu4",
+        "output_options": {"output_directory": output_directory},
+        "ytdl_options": {
+            "postprocessor_args": {"ffmpeg": ["-bitexact"]},
+        },
+    }
+
+
 class TestFileConvert:
+    def test_livestreams_download_filtered(
+        self,
+        music_video_config,
+        livestream_preset_dict,
+        output_directory,
+    ):
+        subscription = Subscription.from_dict(
+            config=music_video_config,
+            preset_name="match_filter_test",
+            preset_dict=livestream_preset_dict,
+        )
+
+        transaction_log = subscription.download(dry_run=False)
+        assert transaction_log.is_empty
+
     @pytest.mark.parametrize("dry_run", [True, False])
     def test_match_filters_empty(
         self,