[FEATURE] Perform match-filter on download, not metadata pull (#612)

* [FEATURE] Apply match-filter on download instead of metadata

* correct ytdl_option ordering

* lint

* test that downloads

src/ytdl_sub/downloaders/url/downloader.py

@@ -11,6 +11,8 @@ from typing import Optional
 from typing import Set
 from typing import Tuple
 
+from yt_dlp.utils import RejectedVideoReached
+
 from ytdl_sub.config.preset_options import Overrides
 from ytdl_sub.downloaders.base_downloader import BaseDownloader
 from ytdl_sub.downloaders.base_downloader import BaseDownloaderOptionsT
@@ -31,6 +33,7 @@ from ytdl_sub.entries.variables.kwargs import REQUESTED_SUBTITLES
 from ytdl_sub.entries.variables.kwargs import SOURCE_ENTRY
 from ytdl_sub.entries.variables.kwargs import SPONSORBLOCK_CHAPTERS
 from ytdl_sub.entries.variables.kwargs import UPLOAD_DATE_INDEX
+from ytdl_sub.entries.variables.kwargs import YTDL_SUB_MATCH_FILTER_REJECT
 from ytdl_sub.plugins.plugin import Plugin
 from ytdl_sub.utils.file_handler import FileHandler
 from ytdl_sub.utils.logger import Logger
@@ -523,6 +526,11 @@ class BaseUrlDownloader(BaseDownloader[BaseDownloaderOptionsT], ABC):
         Returns
         -------
         The entry that was downloaded successfully
+
+        Raises
+        ------
+        RejectedVideoReached
+          If a video was rejected and was not from match_filter
         """
         download_logger.info(
             "Downloading entry %d/%d: %s",
@@ -530,7 +538,17 @@ class BaseUrlDownloader(BaseDownloader[BaseDownloaderOptionsT], ABC):
             self._url_state.entries_total,
             entry.title,
         )
-        download_entry = self._extract_entry_info_with_retry(entry=entry)
+
+        # Match-filters are applied at the download stage (not metadata stage).
+        # If the download is rejected, and match_filter is present in the ytdl options,
+        # then filter downstream in the match_filter plugin
+        try:
+            download_entry = self._extract_entry_info_with_retry(entry=entry)
+        except RejectedVideoReached:
+            if "match_filter" in self.download_ytdl_options:
+                entry.add_kwargs({YTDL_SUB_MATCH_FILTER_REJECT: True})
+                return entry
+            raise
 
         upload_date_idx = self._enhanced_download_archive.mapping.get_num_entries_with_upload_date(
             upload_date_standardized=entry.upload_date_standardized

src/ytdl_sub/entries/variables/kwargs.py

@@ -47,6 +47,7 @@ REQUESTED_SUBTITLES = _("requested_subtitles", backend=True)
 CHAPTERS = _("chapters", backend=True)
 YTDL_SUB_CUSTOM_CHAPTERS = _("ytdl_sub_custom_chapters", backend=True)
 YTDL_SUB_REGEX_SOURCE_VARS = _("ytdl_sub_regex_source_vars", backend=True)
+YTDL_SUB_MATCH_FILTER_REJECT = _("ytdl_sub_match_filter_reject", backend=True)
 SPONSORBLOCK_CHAPTERS = _("sponsorblock_chapters", backend=True)
 SPLIT_BY_CHAPTERS_PARENT_ENTRY = _("split_by_chapters_parent_entry", backend=True)
 COMMENTS = _("comments", backend=True)

src/ytdl_sub/plugins/match_filters.py

@@ -5,8 +5,11 @@ from typing import Optional
 
 from yt_dlp import match_filter_func
 
+from ytdl_sub.entries.entry import Entry
+from ytdl_sub.entries.variables.kwargs import YTDL_SUB_MATCH_FILTER_REJECT
 from ytdl_sub.plugins.plugin import Plugin
 from ytdl_sub.plugins.plugin import PluginOptions
+from ytdl_sub.plugins.plugin import PluginPriority
 from ytdl_sub.utils.logger import Logger
 from ytdl_sub.validators.validators import StringListValidator
 
@@ -63,6 +66,7 @@ class MatchFiltersOptions(PluginOptions):
 
 class MatchFiltersPlugin(Plugin[MatchFiltersOptions]):
     plugin_options_type = MatchFiltersOptions
+    priority = PluginPriority(modify_entry=PluginPriority.MODIFY_ENTRY_FIRST)
 
     def ytdl_options(self) -> Optional[Dict]:
         """
@@ -75,4 +79,17 @@ class MatchFiltersPlugin(Plugin[MatchFiltersOptions]):
             logger.debug("Adding match-filter %s", filter_str)
             match_filters.append(filter_str)
 
-        return {"match_filter": match_filter_func(match_filters)}
+        return {
+            "match_filter": match_filter_func(match_filters),
+        }
+
+    def modify_entry(self, entry: Entry) -> Optional[Entry]:
+        """
+        If an entry is marked as not being downloaded due to a match_filter reject,
+        do not propagate the entry further (especially since there is no entry file!)
+        """
+        if entry.kwargs_get(YTDL_SUB_MATCH_FILTER_REJECT, False):
+            logger.info("Entry rejected by match-filter, skipping ..")
+            return None
+
+        return entry

src/ytdl_sub/plugins/plugin.py

@@ -24,7 +24,9 @@ class PluginPriority:
     # If modify_entry priority is >= to this value, run after split
     MODIFY_ENTRY_AFTER_SPLIT = 10
 
-    def __init__(self, modify_entry: int = 0, post_process: int = 0):
+    MODIFY_ENTRY_FIRST = 0
+
+    def __init__(self, modify_entry: int = 5, post_process: int = 5):
         self.modify_entry = modify_entry
         self.post_process = post_process
 

src/ytdl_sub/subscriptions/subscription_ytdl_options.py

@@ -78,6 +78,10 @@ class SubscriptionYTDLOptions:
             "writeinfojson": True,
         }
 
+    @property
+    def _download_only_options(self) -> Dict:
+        return {"break_on_reject": True}
+
     @property
     def _output_options(self) -> Dict:
         ytdl_options = {}
@@ -110,7 +114,6 @@ class SubscriptionYTDLOptions:
             self._global_options,
             self._output_options,
             self._plugin_ytdl_options(DateRangePlugin),
-            self._plugin_ytdl_options(MatchFiltersPlugin),
             self._user_ytdl_options,  # user ytdl options...
             self._info_json_only_options,  # then info_json_only options
         )
@@ -125,12 +128,13 @@ class SubscriptionYTDLOptions:
         ytdl_options_builder = YTDLOptionsBuilder().add(
             self._global_options,
             self._output_options,
-            self._plugin_ytdl_options(DateRangePlugin),
+            self._plugin_ytdl_options(MatchFiltersPlugin),
             self._plugin_ytdl_options(FileConvertPlugin),
             self._plugin_ytdl_options(SubtitlesPlugin),
             self._plugin_ytdl_options(ChaptersPlugin),
             self._plugin_ytdl_options(AudioExtractPlugin),
             self._user_ytdl_options,  # user ytdl options...
+            self._download_only_options,  # then download_only options
         )
         # Add dry run options last if enabled
         if self._dry_run:

tests/e2e/plugins/test_match_filters.py

@@ -1,4 +1,6 @@
 import pytest
+from expected_download import assert_expected_downloads
+from expected_transaction_log import assert_transaction_log_matches
 
 from ytdl_sub.subscriptions.subscription import Subscription
 
@@ -17,9 +19,23 @@ def preset_dict(output_directory):
     }
 
 
+@pytest.fixture
+def playlist_preset_dict(output_directory):
+    return {
+        "preset": "music_video",
+        "download": {"url": "https://www.youtube.com/playlist?list=PL5BC0FC26BECA5A35"},
+        "output_options": {"output_directory": output_directory},
+        # download the worst format so it is fast
+        "ytdl_options": {
+            "postprocessor_args": {"ffmpeg": ["-bitexact"]},  # Must add this for reproducibility
+        },
+        "match_filters": {"filters": ["view_count > 20000"]},
+    }
+
+
 class TestFileConvert:
     @pytest.mark.parametrize("dry_run", [True, False])
-    def test_match_filters(
+    def test_match_filters_empty(
         self,
         music_video_config,
         preset_dict,
@@ -34,3 +50,31 @@ class TestFileConvert:
 
         transaction_log = subscription.download(dry_run=dry_run)
         assert transaction_log.is_empty
+
+    @pytest.mark.parametrize("dry_run", [True, False])
+    def test_match_filters_partial(
+        self,
+        music_video_config,
+        playlist_preset_dict,
+        output_directory,
+        dry_run,
+    ):
+        subscription = Subscription.from_dict(
+            config=music_video_config,
+            preset_name="match_filter_test",
+            preset_dict=playlist_preset_dict,
+        )
+
+        transaction_log = subscription.download(dry_run=dry_run)
+
+        summary_path = "plugins/match_filters"
+        assert_transaction_log_matches(
+            output_directory=output_directory,
+            transaction_log=transaction_log,
+            transaction_log_summary_file_name=f"{summary_path}/test_match_filters_partial.txt",
+        )
+        assert_expected_downloads(
+            output_directory=output_directory,
+            dry_run=dry_run,
+            expected_download_summary_file_name=f"{summary_path}/test_match_filters_partial.json",
+        )

tests/resources/expected_downloads_summaries/plugins/match_filters/test_match_filters_partial.json

@@ -0,0 +1,7 @@
+{
+  ".ytdl-sub-match_filter_test-download-archive.json": "6017105735dd8fdfe18932c9c048dccf",
+  "Project Zombie/Project Zombie - Jesse's Minecraft Server [Trailer - Mar.21]-thumb.jpg": "e7830aa8a64b0cde65ba3f7e5fc56530",
+  "Project Zombie/Project Zombie - Jesse's Minecraft Server [Trailer - Mar.21].info.json": "ddfda0c60aafd9779da692d6c165385b",
+  "Project Zombie/Project Zombie - Jesse's Minecraft Server [Trailer - Mar.21].nfo": "17c303f07e9b3af77c90975f843c95bd",
+  "Project Zombie/Project Zombie - Jesse's Minecraft Server [Trailer - Mar.21].webm": "ad90ebef1ea6cbdb924670cdd7d1b9b9"
+}

tests/resources/transaction_log_summaries/plugins/match_filters/test_match_filters_partial.txt

@@ -0,0 +1,15 @@
+Files created:
+----------------------------------------
+{output_directory}
+  .ytdl-sub-match_filter_test-download-archive.json
+{output_directory}/Project Zombie
+  Project Zombie - Jesse's Minecraft Server [Trailer - Mar.21]-thumb.jpg
+  Project Zombie - Jesse's Minecraft Server [Trailer - Mar.21].info.json
+  Project Zombie - Jesse's Minecraft Server [Trailer - Mar.21].nfo
+    NFO tags:
+      musicvideo:
+        album: Music Videos
+        artist: Project Zombie
+        title: Jesse's Minecraft Server [Trailer - Mar.21]
+        year: 2011
+  Project Zombie - Jesse's Minecraft Server [Trailer - Mar.21].webm