diff --git a/src/ytdl_sub/downloaders/url/downloader.py b/src/ytdl_sub/downloaders/url/downloader.py index 1f9aeef3..fae279e6 100644 --- a/src/ytdl_sub/downloaders/url/downloader.py +++ b/src/ytdl_sub/downloaders/url/downloader.py @@ -11,6 +11,8 @@ from typing import Optional from typing import Set from typing import Tuple +from yt_dlp.utils import RejectedVideoReached + from ytdl_sub.config.preset_options import Overrides from ytdl_sub.downloaders.base_downloader import BaseDownloader from ytdl_sub.downloaders.base_downloader import BaseDownloaderOptionsT @@ -31,6 +33,7 @@ from ytdl_sub.entries.variables.kwargs import REQUESTED_SUBTITLES from ytdl_sub.entries.variables.kwargs import SOURCE_ENTRY from ytdl_sub.entries.variables.kwargs import SPONSORBLOCK_CHAPTERS from ytdl_sub.entries.variables.kwargs import UPLOAD_DATE_INDEX +from ytdl_sub.entries.variables.kwargs import YTDL_SUB_MATCH_FILTER_REJECT from ytdl_sub.plugins.plugin import Plugin from ytdl_sub.utils.file_handler import FileHandler from ytdl_sub.utils.logger import Logger @@ -523,6 +526,11 @@ class BaseUrlDownloader(BaseDownloader[BaseDownloaderOptionsT], ABC): Returns ------- The entry that was downloaded successfully + + Raises + ------ + RejectedVideoReached + If a video was rejected and was not from match_filter """ download_logger.info( "Downloading entry %d/%d: %s", @@ -530,7 +538,17 @@ class BaseUrlDownloader(BaseDownloader[BaseDownloaderOptionsT], ABC): self._url_state.entries_total, entry.title, ) - download_entry = self._extract_entry_info_with_retry(entry=entry) + + # Match-filters are applied at the download stage (not metadata stage). + # If the download is rejected, and match_filter is present in the ytdl options, + # then filter downstream in the match_filter plugin + try: + download_entry = self._extract_entry_info_with_retry(entry=entry) + except RejectedVideoReached: + if "match_filter" in self.download_ytdl_options: + entry.add_kwargs({YTDL_SUB_MATCH_FILTER_REJECT: True}) + return entry + raise upload_date_idx = self._enhanced_download_archive.mapping.get_num_entries_with_upload_date( upload_date_standardized=entry.upload_date_standardized diff --git a/src/ytdl_sub/entries/variables/kwargs.py b/src/ytdl_sub/entries/variables/kwargs.py index 9e46062d..2adc3535 100644 --- a/src/ytdl_sub/entries/variables/kwargs.py +++ b/src/ytdl_sub/entries/variables/kwargs.py @@ -47,6 +47,7 @@ REQUESTED_SUBTITLES = _("requested_subtitles", backend=True) CHAPTERS = _("chapters", backend=True) YTDL_SUB_CUSTOM_CHAPTERS = _("ytdl_sub_custom_chapters", backend=True) YTDL_SUB_REGEX_SOURCE_VARS = _("ytdl_sub_regex_source_vars", backend=True) +YTDL_SUB_MATCH_FILTER_REJECT = _("ytdl_sub_match_filter_reject", backend=True) SPONSORBLOCK_CHAPTERS = _("sponsorblock_chapters", backend=True) SPLIT_BY_CHAPTERS_PARENT_ENTRY = _("split_by_chapters_parent_entry", backend=True) COMMENTS = _("comments", backend=True) diff --git a/src/ytdl_sub/plugins/match_filters.py b/src/ytdl_sub/plugins/match_filters.py index 2ad8aa1a..27e9dce6 100644 --- a/src/ytdl_sub/plugins/match_filters.py +++ b/src/ytdl_sub/plugins/match_filters.py @@ -5,8 +5,11 @@ from typing import Optional from yt_dlp import match_filter_func +from ytdl_sub.entries.entry import Entry +from ytdl_sub.entries.variables.kwargs import YTDL_SUB_MATCH_FILTER_REJECT from ytdl_sub.plugins.plugin import Plugin from ytdl_sub.plugins.plugin import PluginOptions +from ytdl_sub.plugins.plugin import PluginPriority from ytdl_sub.utils.logger import Logger from ytdl_sub.validators.validators import StringListValidator @@ -63,6 +66,7 @@ class MatchFiltersOptions(PluginOptions): class MatchFiltersPlugin(Plugin[MatchFiltersOptions]): plugin_options_type = MatchFiltersOptions + priority = PluginPriority(modify_entry=PluginPriority.MODIFY_ENTRY_FIRST) def ytdl_options(self) -> Optional[Dict]: """ @@ -75,4 +79,17 @@ class MatchFiltersPlugin(Plugin[MatchFiltersOptions]): logger.debug("Adding match-filter %s", filter_str) match_filters.append(filter_str) - return {"match_filter": match_filter_func(match_filters)} + return { + "match_filter": match_filter_func(match_filters), + } + + def modify_entry(self, entry: Entry) -> Optional[Entry]: + """ + If an entry is marked as not being downloaded due to a match_filter reject, + do not propagate the entry further (especially since there is no entry file!) + """ + if entry.kwargs_get(YTDL_SUB_MATCH_FILTER_REJECT, False): + logger.info("Entry rejected by match-filter, skipping ..") + return None + + return entry diff --git a/src/ytdl_sub/plugins/plugin.py b/src/ytdl_sub/plugins/plugin.py index ccf21acb..35488dfd 100644 --- a/src/ytdl_sub/plugins/plugin.py +++ b/src/ytdl_sub/plugins/plugin.py @@ -24,7 +24,9 @@ class PluginPriority: # If modify_entry priority is >= to this value, run after split MODIFY_ENTRY_AFTER_SPLIT = 10 - def __init__(self, modify_entry: int = 0, post_process: int = 0): + MODIFY_ENTRY_FIRST = 0 + + def __init__(self, modify_entry: int = 5, post_process: int = 5): self.modify_entry = modify_entry self.post_process = post_process diff --git a/src/ytdl_sub/subscriptions/subscription_ytdl_options.py b/src/ytdl_sub/subscriptions/subscription_ytdl_options.py index 961c6544..70daaecc 100644 --- a/src/ytdl_sub/subscriptions/subscription_ytdl_options.py +++ b/src/ytdl_sub/subscriptions/subscription_ytdl_options.py @@ -78,6 +78,10 @@ class SubscriptionYTDLOptions: "writeinfojson": True, } + @property + def _download_only_options(self) -> Dict: + return {"break_on_reject": True} + @property def _output_options(self) -> Dict: ytdl_options = {} @@ -110,7 +114,6 @@ class SubscriptionYTDLOptions: self._global_options, self._output_options, self._plugin_ytdl_options(DateRangePlugin), - self._plugin_ytdl_options(MatchFiltersPlugin), self._user_ytdl_options, # user ytdl options... self._info_json_only_options, # then info_json_only options ) @@ -125,12 +128,13 @@ class SubscriptionYTDLOptions: ytdl_options_builder = YTDLOptionsBuilder().add( self._global_options, self._output_options, - self._plugin_ytdl_options(DateRangePlugin), + self._plugin_ytdl_options(MatchFiltersPlugin), self._plugin_ytdl_options(FileConvertPlugin), self._plugin_ytdl_options(SubtitlesPlugin), self._plugin_ytdl_options(ChaptersPlugin), self._plugin_ytdl_options(AudioExtractPlugin), self._user_ytdl_options, # user ytdl options... + self._download_only_options, # then download_only options ) # Add dry run options last if enabled if self._dry_run: diff --git a/tests/e2e/plugins/test_match_filters.py b/tests/e2e/plugins/test_match_filters.py index 3f9cf029..8bcfb669 100644 --- a/tests/e2e/plugins/test_match_filters.py +++ b/tests/e2e/plugins/test_match_filters.py @@ -1,4 +1,6 @@ import pytest +from expected_download import assert_expected_downloads +from expected_transaction_log import assert_transaction_log_matches from ytdl_sub.subscriptions.subscription import Subscription @@ -17,9 +19,23 @@ def preset_dict(output_directory): } +@pytest.fixture +def playlist_preset_dict(output_directory): + return { + "preset": "music_video", + "download": {"url": "https://www.youtube.com/playlist?list=PL5BC0FC26BECA5A35"}, + "output_options": {"output_directory": output_directory}, + # download the worst format so it is fast + "ytdl_options": { + "postprocessor_args": {"ffmpeg": ["-bitexact"]}, # Must add this for reproducibility + }, + "match_filters": {"filters": ["view_count > 20000"]}, + } + + class TestFileConvert: @pytest.mark.parametrize("dry_run", [True, False]) - def test_match_filters( + def test_match_filters_empty( self, music_video_config, preset_dict, @@ -34,3 +50,31 @@ class TestFileConvert: transaction_log = subscription.download(dry_run=dry_run) assert transaction_log.is_empty + + @pytest.mark.parametrize("dry_run", [True, False]) + def test_match_filters_partial( + self, + music_video_config, + playlist_preset_dict, + output_directory, + dry_run, + ): + subscription = Subscription.from_dict( + config=music_video_config, + preset_name="match_filter_test", + preset_dict=playlist_preset_dict, + ) + + transaction_log = subscription.download(dry_run=dry_run) + + summary_path = "plugins/match_filters" + assert_transaction_log_matches( + output_directory=output_directory, + transaction_log=transaction_log, + transaction_log_summary_file_name=f"{summary_path}/test_match_filters_partial.txt", + ) + assert_expected_downloads( + output_directory=output_directory, + dry_run=dry_run, + expected_download_summary_file_name=f"{summary_path}/test_match_filters_partial.json", + ) diff --git a/tests/resources/expected_downloads_summaries/plugins/match_filters/test_match_filters_partial.json b/tests/resources/expected_downloads_summaries/plugins/match_filters/test_match_filters_partial.json new file mode 100644 index 00000000..fe968a5e --- /dev/null +++ b/tests/resources/expected_downloads_summaries/plugins/match_filters/test_match_filters_partial.json @@ -0,0 +1,7 @@ +{ + ".ytdl-sub-match_filter_test-download-archive.json": "6017105735dd8fdfe18932c9c048dccf", + "Project Zombie/Project Zombie - Jesse's Minecraft Server [Trailer - Mar.21]-thumb.jpg": "e7830aa8a64b0cde65ba3f7e5fc56530", + "Project Zombie/Project Zombie - Jesse's Minecraft Server [Trailer - Mar.21].info.json": "ddfda0c60aafd9779da692d6c165385b", + "Project Zombie/Project Zombie - Jesse's Minecraft Server [Trailer - Mar.21].nfo": "17c303f07e9b3af77c90975f843c95bd", + "Project Zombie/Project Zombie - Jesse's Minecraft Server [Trailer - Mar.21].webm": "ad90ebef1ea6cbdb924670cdd7d1b9b9" +} \ No newline at end of file diff --git a/tests/resources/transaction_log_summaries/plugins/match_filters/test_match_filters_partial.txt b/tests/resources/transaction_log_summaries/plugins/match_filters/test_match_filters_partial.txt new file mode 100644 index 00000000..4b827ab2 --- /dev/null +++ b/tests/resources/transaction_log_summaries/plugins/match_filters/test_match_filters_partial.txt @@ -0,0 +1,15 @@ +Files created: +---------------------------------------- +{output_directory} + .ytdl-sub-match_filter_test-download-archive.json +{output_directory}/Project Zombie + Project Zombie - Jesse's Minecraft Server [Trailer - Mar.21]-thumb.jpg + Project Zombie - Jesse's Minecraft Server [Trailer - Mar.21].info.json + Project Zombie - Jesse's Minecraft Server [Trailer - Mar.21].nfo + NFO tags: + musicvideo: + album: Music Videos + artist: Project Zombie + title: Jesse's Minecraft Server [Trailer - Mar.21] + year: 2011 + Project Zombie - Jesse's Minecraft Server [Trailer - Mar.21].webm \ No newline at end of file