From 186e7ca73915d58a37519b0e3fa843a3041f60a3 Mon Sep 17 00:00:00 2001 From: Jesse Bannon Date: Wed, 17 May 2023 23:09:05 -0700 Subject: [PATCH] [FEATURE] Apply match-filter on download instead of metadata --- src/ytdl_sub/downloaders/url/downloader.py | 16 ++++++++++++++-- src/ytdl_sub/entries/variables/kwargs.py | 1 + src/ytdl_sub/plugins/match_filters.py | 17 +++++++++++++++-- src/ytdl_sub/plugins/plugin.py | 4 +++- .../subscriptions/subscription_ytdl_options.py | 8 +++++++- 5 files changed, 40 insertions(+), 6 deletions(-) diff --git a/src/ytdl_sub/downloaders/url/downloader.py b/src/ytdl_sub/downloaders/url/downloader.py index 1f9aeef3..42717eca 100644 --- a/src/ytdl_sub/downloaders/url/downloader.py +++ b/src/ytdl_sub/downloaders/url/downloader.py @@ -11,6 +11,8 @@ from typing import Optional from typing import Set from typing import Tuple +from yt_dlp.utils import RejectedVideoReached + from ytdl_sub.config.preset_options import Overrides from ytdl_sub.downloaders.base_downloader import BaseDownloader from ytdl_sub.downloaders.base_downloader import BaseDownloaderOptionsT @@ -23,7 +25,7 @@ from ytdl_sub.downloaders.ytdl_options_builder import YTDLOptionsBuilder from ytdl_sub.downloaders.ytdlp import YTDLP from ytdl_sub.entries.entry import Entry from ytdl_sub.entries.entry_parent import EntryParent -from ytdl_sub.entries.variables.kwargs import COLLECTION_URL +from ytdl_sub.entries.variables.kwargs import COLLECTION_URL, YTDL_SUB_MATCH_FILTER_REJECT from ytdl_sub.entries.variables.kwargs import COMMENTS from ytdl_sub.entries.variables.kwargs import DOWNLOAD_INDEX from ytdl_sub.entries.variables.kwargs import PLAYLIST_ENTRY @@ -530,7 +532,17 @@ class BaseUrlDownloader(BaseDownloader[BaseDownloaderOptionsT], ABC): self._url_state.entries_total, entry.title, ) - download_entry = self._extract_entry_info_with_retry(entry=entry) + + # Match-filters are applied at the download stage (not metadata stage). + # If the download is rejected, and match_filter is present in the ytdl options, + # then filter downstream in the match_filter plugin + try: + download_entry = self._extract_entry_info_with_retry(entry=entry) + except RejectedVideoReached: + if 'match_filter' in self.download_ytdl_options: + entry.add_kwargs({YTDL_SUB_MATCH_FILTER_REJECT: True}) + return entry + raise upload_date_idx = self._enhanced_download_archive.mapping.get_num_entries_with_upload_date( upload_date_standardized=entry.upload_date_standardized diff --git a/src/ytdl_sub/entries/variables/kwargs.py b/src/ytdl_sub/entries/variables/kwargs.py index 9e46062d..2adc3535 100644 --- a/src/ytdl_sub/entries/variables/kwargs.py +++ b/src/ytdl_sub/entries/variables/kwargs.py @@ -47,6 +47,7 @@ REQUESTED_SUBTITLES = _("requested_subtitles", backend=True) CHAPTERS = _("chapters", backend=True) YTDL_SUB_CUSTOM_CHAPTERS = _("ytdl_sub_custom_chapters", backend=True) YTDL_SUB_REGEX_SOURCE_VARS = _("ytdl_sub_regex_source_vars", backend=True) +YTDL_SUB_MATCH_FILTER_REJECT = _("ytdl_sub_match_filter_reject", backend=True) SPONSORBLOCK_CHAPTERS = _("sponsorblock_chapters", backend=True) SPLIT_BY_CHAPTERS_PARENT_ENTRY = _("split_by_chapters_parent_entry", backend=True) COMMENTS = _("comments", backend=True) diff --git a/src/ytdl_sub/plugins/match_filters.py b/src/ytdl_sub/plugins/match_filters.py index 2ad8aa1a..85334d84 100644 --- a/src/ytdl_sub/plugins/match_filters.py +++ b/src/ytdl_sub/plugins/match_filters.py @@ -5,7 +5,9 @@ from typing import Optional from yt_dlp import match_filter_func -from ytdl_sub.plugins.plugin import Plugin +from ytdl_sub.entries.entry import Entry +from ytdl_sub.entries.variables.kwargs import YTDL_SUB_MATCH_FILTER_REJECT +from ytdl_sub.plugins.plugin import Plugin, PluginPriority from ytdl_sub.plugins.plugin import PluginOptions from ytdl_sub.utils.logger import Logger from ytdl_sub.validators.validators import StringListValidator @@ -63,6 +65,9 @@ class MatchFiltersOptions(PluginOptions): class MatchFiltersPlugin(Plugin[MatchFiltersOptions]): plugin_options_type = MatchFiltersOptions + priority = PluginPriority( + modify_entry=PluginPriority.MODIFY_ENTRY_FIRST + ) def ytdl_options(self) -> Optional[Dict]: """ @@ -75,4 +80,12 @@ class MatchFiltersPlugin(Plugin[MatchFiltersOptions]): logger.debug("Adding match-filter %s", filter_str) match_filters.append(filter_str) - return {"match_filter": match_filter_func(match_filters)} + return { + "match_filter": match_filter_func(match_filters), + } + + def modify_entry(self, entry: Entry) -> Optional[Entry]: + if entry.kwargs_get(YTDL_SUB_MATCH_FILTER_REJECT, False): + return None + + return entry diff --git a/src/ytdl_sub/plugins/plugin.py b/src/ytdl_sub/plugins/plugin.py index ccf21acb..35488dfd 100644 --- a/src/ytdl_sub/plugins/plugin.py +++ b/src/ytdl_sub/plugins/plugin.py @@ -24,7 +24,9 @@ class PluginPriority: # If modify_entry priority is >= to this value, run after split MODIFY_ENTRY_AFTER_SPLIT = 10 - def __init__(self, modify_entry: int = 0, post_process: int = 0): + MODIFY_ENTRY_FIRST = 0 + + def __init__(self, modify_entry: int = 5, post_process: int = 5): self.modify_entry = modify_entry self.post_process = post_process diff --git a/src/ytdl_sub/subscriptions/subscription_ytdl_options.py b/src/ytdl_sub/subscriptions/subscription_ytdl_options.py index 961c6544..9149c49f 100644 --- a/src/ytdl_sub/subscriptions/subscription_ytdl_options.py +++ b/src/ytdl_sub/subscriptions/subscription_ytdl_options.py @@ -78,6 +78,12 @@ class SubscriptionYTDLOptions: "writeinfojson": True, } + @property + def _download_only_options(self) -> Dict: + return { + "break_on_reject": True + } + @property def _output_options(self) -> Dict: ytdl_options = {} @@ -125,12 +131,12 @@ class SubscriptionYTDLOptions: ytdl_options_builder = YTDLOptionsBuilder().add( self._global_options, self._output_options, - self._plugin_ytdl_options(DateRangePlugin), self._plugin_ytdl_options(FileConvertPlugin), self._plugin_ytdl_options(SubtitlesPlugin), self._plugin_ytdl_options(ChaptersPlugin), self._plugin_ytdl_options(AudioExtractPlugin), self._user_ytdl_options, # user ytdl options... + self._download_only_options, # then download_only options ) # Add dry run options last if enabled if self._dry_run: