From 1a118e225af83f1770c4a8f0e7bd93a20cd70127 Mon Sep 17 00:00:00 2001 From: Jesse Bannon Date: Tue, 19 Sep 2023 11:19:15 -0700 Subject: [PATCH] [BACKEND] Perform match-filters at metadata stage, use match-filters for `date_range` plugin (#725) Solved Issue: https://github.com/jmbannon/ytdl-sub/issues/706 From our discussion with yt-dlp folks (https://github.com/yt-dlp/yt-dlp/issues/8108), we learned that best practices for date--range filtering is done using match-filters. We now set the `date_range` plugin values to be a 'breaking' match filter, and any custom ones specified from users in the `match_filters` plugin as non-breaking. This means match-filters can be performed at the metadata stage, which should yield a significant speedup Relevant docs: - date_range: https://ytdl-sub.readthedocs.io/en/latest/config.html#date-range - match_filters: https://ytdl-sub.readthedocs.io/en/latest/config.html#match-filters --- examples/tv_show_config.yaml | 4 --- src/ytdl_sub/config/plugin.py | 8 +++++ src/ytdl_sub/downloaders/url/downloader.py | 14 +-------- src/ytdl_sub/entries/variables/kwargs.py | 1 - src/ytdl_sub/plugins/date_range.py | 31 ++++++++----------- src/ytdl_sub/plugins/match_filters.py | 27 ++-------------- .../subscription_ytdl_options.py | 23 +++++++++++--- 7 files changed, 44 insertions(+), 64 deletions(-) diff --git a/examples/tv_show_config.yaml b/examples/tv_show_config.yaml index e2763980..f81ef60e 100644 --- a/examples/tv_show_config.yaml +++ b/examples/tv_show_config.yaml @@ -69,10 +69,6 @@ presets: date_range: after: "today-{download_range}" - # Stops fetching metadata if the video is out of range (saves time + bandwidth) - ytdl_options: - break_on_reject: True - # Deletes any videos older than download_range. WARNING: do not use # "season_by_year__episode_by_download_index" if you plan to delete older videos output_options: diff --git a/src/ytdl_sub/config/plugin.py b/src/ytdl_sub/config/plugin.py index 6930e632..9ddc083c 100644 --- a/src/ytdl_sub/config/plugin.py +++ b/src/ytdl_sub/config/plugin.py @@ -72,6 +72,14 @@ class Plugin(BasePlugin[TOptionsValidator], Generic[TOptionsValidator], ABC): Class to define the new plugin functionality """ + def ytdl_options_match_filters(self) -> Tuple[List[str], List[str]]: + """ + Returns + ------- + Tuple of match-filters to apply, first one being non-breaking, second breaking + """ + return [], [] + def ytdl_options(self) -> Optional[Dict]: """ Returns diff --git a/src/ytdl_sub/downloaders/url/downloader.py b/src/ytdl_sub/downloaders/url/downloader.py index 5cbadc4f..0032ca8f 100644 --- a/src/ytdl_sub/downloaders/url/downloader.py +++ b/src/ytdl_sub/downloaders/url/downloader.py @@ -9,8 +9,6 @@ from typing import Optional from typing import Set from typing import Tuple -from yt_dlp.utils import RejectedVideoReached - from ytdl_sub.config.plugin import PluginPriority from ytdl_sub.config.preset_options import Overrides from ytdl_sub.downloaders.source_plugin import SourcePlugin @@ -30,7 +28,6 @@ from ytdl_sub.entries.variables.kwargs import REQUESTED_SUBTITLES from ytdl_sub.entries.variables.kwargs import SOURCE_ENTRY from ytdl_sub.entries.variables.kwargs import SPONSORBLOCK_CHAPTERS from ytdl_sub.entries.variables.kwargs import UPLOAD_DATE_INDEX -from ytdl_sub.entries.variables.kwargs import YTDL_SUB_MATCH_FILTER_REJECT from ytdl_sub.utils.file_handler import FileHandler from ytdl_sub.utils.logger import Logger from ytdl_sub.utils.thumbnail import ThumbnailTypes @@ -485,16 +482,7 @@ class MultiUrlDownloader(SourcePlugin[MultiUrlValidator]): entry.title, ) - # Match-filters are applied at the download stage (not metadata stage). - # If the download is rejected, and match_filter is present in the ytdl options, - # then filter downstream in the match_filter plugin - try: - download_entry = self._extract_entry_info_with_retry(entry=entry) - except RejectedVideoReached: - if "match_filter" in self.download_ytdl_options: - entry.add_kwargs({YTDL_SUB_MATCH_FILTER_REJECT: True}) - return entry - raise + download_entry = self._extract_entry_info_with_retry(entry=entry) upload_date_idx = self._enhanced_download_archive.mapping.get_num_entries_with_upload_date( upload_date_standardized=entry.upload_date_standardized diff --git a/src/ytdl_sub/entries/variables/kwargs.py b/src/ytdl_sub/entries/variables/kwargs.py index 48cfa019..84f729f9 100644 --- a/src/ytdl_sub/entries/variables/kwargs.py +++ b/src/ytdl_sub/entries/variables/kwargs.py @@ -47,7 +47,6 @@ REQUESTED_SUBTITLES = _("requested_subtitles", backend=True) CHAPTERS = _("chapters", backend=True) YTDL_SUB_CUSTOM_CHAPTERS = _("ytdl_sub_custom_chapters", backend=True) YTDL_SUB_REGEX_SOURCE_VARS = _("ytdl_sub_regex_source_vars", backend=True) -YTDL_SUB_MATCH_FILTER_REJECT = _("ytdl_sub_match_filter_reject", backend=True) SPONSORBLOCK_CHAPTERS = _("sponsorblock_chapters", backend=True) SPLIT_BY_CHAPTERS_PARENT_ENTRY = _("split_by_chapters_parent_entry", backend=True) COMMENTS = _("comments", backend=True) diff --git a/src/ytdl_sub/plugins/date_range.py b/src/ytdl_sub/plugins/date_range.py index 3ad3932b..ca86e27b 100644 --- a/src/ytdl_sub/plugins/date_range.py +++ b/src/ytdl_sub/plugins/date_range.py @@ -1,10 +1,9 @@ -from typing import Dict +from typing import List from typing import Optional +from typing import Tuple from ytdl_sub.config.plugin import Plugin from ytdl_sub.config.preset_options import OptionsDictValidator -from ytdl_sub.downloaders.ytdl_options_builder import YTDLOptionsBuilder -from ytdl_sub.utils.datetime import to_date_range from ytdl_sub.validators.string_datetime import StringDatetimeValidator @@ -48,25 +47,21 @@ class DateRangeOptions(OptionsDictValidator): class DateRangePlugin(Plugin[DateRangeOptions]): plugin_options_type = DateRangeOptions - def ytdl_options(self) -> Optional[Dict]: + def ytdl_options_match_filters(self) -> Tuple[List[str], List[str]]: """ Returns ------- - YTDL options for setting a date range + match filters for before (non-breaking) and after (breaking) """ - ytdl_options_builder = YTDLOptionsBuilder() + match_filters: List[str] = [] + breaking_match_filters: List[str] = [] - source_date_range = to_date_range( - before=self.plugin_options.before, - after=self.plugin_options.after, - overrides=self.overrides, - ) - if source_date_range: - ytdl_options_builder.add({"daterange": source_date_range}) + if self.plugin_options.before: + before_str = self.overrides.apply_formatter(formatter=self.plugin_options.before) + match_filters.append(f"upload_date < {before_str}") - # Only add break_on_reject if after is specified, but not before. - # Otherwise, it can break on first metadata pull if it's after the 'before' - if self.plugin_options.after and not self.plugin_options.before: - ytdl_options_builder.add({"break_on_reject": True}) + if self.plugin_options.after: + after_str = self.overrides.apply_formatter(formatter=self.plugin_options.after) + breaking_match_filters.append(f"upload_date >= {after_str}") - return ytdl_options_builder.to_dict() + return match_filters, breaking_match_filters diff --git a/src/ytdl_sub/plugins/match_filters.py b/src/ytdl_sub/plugins/match_filters.py index dda5f370..62a4bd46 100644 --- a/src/ytdl_sub/plugins/match_filters.py +++ b/src/ytdl_sub/plugins/match_filters.py @@ -1,15 +1,9 @@ from typing import Any -from typing import Dict from typing import List -from typing import Optional - -from yt_dlp import match_filter_func +from typing import Tuple from ytdl_sub.config.plugin import Plugin -from ytdl_sub.config.plugin import PluginPriority from ytdl_sub.config.preset_options import OptionsDictValidator -from ytdl_sub.entries.entry import Entry -from ytdl_sub.entries.variables.kwargs import YTDL_SUB_MATCH_FILTER_REJECT from ytdl_sub.utils.logger import Logger from ytdl_sub.validators.validators import StringListValidator @@ -71,9 +65,8 @@ class MatchFiltersOptions(OptionsDictValidator): class MatchFiltersPlugin(Plugin[MatchFiltersOptions]): plugin_options_type = MatchFiltersOptions - priority = PluginPriority(modify_entry=PluginPriority.MODIFY_ENTRY_FIRST) - def ytdl_options(self) -> Optional[Dict]: + def ytdl_options_match_filters(self) -> Tuple[List[str], List[str]]: """ Returns ------- @@ -81,20 +74,6 @@ class MatchFiltersPlugin(Plugin[MatchFiltersOptions]): """ match_filters: List[str] = [] for filter_str in self.plugin_options.filters: - logger.debug("Adding match-filter %s", filter_str) match_filters.append(filter_str) - return { - "match_filter": match_filter_func(match_filters), - } - - def modify_entry(self, entry: Entry) -> Optional[Entry]: - """ - If an entry is marked as not being downloaded due to a match_filter reject, - do not propagate the entry further (especially since there is no entry file!) - """ - if entry.kwargs_get(YTDL_SUB_MATCH_FILTER_REJECT, False): - logger.info("Entry rejected by match-filter, skipping ..") - return None - - return entry + return match_filters, [] diff --git a/src/ytdl_sub/subscriptions/subscription_ytdl_options.py b/src/ytdl_sub/subscriptions/subscription_ytdl_options.py index ab492a85..a2db8403 100644 --- a/src/ytdl_sub/subscriptions/subscription_ytdl_options.py +++ b/src/ytdl_sub/subscriptions/subscription_ytdl_options.py @@ -5,15 +5,15 @@ from typing import Optional from typing import Type from typing import TypeVar +from yt_dlp import match_filter_func + from ytdl_sub.config.plugin import Plugin from ytdl_sub.config.preset import Preset from ytdl_sub.downloaders.ytdl_options_builder import YTDLOptionsBuilder from ytdl_sub.plugins.audio_extract import AudioExtractPlugin from ytdl_sub.plugins.chapters import ChaptersPlugin -from ytdl_sub.plugins.date_range import DateRangePlugin from ytdl_sub.plugins.file_convert import FileConvertPlugin from ytdl_sub.plugins.format import FormatPlugin -from ytdl_sub.plugins.match_filters import MatchFiltersPlugin from ytdl_sub.plugins.subtitles import SubtitlesPlugin from ytdl_sub.utils.ffmpeg import FFMPEG from ytdl_sub.ytdl_additions.enhanced_download_archive import EnhancedDownloadArchive @@ -97,6 +97,22 @@ class SubscriptionYTDLOptions: def _user_ytdl_options(self) -> Dict: return self._preset.ytdl_options.dict + @property + def _plugin_match_filters(self) -> Dict: + match_filters: List[str] = [] + breaking_match_filters: List[str] = [] + for plugin in self._plugins: + pl_match_filters, pl_breaking_match_filters = plugin.ytdl_options_match_filters() + + match_filters.extend(pl_match_filters) + breaking_match_filters.extend(pl_breaking_match_filters) + + return { + "match_filter": match_filter_func( + filters=match_filters, breaking_filters=breaking_match_filters + ) + } + def metadata_builder(self) -> YTDLOptionsBuilder: """ Returns @@ -107,7 +123,7 @@ class SubscriptionYTDLOptions: return YTDLOptionsBuilder().add( self._global_options, self._output_options, - self._plugin_ytdl_options(DateRangePlugin), + self._plugin_match_filters, self._plugin_ytdl_options(FormatPlugin), self._user_ytdl_options, # user ytdl options... self._info_json_only_options, # then info_json_only options @@ -123,7 +139,6 @@ class SubscriptionYTDLOptions: ytdl_options_builder = YTDLOptionsBuilder().add( self._global_options, self._output_options, - self._plugin_ytdl_options(MatchFiltersPlugin), self._plugin_ytdl_options(FileConvertPlugin), self._plugin_ytdl_options(SubtitlesPlugin), self._plugin_ytdl_options(ChaptersPlugin),