From 9b3533b1bc3d5eba637772675b37a339162d40c2 Mon Sep 17 00:00:00 2001 From: Jesse Bannon Date: Tue, 19 Sep 2023 10:34:27 -0700 Subject: [PATCH] [BACKEND] Use match-filters for date_range plugin --- src/ytdl_sub/config/plugin.py | 8 +++++ src/ytdl_sub/plugins/date_range.py | 31 ++++++++----------- src/ytdl_sub/plugins/match_filters.py | 11 ++----- .../subscription_ytdl_options.py | 23 +++++++++++--- 4 files changed, 43 insertions(+), 30 deletions(-) diff --git a/src/ytdl_sub/config/plugin.py b/src/ytdl_sub/config/plugin.py index 6930e632..9ddc083c 100644 --- a/src/ytdl_sub/config/plugin.py +++ b/src/ytdl_sub/config/plugin.py @@ -72,6 +72,14 @@ class Plugin(BasePlugin[TOptionsValidator], Generic[TOptionsValidator], ABC): Class to define the new plugin functionality """ + def ytdl_options_match_filters(self) -> Tuple[List[str], List[str]]: + """ + Returns + ------- + Tuple of match-filters to apply, first one being non-breaking, second breaking + """ + return [], [] + def ytdl_options(self) -> Optional[Dict]: """ Returns diff --git a/src/ytdl_sub/plugins/date_range.py b/src/ytdl_sub/plugins/date_range.py index 3ad3932b..ca86e27b 100644 --- a/src/ytdl_sub/plugins/date_range.py +++ b/src/ytdl_sub/plugins/date_range.py @@ -1,10 +1,9 @@ -from typing import Dict +from typing import List from typing import Optional +from typing import Tuple from ytdl_sub.config.plugin import Plugin from ytdl_sub.config.preset_options import OptionsDictValidator -from ytdl_sub.downloaders.ytdl_options_builder import YTDLOptionsBuilder -from ytdl_sub.utils.datetime import to_date_range from ytdl_sub.validators.string_datetime import StringDatetimeValidator @@ -48,25 +47,21 @@ class DateRangeOptions(OptionsDictValidator): class DateRangePlugin(Plugin[DateRangeOptions]): plugin_options_type = DateRangeOptions - def ytdl_options(self) -> Optional[Dict]: + def ytdl_options_match_filters(self) -> Tuple[List[str], List[str]]: """ Returns ------- - YTDL options for setting a date range + match filters for before (non-breaking) and after (breaking) """ - ytdl_options_builder = YTDLOptionsBuilder() + match_filters: List[str] = [] + breaking_match_filters: List[str] = [] - source_date_range = to_date_range( - before=self.plugin_options.before, - after=self.plugin_options.after, - overrides=self.overrides, - ) - if source_date_range: - ytdl_options_builder.add({"daterange": source_date_range}) + if self.plugin_options.before: + before_str = self.overrides.apply_formatter(formatter=self.plugin_options.before) + match_filters.append(f"upload_date < {before_str}") - # Only add break_on_reject if after is specified, but not before. - # Otherwise, it can break on first metadata pull if it's after the 'before' - if self.plugin_options.after and not self.plugin_options.before: - ytdl_options_builder.add({"break_on_reject": True}) + if self.plugin_options.after: + after_str = self.overrides.apply_formatter(formatter=self.plugin_options.after) + breaking_match_filters.append(f"upload_date >= {after_str}") - return ytdl_options_builder.to_dict() + return match_filters, breaking_match_filters diff --git a/src/ytdl_sub/plugins/match_filters.py b/src/ytdl_sub/plugins/match_filters.py index dda5f370..d8c24cc2 100644 --- a/src/ytdl_sub/plugins/match_filters.py +++ b/src/ytdl_sub/plugins/match_filters.py @@ -1,9 +1,7 @@ from typing import Any -from typing import Dict from typing import List from typing import Optional - -from yt_dlp import match_filter_func +from typing import Tuple from ytdl_sub.config.plugin import Plugin from ytdl_sub.config.plugin import PluginPriority @@ -73,7 +71,7 @@ class MatchFiltersPlugin(Plugin[MatchFiltersOptions]): plugin_options_type = MatchFiltersOptions priority = PluginPriority(modify_entry=PluginPriority.MODIFY_ENTRY_FIRST) - def ytdl_options(self) -> Optional[Dict]: + def ytdl_options_match_filters(self) -> Tuple[List[str], List[str]]: """ Returns ------- @@ -81,12 +79,9 @@ class MatchFiltersPlugin(Plugin[MatchFiltersOptions]): """ match_filters: List[str] = [] for filter_str in self.plugin_options.filters: - logger.debug("Adding match-filter %s", filter_str) match_filters.append(filter_str) - return { - "match_filter": match_filter_func(match_filters), - } + return match_filters, [] def modify_entry(self, entry: Entry) -> Optional[Entry]: """ diff --git a/src/ytdl_sub/subscriptions/subscription_ytdl_options.py b/src/ytdl_sub/subscriptions/subscription_ytdl_options.py index ab492a85..a2db8403 100644 --- a/src/ytdl_sub/subscriptions/subscription_ytdl_options.py +++ b/src/ytdl_sub/subscriptions/subscription_ytdl_options.py @@ -5,15 +5,15 @@ from typing import Optional from typing import Type from typing import TypeVar +from yt_dlp import match_filter_func + from ytdl_sub.config.plugin import Plugin from ytdl_sub.config.preset import Preset from ytdl_sub.downloaders.ytdl_options_builder import YTDLOptionsBuilder from ytdl_sub.plugins.audio_extract import AudioExtractPlugin from ytdl_sub.plugins.chapters import ChaptersPlugin -from ytdl_sub.plugins.date_range import DateRangePlugin from ytdl_sub.plugins.file_convert import FileConvertPlugin from ytdl_sub.plugins.format import FormatPlugin -from ytdl_sub.plugins.match_filters import MatchFiltersPlugin from ytdl_sub.plugins.subtitles import SubtitlesPlugin from ytdl_sub.utils.ffmpeg import FFMPEG from ytdl_sub.ytdl_additions.enhanced_download_archive import EnhancedDownloadArchive @@ -97,6 +97,22 @@ class SubscriptionYTDLOptions: def _user_ytdl_options(self) -> Dict: return self._preset.ytdl_options.dict + @property + def _plugin_match_filters(self) -> Dict: + match_filters: List[str] = [] + breaking_match_filters: List[str] = [] + for plugin in self._plugins: + pl_match_filters, pl_breaking_match_filters = plugin.ytdl_options_match_filters() + + match_filters.extend(pl_match_filters) + breaking_match_filters.extend(pl_breaking_match_filters) + + return { + "match_filter": match_filter_func( + filters=match_filters, breaking_filters=breaking_match_filters + ) + } + def metadata_builder(self) -> YTDLOptionsBuilder: """ Returns @@ -107,7 +123,7 @@ class SubscriptionYTDLOptions: return YTDLOptionsBuilder().add( self._global_options, self._output_options, - self._plugin_ytdl_options(DateRangePlugin), + self._plugin_match_filters, self._plugin_ytdl_options(FormatPlugin), self._user_ytdl_options, # user ytdl options... self._info_json_only_options, # then info_json_only options @@ -123,7 +139,6 @@ class SubscriptionYTDLOptions: ytdl_options_builder = YTDLOptionsBuilder().add( self._global_options, self._output_options, - self._plugin_ytdl_options(MatchFiltersPlugin), self._plugin_ytdl_options(FileConvertPlugin), self._plugin_ytdl_options(SubtitlesPlugin), self._plugin_ytdl_options(ChaptersPlugin),