diff --git a/docs/config.rst b/docs/config.rst index b95a5957..b8aae8e4 100644 --- a/docs/config.rst +++ b/docs/config.rst @@ -193,6 +193,7 @@ match_filters ''''''''''''' .. autoclass:: ytdl_sub.plugins.match_filters.MatchFiltersOptions() :members: + :member-order: bysource :exclude-members: partial_validate ------------------------------------------------------------------------------- diff --git a/src/ytdl_sub/config/plugin.py b/src/ytdl_sub/config/plugin.py index 9ddc083c..6c3cac71 100644 --- a/src/ytdl_sub/config/plugin.py +++ b/src/ytdl_sub/config/plugin.py @@ -72,6 +72,15 @@ class Plugin(BasePlugin[TOptionsValidator], Generic[TOptionsValidator], ABC): Class to define the new plugin functionality """ + @classmethod + def default_ytdl_options(cls) -> Dict: + """ + Returns + ------- + ytdl options to enable if the plugin is not specified in the download + """ + return {} + def ytdl_options_match_filters(self) -> Tuple[List[str], List[str]]: """ Returns diff --git a/src/ytdl_sub/downloaders/info_json/info_json_downloader.py b/src/ytdl_sub/downloaders/info_json/info_json_downloader.py index f0c37083..a7fac294 100644 --- a/src/ytdl_sub/downloaders/info_json/info_json_downloader.py +++ b/src/ytdl_sub/downloaders/info_json/info_json_downloader.py @@ -4,6 +4,7 @@ from pathlib import Path from typing import Dict from typing import Iterable from typing import List +from typing import Optional from ytdl_sub.config.preset_options import OptionsDictValidator from ytdl_sub.config.preset_options import Overrides @@ -123,7 +124,7 @@ class InfoJsonDownloader(SourcePlugin[InfoJsonDownloaderOptions]): ): self._enhanced_download_archive.num_entries_removed += 1 - def download(self, entry: Entry) -> Entry: + def download(self, entry: Entry) -> Optional[Entry]: """ Mock the download by copying the entry file from the output directory into the working directory diff --git a/src/ytdl_sub/downloaders/source_plugin.py b/src/ytdl_sub/downloaders/source_plugin.py index 59958ba1..b11b1940 100644 --- a/src/ytdl_sub/downloaders/source_plugin.py +++ b/src/ytdl_sub/downloaders/source_plugin.py @@ -56,7 +56,7 @@ class SourcePlugin(BasePlugin[TOptionsValidator], Generic[TOptionsValidator], AB """Gathers metadata of all entries to download""" @abc.abstractmethod - def download(self, entry: Entry) -> Entry: + def download(self, entry: Entry) -> Optional[Entry]: """The function to perform the download of all media entries""" @final diff --git a/src/ytdl_sub/downloaders/url/downloader.py b/src/ytdl_sub/downloaders/url/downloader.py index 0032ca8f..e97f4223 100644 --- a/src/ytdl_sub/downloaders/url/downloader.py +++ b/src/ytdl_sub/downloaders/url/downloader.py @@ -9,6 +9,8 @@ from typing import Optional from typing import Set from typing import Tuple +from yt_dlp.utils import RejectedVideoReached + from ytdl_sub.config.plugin import PluginPriority from ytdl_sub.config.preset_options import Overrides from ytdl_sub.downloaders.source_plugin import SourcePlugin @@ -459,7 +461,7 @@ class MultiUrlDownloader(SourcePlugin[MultiUrlValidator]): ) yield entry - def download(self, entry: Entry) -> Entry: + def download(self, entry: Entry) -> Optional[Entry]: """ Parameters ---------- @@ -482,7 +484,13 @@ class MultiUrlDownloader(SourcePlugin[MultiUrlValidator]): entry.title, ) - download_entry = self._extract_entry_info_with_retry(entry=entry) + # Match-filters can be applied at the download stage. If the download is rejected, + # then return None + try: + download_entry = self._extract_entry_info_with_retry(entry=entry) + except RejectedVideoReached: + download_logger.info("Entry rejected by download match-filter, skipping ..") + return None upload_date_idx = self._enhanced_download_archive.mapping.get_num_entries_with_upload_date( upload_date_standardized=entry.upload_date_standardized diff --git a/src/ytdl_sub/plugins/match_filters.py b/src/ytdl_sub/plugins/match_filters.py index 62a4bd46..4e58d31f 100644 --- a/src/ytdl_sub/plugins/match_filters.py +++ b/src/ytdl_sub/plugins/match_filters.py @@ -1,7 +1,11 @@ from typing import Any +from typing import Dict from typing import List +from typing import Optional from typing import Tuple +from yt_dlp import match_filter_func + from ytdl_sub.config.plugin import Plugin from ytdl_sub.config.preset_options import OptionsDictValidator from ytdl_sub.utils.logger import Logger @@ -9,6 +13,8 @@ from ytdl_sub.validators.validators import StringListValidator logger = Logger.get("match_filters") +_DEFAULT_DOWNLOAD_MATCH_FILTERS: List[str] = ["!is_live & !is_upcoming & !post_live"] + class MatchFiltersOptions(OptionsDictValidator): """ @@ -40,40 +46,79 @@ class MatchFiltersOptions(OptionsDictValidator): # - "availability=?public" """ - _required_keys = {"filters"} + _optional_keys = {"filters", "download_filters"} @classmethod def partial_validate(cls, name: str, value: Any) -> None: """Ensure filters looks right""" if isinstance(value, dict): value["filters"] = value.get("filters", [""]) + value["download_filters"] = value.get("download_filters", [""]) _ = cls(name, value) def __init__(self, name, value): super().__init__(name, value) - self._filters = self._validate_key(key="filters", validator=StringListValidator).list + self._filters = self._validate_key_if_present(key="filters", validator=StringListValidator) + self._download_filters = self._validate_key( + key="download_filters", + validator=StringListValidator, + default=_DEFAULT_DOWNLOAD_MATCH_FILTERS, + ) @property def filters(self) -> List[str]: """ The filters themselves. If used multiple times, the filter matches if at least one of the conditions are met. For logical AND's between match filters, use the ``&`` operator in - a single match filter. + a single match filter. These are applied when gathering metadata. """ - return [validator.value for validator in self._filters] + return [validator.value for validator in self._filters.list] if self._filters else [] + + @property + def download_filters(self) -> List[str]: + """ + Filters to apply during the download stage. This can be useful when building presets + that contain match-filters that you do not want to conflict with metadata-based + match-filters since they act as logical OR's. + + By default, if no download_filters are present, then the filter + ``"!is_live & !is_upcoming & !post_live"`` is added. + """ + return [validator.value for validator in self._download_filters.list] class MatchFiltersPlugin(Plugin[MatchFiltersOptions]): plugin_options_type = MatchFiltersOptions + @classmethod + def default_ytdl_options(cls) -> Dict: + """ + Returns + ------- + match-filter to filter out live + upcoming videos when downloading + """ + return { + "match_filter": match_filter_func( + filters=[], breaking_filters=_DEFAULT_DOWNLOAD_MATCH_FILTERS + ), + } + def ytdl_options_match_filters(self) -> Tuple[List[str], List[str]]: """ Returns ------- - match_filter after calling the utility function for it + match_filters to apply at the metadata stage """ - match_filters: List[str] = [] - for filter_str in self.plugin_options.filters: - match_filters.append(filter_str) + return self.plugin_options.filters, [] - return match_filters, [] + def ytdl_options(self) -> Optional[Dict]: + """ + Returns + ------- + match_filters to apply at the download stage + """ + return { + "match_filter": match_filter_func( + filters=[], breaking_filters=self.plugin_options.download_filters + ), + } diff --git a/src/ytdl_sub/subscriptions/subscription_download.py b/src/ytdl_sub/subscriptions/subscription_download.py index 60d580bb..4e6eebcc 100644 --- a/src/ytdl_sub/subscriptions/subscription_download.py +++ b/src/ytdl_sub/subscriptions/subscription_download.py @@ -295,6 +295,9 @@ class SubscriptionDownload(BaseSubscription, ABC): continue entry = downloader.download(entry) + if entry is None: + continue + entry_metadata = FileMetadata() if isinstance(entry, tuple): entry, entry_metadata = entry diff --git a/src/ytdl_sub/subscriptions/subscription_ytdl_options.py b/src/ytdl_sub/subscriptions/subscription_ytdl_options.py index a2db8403..df6fb742 100644 --- a/src/ytdl_sub/subscriptions/subscription_ytdl_options.py +++ b/src/ytdl_sub/subscriptions/subscription_ytdl_options.py @@ -14,6 +14,7 @@ from ytdl_sub.plugins.audio_extract import AudioExtractPlugin from ytdl_sub.plugins.chapters import ChaptersPlugin from ytdl_sub.plugins.file_convert import FileConvertPlugin from ytdl_sub.plugins.format import FormatPlugin +from ytdl_sub.plugins.match_filters import MatchFiltersPlugin from ytdl_sub.plugins.subtitles import SubtitlesPlugin from ytdl_sub.utils.ffmpeg import FFMPEG from ytdl_sub.ytdl_additions.enhanced_download_archive import EnhancedDownloadArchive @@ -88,10 +89,10 @@ class SubscriptionYTDLOptions: return ytdl_options def _plugin_ytdl_options(self, plugin: Type[PluginT]) -> Dict: - if not (audio_extract_plugin := self._get_plugin(plugin)): - return {} + if not (plugin_obj := self._get_plugin(plugin)): + return plugin.default_ytdl_options() - return audio_extract_plugin.ytdl_options() + return plugin_obj.ytdl_options() @property def _user_ytdl_options(self) -> Dict: @@ -144,6 +145,7 @@ class SubscriptionYTDLOptions: self._plugin_ytdl_options(ChaptersPlugin), self._plugin_ytdl_options(AudioExtractPlugin), self._plugin_ytdl_options(FormatPlugin), + self._plugin_ytdl_options(MatchFiltersPlugin), self._user_ytdl_options, # user ytdl options... self._download_only_options, # then download_only options ) diff --git a/tests/e2e/plugins/test_match_filters.py b/tests/e2e/plugins/test_match_filters.py index 6e07e9a4..37b86f39 100644 --- a/tests/e2e/plugins/test_match_filters.py +++ b/tests/e2e/plugins/test_match_filters.py @@ -11,7 +11,6 @@ def preset_dict(output_directory): "preset": "music_video", "download": "https://www.youtube.com/watch?v=2zYF9JLHDmA", "output_options": {"output_directory": output_directory}, - # download the worst format so it is fast "ytdl_options": { "postprocessor_args": {"ffmpeg": ["-bitexact"]}, # Must add this for reproducibility }, @@ -25,7 +24,6 @@ def playlist_preset_dict(output_directory): "preset": "music_video", "download": "https://www.youtube.com/playlist?list=PL5BC0FC26BECA5A35", "output_options": {"output_directory": output_directory}, - # download the worst format so it is fast "ytdl_options": { "postprocessor_args": {"ffmpeg": ["-bitexact"]}, # Must add this for reproducibility }, @@ -33,7 +31,34 @@ def playlist_preset_dict(output_directory): } +@pytest.fixture +def livestream_preset_dict(output_directory): + return { + "preset": "music_video", + "download": "https://www.youtube.com/watch?v=DoUOrTJbIu4", + "output_options": {"output_directory": output_directory}, + "ytdl_options": { + "postprocessor_args": {"ffmpeg": ["-bitexact"]}, + }, + } + + class TestFileConvert: + def test_livestreams_download_filtered( + self, + music_video_config, + livestream_preset_dict, + output_directory, + ): + subscription = Subscription.from_dict( + config=music_video_config, + preset_name="match_filter_test", + preset_dict=livestream_preset_dict, + ) + + transaction_log = subscription.download(dry_run=False) + assert transaction_log.is_empty + @pytest.mark.parametrize("dry_run", [True, False]) def test_match_filters_empty( self,