From 3b3c23963e7d67ed44418e9a2e375eb6d1f43e92 Mon Sep 17 00:00:00 2001 From: jbannon Date: Wed, 10 Aug 2022 23:39:27 +0000 Subject: [PATCH] ytdl options builder --- src/ytdl_sub/config/preset.py | 2 + src/ytdl_sub/downloaders/downloader.py | 59 ++++++++----------- src/ytdl_sub/downloaders/youtube/channel.py | 5 +- .../downloaders/ytdl_options_builder.py | 59 +++++++++++++++++++ src/ytdl_sub/subscriptions/subscription.py | 6 +- .../subscription_ytdl_options.py | 54 +++++++++-------- tests/unit/entries/conftest.py | 2 + 7 files changed, 124 insertions(+), 63 deletions(-) create mode 100644 src/ytdl_sub/downloaders/ytdl_options_builder.py diff --git a/src/ytdl_sub/config/preset.py b/src/ytdl_sub/config/preset.py index 5b0e70de..210d291a 100644 --- a/src/ytdl_sub/config/preset.py +++ b/src/ytdl_sub/config/preset.py @@ -79,6 +79,8 @@ class DownloadStrategyValidator(StrictDictValidator): raise self._validation_exception(error_message=value_exc) +# Preset is the meat of ytdl-sub, the linter is wrong here +# pylint: disable=too-many-instance-attributes class Preset(StrictDictValidator): # Have all present keys optional since parent presets could not have all the # required keys. They will get validated in the init after the mergedeep of dicts diff --git a/src/ytdl_sub/downloaders/downloader.py b/src/ytdl_sub/downloaders/downloader.py index d7fe00ba..7fb7545a 100644 --- a/src/ytdl_sub/downloaders/downloader.py +++ b/src/ytdl_sub/downloaders/downloader.py @@ -1,6 +1,5 @@ import abc import contextlib -import copy import json import os import time @@ -22,6 +21,7 @@ from yt_dlp.utils import ExistingVideoReached from yt_dlp.utils import RejectedVideoReached from ytdl_sub.config.preset_options import Overrides +from ytdl_sub.downloaders.ytdl_options_builder import YTDLOptionsBuilder from ytdl_sub.entries.base_entry import BaseEntry from ytdl_sub.entries.entry import Entry from ytdl_sub.thread.log_entries_downloaded_listener import LogEntriesDownloadedListener @@ -70,25 +70,11 @@ class Downloader(DownloadArchiver, Generic[DownloaderOptionsT, DownloaderEntryT] """ return {"ignoreerrors": True} - @classmethod - def _configure_ytdl_options( - cls, - ytdl_options: Optional[Dict], - ) -> Dict: - """Configure the ytdl options for the downloader""" - if ytdl_options is None: - ytdl_options = {} - - # Overwrite defaults with input - ytdl_options = dict(cls.ytdl_option_defaults(), **ytdl_options) - - return ytdl_options - def __init__( self, download_options: DownloaderOptionsT, enhanced_download_archive: EnhancedDownloadArchive, - ytdl_options: Optional[Dict] = None, + ytdl_options_builder: YTDLOptionsBuilder, ): """ Parameters @@ -97,13 +83,14 @@ class Downloader(DownloadArchiver, Generic[DownloaderOptionsT, DownloaderEntryT] Options validator for this downloader enhanced_download_archive Download archive - ytdl_options - YTDL options validator + ytdl_options_builder + YTDL options builder """ DownloadArchiver.__init__(self=self, enhanced_download_archive=enhanced_download_archive) self.download_options = download_options - self.ytdl_options = self._configure_ytdl_options( - ytdl_options=ytdl_options, + + self._ytdl_options_builder = ytdl_options_builder.clone().add( + self.ytdl_option_defaults(), before=True ) @contextmanager @@ -111,9 +98,7 @@ class Downloader(DownloadArchiver, Generic[DownloaderOptionsT, DownloaderEntryT] """ Context manager to interact with yt_dlp. """ - ytdl_options = self.ytdl_options - if ytdl_options_overrides is not None: - ytdl_options = dict(ytdl_options, **ytdl_options_overrides) + ytdl_options = self._ytdl_options_builder.clone().add(ytdl_options_overrides).to_dict() download_logger.debug("ytdl_options: %s", str(ytdl_options)) with Logger.handle_external_logs(name="yt-dlp"): @@ -127,7 +112,7 @@ class Downloader(DownloadArchiver, Generic[DownloaderOptionsT, DownloaderEntryT] ------- True if dry-run is enabled. False otherwise. """ - return self.ytdl_options.get("skip_download", False) + return self._ytdl_options_builder.to_dict().get("skip_download", False) def extract_info(self, ytdl_options_overrides: Optional[Dict] = None, **kwargs) -> Dict: """ @@ -173,7 +158,6 @@ class Downloader(DownloadArchiver, Generic[DownloaderOptionsT, DownloaderEntryT] """ num_tries = 0 entry_files_exist = False - ytdl_options_overrides = copy.deepcopy(ytdl_options_overrides) while not entry_files_exist and num_tries < self._extract_entry_num_retries: entry_dict = self.extract_info(ytdl_options_overrides=ytdl_options_overrides, **kwargs) @@ -185,7 +169,11 @@ class Downloader(DownloadArchiver, Generic[DownloaderOptionsT, DownloaderEntryT] # Remove the download archive so it can retry without thinking its already downloaded, # even though it is not - ytdl_options_overrides["download_archive"] = None + ytdl_options_overrides = ( + YTDLOptionsBuilder() + .add(ytdl_options_overrides, {"download_archive": None}) + .to_dict() + ) if num_tries < self._extract_entry_retry_wait_sec: download_logger.debug( @@ -299,20 +287,25 @@ class Downloader(DownloadArchiver, Generic[DownloaderOptionsT, DownloaderEntryT] **kwargs arguments passed directory to YoutubeDL extract_info """ + ytdl_options_builder = self._ytdl_options_builder.clone() if ytdl_options_overrides is None: ytdl_options_overrides = {} - extract_info_ytdl_options = {"writeinfojson": True} + ytdl_options_builder.add({"writeinfojson": True}, ytdl_options_overrides) if only_info_json: - extract_info_ytdl_options["skip_download"] = True - extract_info_ytdl_options["writethumbnail"] = False - extract_info_ytdl_options["writesubtitles"] = False - - ytdl_options_overrides = dict(ytdl_options_overrides, **extract_info_ytdl_options) + ytdl_options_builder.add( + { + "skip_download": True, + "writethumbnail": False, + "writesubtitles": False, + } + ) try: with self._listen_and_log_downloaded_info_json(log_prefix=log_prefix_on_info_json_dl): - _ = self.extract_info(ytdl_options_overrides=ytdl_options_overrides, **kwargs) + _ = self.extract_info( + ytdl_options_overrides=ytdl_options_builder.to_dict(), **kwargs + ) except RejectedVideoReached: download_logger.debug("RejectedVideoReached, stopping additional downloads") except ExistingVideoReached: diff --git a/src/ytdl_sub/downloaders/youtube/channel.py b/src/ytdl_sub/downloaders/youtube/channel.py index f132393b..e84056dd 100644 --- a/src/ytdl_sub/downloaders/youtube/channel.py +++ b/src/ytdl_sub/downloaders/youtube/channel.py @@ -9,6 +9,7 @@ from ytdl_sub.downloaders.downloader import DownloaderOptionsT from ytdl_sub.downloaders.downloader import download_logger from ytdl_sub.downloaders.youtube.abc import YoutubeDownloader from ytdl_sub.downloaders.youtube.abc import YoutubeDownloaderOptions +from ytdl_sub.downloaders.ytdl_options_builder import YTDLOptionsBuilder from ytdl_sub.entries.youtube import YoutubeChannel from ytdl_sub.entries.youtube import YoutubeVideo from ytdl_sub.utils.thumbnail import convert_url_thumbnail @@ -115,12 +116,12 @@ class YoutubeChannelDownloader(YoutubeDownloader[YoutubeChannelDownloaderOptions self, download_options: DownloaderOptionsT, enhanced_download_archive: EnhancedDownloadArchive, - ytdl_options: Optional[Dict] = None, + ytdl_options_builder: YTDLOptionsBuilder, ): super().__init__( download_options=download_options, enhanced_download_archive=enhanced_download_archive, - ytdl_options=ytdl_options, + ytdl_options_builder=ytdl_options_builder, ) self.channel: Optional[YoutubeChannel] = None diff --git a/src/ytdl_sub/downloaders/ytdl_options_builder.py b/src/ytdl_sub/downloaders/ytdl_options_builder.py new file mode 100644 index 00000000..b159da42 --- /dev/null +++ b/src/ytdl_sub/downloaders/ytdl_options_builder.py @@ -0,0 +1,59 @@ +import copy +from typing import Dict +from typing import Optional + +import mergedeep + + +class YTDLOptionsBuilder: + """ + A class to track any modifications made to ytdl options + """ + + def __init__(self): + self._ytdl_options: Dict = {} + + def add( + self, + *ytdl_option_dicts: Optional[Dict], + before: bool = False, + strategy: mergedeep.Strategy = mergedeep.Strategy.TYPESAFE_ADDITIVE + ) -> "YTDLOptionsBuilder": + """ + Parameters + ---------- + *ytdl_option_dicts + One or many ytdl_option dicts. Can also contain None's for convenience + before + Optional. Whether to add these dicts before or after the original + strategy + Optional. mergedeep strategy. Defaults to TYPESAFE_ADDITIVE + + Returns + ------- + instance with the added ytdl_option dict(s) + """ + non_empty = [ytdl_options for ytdl_options in ytdl_option_dicts if ytdl_options is not None] + + if before: + non_empty.append(self.to_dict()) + self._ytdl_options = {} + + mergedeep.merge(self._ytdl_options, *non_empty, strategy=strategy) + return self + + def clone(self) -> "YTDLOptionsBuilder": + """ + Returns + ------- + Deep-copied instance + """ + return copy.deepcopy(self) + + def to_dict(self) -> Dict: + """ + Returns + ------- + Deep-copied dict of the current builder state + """ + return copy.deepcopy(self._ytdl_options) diff --git a/src/ytdl_sub/subscriptions/subscription.py b/src/ytdl_sub/subscriptions/subscription.py index 28de6e94..2e69a850 100644 --- a/src/ytdl_sub/subscriptions/subscription.py +++ b/src/ytdl_sub/subscriptions/subscription.py @@ -287,19 +287,19 @@ class Subscription: directory. """ self._enhanced_download_archive.reinitialize(dry_run=dry_run) - ytdl_options = SubscriptionYTDLOptions( + ytdl_options_builder = SubscriptionYTDLOptions( preset=self.__preset_options, enhanced_download_archive=self._enhanced_download_archive, working_directory=self.working_directory, dry_run=dry_run, - ).to_dict() + ).builder() plugins = self._initialize_plugins() with self._subscription_download_context_managers(): downloader = self.downloader_class( download_options=self.downloader_options, enhanced_download_archive=self._enhanced_download_archive, - ytdl_options=ytdl_options, + ytdl_options_builder=ytdl_options_builder, ) for entry in downloader.download(): diff --git a/src/ytdl_sub/subscriptions/subscription_ytdl_options.py b/src/ytdl_sub/subscriptions/subscription_ytdl_options.py index c8752149..ff1f4433 100644 --- a/src/ytdl_sub/subscriptions/subscription_ytdl_options.py +++ b/src/ytdl_sub/subscriptions/subscription_ytdl_options.py @@ -1,9 +1,8 @@ from pathlib import Path from typing import Dict -import mergedeep - from ytdl_sub.config.preset import Preset +from ytdl_sub.downloaders.ytdl_options_builder import YTDLOptionsBuilder from ytdl_sub.ytdl_additions.enhanced_download_archive import EnhancedDownloadArchive @@ -22,11 +21,26 @@ class SubscriptionYTDLOptions: @property def _global_options(self) -> Dict: - return { + """ + Returns + ------- + ytdl-options to apply to every run no matter what + """ + ytdl_options = { # Download all files in the format of {id}.{ext} "outtmpl": str(Path(self._working_directory) / "%(id)s.%(ext)s") } + if ( + self._preset.downloader.supports_download_archive + and self._preset.output_options.maintain_download_archive + ): + ytdl_options["download_archive"] = str( + Path(self._working_directory) / self._enhanced_download_archive.archive_file_name + ) + + return ytdl_options + @property def _dry_run_options(self) -> Dict: return { @@ -42,13 +56,6 @@ class SubscriptionYTDLOptions: if output_options.thumbnail_name: ytdl_options["writethumbnail"] = True - if ( - self._preset.downloader.supports_download_archive - and output_options.maintain_download_archive - ): - ytdl_options["download_archive"] = str( - Path(self._working_directory) / self._enhanced_download_archive.archive_file_name - ) return ytdl_options @@ -83,22 +90,19 @@ class SubscriptionYTDLOptions: def _user_ytdl_options(self) -> Dict: return self._preset.ytdl_options.dict - def to_dict(self) -> Dict: - ytdl_options = self._global_options + def builder(self) -> YTDLOptionsBuilder: + """ + Returns + ------- + YTDLOptionsBuilder + Builder with values set based on the subscription + """ + ytdl_options_builder = YTDLOptionsBuilder().add(self._global_options) if self._dry_run: - mergedeep.merge( - ytdl_options, - self._dry_run_options, - self._user_ytdl_options, - strategy=mergedeep.Strategy.TYPESAFE_ADDITIVE, - ) + ytdl_options_builder.add(self._dry_run_options, self._user_ytdl_options) else: - mergedeep.merge( - ytdl_options, - self._output_options, - self._subtitle_options, - self._user_ytdl_options, - strategy=mergedeep.Strategy.TYPESAFE_ADDITIVE, + ytdl_options_builder.add( + self._output_options, self._subtitle_options, self._user_ytdl_options ) - return ytdl_options + return ytdl_options_builder diff --git a/tests/unit/entries/conftest.py b/tests/unit/entries/conftest.py index 0ab92692..0bf2faf3 100644 --- a/tests/unit/entries/conftest.py +++ b/tests/unit/entries/conftest.py @@ -54,10 +54,12 @@ def ext(): def thumbnail_ext(): return "jpg" + @pytest.fixture def subtitles_ext(): return "srt" + @pytest.fixture def download_thumbnail_name(uid, thumbnail_ext): return f"{uid}.{thumbnail_ext}"