ytdl options builder

src/ytdl_sub/config/preset.py

@@ -79,6 +79,8 @@ class DownloadStrategyValidator(StrictDictValidator):
             raise self._validation_exception(error_message=value_exc)
 
 
+# Preset is the meat of ytdl-sub, the linter is wrong here
+# pylint: disable=too-many-instance-attributes
 class Preset(StrictDictValidator):
     # Have all present keys optional since parent presets could not have all the
     # required keys. They will get validated in the init after the mergedeep of dicts

src/ytdl_sub/downloaders/downloader.py

@@ -1,6 +1,5 @@
 import abc
 import contextlib
-import copy
 import json
 import os
 import time
@@ -22,6 +21,7 @@ from yt_dlp.utils import ExistingVideoReached
 from yt_dlp.utils import RejectedVideoReached
 
 from ytdl_sub.config.preset_options import Overrides
+from ytdl_sub.downloaders.ytdl_options_builder import YTDLOptionsBuilder
 from ytdl_sub.entries.base_entry import BaseEntry
 from ytdl_sub.entries.entry import Entry
 from ytdl_sub.thread.log_entries_downloaded_listener import LogEntriesDownloadedListener
@@ -70,25 +70,11 @@ class Downloader(DownloadArchiver, Generic[DownloaderOptionsT, DownloaderEntryT]
         """
         return {"ignoreerrors": True}
 
-    @classmethod
-    def _configure_ytdl_options(
-        cls,
-        ytdl_options: Optional[Dict],
-    ) -> Dict:
-        """Configure the ytdl options for the downloader"""
-        if ytdl_options is None:
-            ytdl_options = {}
-
-        # Overwrite defaults with input
-        ytdl_options = dict(cls.ytdl_option_defaults(), **ytdl_options)
-
-        return ytdl_options
-
     def __init__(
         self,
         download_options: DownloaderOptionsT,
         enhanced_download_archive: EnhancedDownloadArchive,
-        ytdl_options: Optional[Dict] = None,
+        ytdl_options_builder: YTDLOptionsBuilder,
     ):
         """
         Parameters
@@ -97,13 +83,14 @@ class Downloader(DownloadArchiver, Generic[DownloaderOptionsT, DownloaderEntryT]
             Options validator for this downloader
         enhanced_download_archive
             Download archive
-        ytdl_options
+        ytdl_options_builder
-            YTDL options validator
+            YTDL options builder
         """
         DownloadArchiver.__init__(self=self, enhanced_download_archive=enhanced_download_archive)
         self.download_options = download_options
-        self.ytdl_options = self._configure_ytdl_options(
+
-            ytdl_options=ytdl_options,
+        self._ytdl_options_builder = ytdl_options_builder.clone().add(
+            self.ytdl_option_defaults(), before=True
         )
 
     @contextmanager
@@ -111,9 +98,7 @@ class Downloader(DownloadArchiver, Generic[DownloaderOptionsT, DownloaderEntryT]
         """
         Context manager to interact with yt_dlp.
         """
-        ytdl_options = self.ytdl_options
+        ytdl_options = self._ytdl_options_builder.clone().add(ytdl_options_overrides).to_dict()
-        if ytdl_options_overrides is not None:
-            ytdl_options = dict(ytdl_options, **ytdl_options_overrides)
 
         download_logger.debug("ytdl_options: %s", str(ytdl_options))
         with Logger.handle_external_logs(name="yt-dlp"):
@@ -127,7 +112,7 @@ class Downloader(DownloadArchiver, Generic[DownloaderOptionsT, DownloaderEntryT]
         -------
         True if dry-run is enabled. False otherwise.
         """
-        return self.ytdl_options.get("skip_download", False)
+        return self._ytdl_options_builder.to_dict().get("skip_download", False)
 
     def extract_info(self, ytdl_options_overrides: Optional[Dict] = None, **kwargs) -> Dict:
         """
@@ -173,7 +158,6 @@ class Downloader(DownloadArchiver, Generic[DownloaderOptionsT, DownloaderEntryT]
         """
         num_tries = 0
         entry_files_exist = False
-        ytdl_options_overrides = copy.deepcopy(ytdl_options_overrides)
 
         while not entry_files_exist and num_tries < self._extract_entry_num_retries:
             entry_dict = self.extract_info(ytdl_options_overrides=ytdl_options_overrides, **kwargs)
@@ -185,7 +169,11 @@ class Downloader(DownloadArchiver, Generic[DownloaderOptionsT, DownloaderEntryT]
 
             # Remove the download archive so it can retry without thinking its already downloaded,
             # even though it is not
-            ytdl_options_overrides["download_archive"] = None
+            ytdl_options_overrides = (
+                YTDLOptionsBuilder()
+                .add(ytdl_options_overrides, {"download_archive": None})
+                .to_dict()
+            )
 
             if num_tries < self._extract_entry_retry_wait_sec:
                 download_logger.debug(
@@ -299,20 +287,25 @@ class Downloader(DownloadArchiver, Generic[DownloaderOptionsT, DownloaderEntryT]
         **kwargs
             arguments passed directory to YoutubeDL extract_info
         """
+        ytdl_options_builder = self._ytdl_options_builder.clone()
         if ytdl_options_overrides is None:
             ytdl_options_overrides = {}
 
-        extract_info_ytdl_options = {"writeinfojson": True}
+        ytdl_options_builder.add({"writeinfojson": True}, ytdl_options_overrides)
         if only_info_json:
-            extract_info_ytdl_options["skip_download"] = True
+            ytdl_options_builder.add(
-            extract_info_ytdl_options["writethumbnail"] = False
+                {
-            extract_info_ytdl_options["writesubtitles"] = False
+                    "skip_download": True,
-
+                    "writethumbnail": False,
-        ytdl_options_overrides = dict(ytdl_options_overrides, **extract_info_ytdl_options)
+                    "writesubtitles": False,
+                }
+            )
 
         try:
             with self._listen_and_log_downloaded_info_json(log_prefix=log_prefix_on_info_json_dl):
-                _ = self.extract_info(ytdl_options_overrides=ytdl_options_overrides, **kwargs)
+                _ = self.extract_info(
+                    ytdl_options_overrides=ytdl_options_builder.to_dict(), **kwargs
+                )
         except RejectedVideoReached:
             download_logger.debug("RejectedVideoReached, stopping additional downloads")
         except ExistingVideoReached:

src/ytdl_sub/downloaders/youtube/channel.py

@@ -9,6 +9,7 @@ from ytdl_sub.downloaders.downloader import DownloaderOptionsT
 from ytdl_sub.downloaders.downloader import download_logger
 from ytdl_sub.downloaders.youtube.abc import YoutubeDownloader
 from ytdl_sub.downloaders.youtube.abc import YoutubeDownloaderOptions
+from ytdl_sub.downloaders.ytdl_options_builder import YTDLOptionsBuilder
 from ytdl_sub.entries.youtube import YoutubeChannel
 from ytdl_sub.entries.youtube import YoutubeVideo
 from ytdl_sub.utils.thumbnail import convert_url_thumbnail
@@ -115,12 +116,12 @@ class YoutubeChannelDownloader(YoutubeDownloader[YoutubeChannelDownloaderOptions
         self,
         download_options: DownloaderOptionsT,
         enhanced_download_archive: EnhancedDownloadArchive,
-        ytdl_options: Optional[Dict] = None,
+        ytdl_options_builder: YTDLOptionsBuilder,
     ):
         super().__init__(
             download_options=download_options,
             enhanced_download_archive=enhanced_download_archive,
-            ytdl_options=ytdl_options,
+            ytdl_options_builder=ytdl_options_builder,
         )
         self.channel: Optional[YoutubeChannel] = None
 

src/ytdl_sub/downloaders/ytdl_options_builder.py

@@ -0,0 +1,59 @@
+import copy
+from typing import Dict
+from typing import Optional
+
+import mergedeep
+
+
+class YTDLOptionsBuilder:
+    """
+    A class to track any modifications made to ytdl options
+    """
+
+    def __init__(self):
+        self._ytdl_options: Dict = {}
+
+    def add(
+        self,
+        *ytdl_option_dicts: Optional[Dict],
+        before: bool = False,
+        strategy: mergedeep.Strategy = mergedeep.Strategy.TYPESAFE_ADDITIVE
+    ) -> "YTDLOptionsBuilder":
+        """
+        Parameters
+        ----------
+        *ytdl_option_dicts
+            One or many ytdl_option dicts. Can also contain None's for convenience
+        before
+            Optional. Whether to add these dicts before or after the original
+        strategy
+            Optional. mergedeep strategy. Defaults to TYPESAFE_ADDITIVE
+
+        Returns
+        -------
+        instance with the added ytdl_option dict(s)
+        """
+        non_empty = [ytdl_options for ytdl_options in ytdl_option_dicts if ytdl_options is not None]
+
+        if before:
+            non_empty.append(self.to_dict())
+            self._ytdl_options = {}
+
+        mergedeep.merge(self._ytdl_options, *non_empty, strategy=strategy)
+        return self
+
+    def clone(self) -> "YTDLOptionsBuilder":
+        """
+        Returns
+        -------
+        Deep-copied instance
+        """
+        return copy.deepcopy(self)
+
+    def to_dict(self) -> Dict:
+        """
+        Returns
+        -------
+        Deep-copied dict of the current builder state
+        """
+        return copy.deepcopy(self._ytdl_options)

src/ytdl_sub/subscriptions/subscription.py

@@ -287,19 +287,19 @@ class Subscription:
             directory.
         """
         self._enhanced_download_archive.reinitialize(dry_run=dry_run)
-        ytdl_options = SubscriptionYTDLOptions(
+        ytdl_options_builder = SubscriptionYTDLOptions(
             preset=self.__preset_options,
             enhanced_download_archive=self._enhanced_download_archive,
             working_directory=self.working_directory,
             dry_run=dry_run,
-        ).to_dict()
+        ).builder()
 
         plugins = self._initialize_plugins()
         with self._subscription_download_context_managers():
             downloader = self.downloader_class(
                 download_options=self.downloader_options,
                 enhanced_download_archive=self._enhanced_download_archive,
-                ytdl_options=ytdl_options,
+                ytdl_options_builder=ytdl_options_builder,
             )
 
             for entry in downloader.download():

src/ytdl_sub/subscriptions/subscription_ytdl_options.py

@@ -1,9 +1,8 @@
 from pathlib import Path
 from typing import Dict
 
-import mergedeep
-
 from ytdl_sub.config.preset import Preset
+from ytdl_sub.downloaders.ytdl_options_builder import YTDLOptionsBuilder
 from ytdl_sub.ytdl_additions.enhanced_download_archive import EnhancedDownloadArchive
 
 
@@ -22,11 +21,26 @@ class SubscriptionYTDLOptions:
 
     @property
     def _global_options(self) -> Dict:
-        return {
+        """
+        Returns
+        -------
+        ytdl-options to apply to every run no matter what
+        """
+        ytdl_options = {
             # Download all files in the format of {id}.{ext}
             "outtmpl": str(Path(self._working_directory) / "%(id)s.%(ext)s")
         }
 
+        if (
+            self._preset.downloader.supports_download_archive
+            and self._preset.output_options.maintain_download_archive
+        ):
+            ytdl_options["download_archive"] = str(
+                Path(self._working_directory) / self._enhanced_download_archive.archive_file_name
+            )
+
+        return ytdl_options
+
     @property
     def _dry_run_options(self) -> Dict:
         return {
@@ -42,13 +56,6 @@ class SubscriptionYTDLOptions:
 
         if output_options.thumbnail_name:
             ytdl_options["writethumbnail"] = True
-        if (
-            self._preset.downloader.supports_download_archive
-            and output_options.maintain_download_archive
-        ):
-            ytdl_options["download_archive"] = str(
-                Path(self._working_directory) / self._enhanced_download_archive.archive_file_name
-            )
 
         return ytdl_options
 
@@ -83,22 +90,19 @@ class SubscriptionYTDLOptions:
     def _user_ytdl_options(self) -> Dict:
         return self._preset.ytdl_options.dict
 
-    def to_dict(self) -> Dict:
+    def builder(self) -> YTDLOptionsBuilder:
-        ytdl_options = self._global_options
+        """
+        Returns
+        -------
+        YTDLOptionsBuilder
+            Builder with values set based on the subscription
+        """
+        ytdl_options_builder = YTDLOptionsBuilder().add(self._global_options)
         if self._dry_run:
-            mergedeep.merge(
+            ytdl_options_builder.add(self._dry_run_options, self._user_ytdl_options)
-                ytdl_options,
-                self._dry_run_options,
-                self._user_ytdl_options,
-                strategy=mergedeep.Strategy.TYPESAFE_ADDITIVE,
-            )
         else:
-            mergedeep.merge(
+            ytdl_options_builder.add(
-                ytdl_options,
+                self._output_options, self._subtitle_options, self._user_ytdl_options
-                self._output_options,
-                self._subtitle_options,
-                self._user_ytdl_options,
-                strategy=mergedeep.Strategy.TYPESAFE_ADDITIVE,
             )
 
-        return ytdl_options
+        return ytdl_options_builder

tests/unit/entries/conftest.py

@@ -54,10 +54,12 @@ def ext():
 def thumbnail_ext():
     return "jpg"
 
+
 @pytest.fixture
 def subtitles_ext():
     return "srt"
 
+
 @pytest.fixture
 def download_thumbnail_name(uid, thumbnail_ext):
     return f"{uid}.{thumbnail_ext}"