[REFACTOR] download abstraction overhaul (#663)

* [REFACTOR] `download` abstraction overhaul

* remove plugin options

src/ytdl_sub/config/preset.py

@@ -16,12 +16,12 @@ from ytdl_sub.config.preset_class_mappings import PluginMapping
 from ytdl_sub.config.preset_options import OptionsValidator
 from ytdl_sub.config.preset_options import OutputOptions
 from ytdl_sub.config.preset_options import Overrides
+from ytdl_sub.config.preset_options import TOptionsValidator
 from ytdl_sub.config.preset_options import YTDLOptions
 from ytdl_sub.downloaders.base_downloader import BaseDownloader
-from ytdl_sub.downloaders.base_downloader import BaseDownloaderValidator
+from ytdl_sub.downloaders.downloader_validator import DownloaderValidator
 from ytdl_sub.entries.entry import Entry
 from ytdl_sub.plugins.plugin import Plugin
-from ytdl_sub.plugins.plugin import PluginOptionsT
 from ytdl_sub.prebuilt_presets import PREBUILT_PRESET_NAMES
 from ytdl_sub.prebuilt_presets import PUBLISHED_PRESET_NAMES
 from ytdl_sub.utils.exceptions import ValidationException
@@ -85,7 +85,7 @@ class PresetPlugins:
         """
         return zip(self.plugin_types, self.plugin_options)
 
-    def get(self, plugin_type: Type[PluginOptionsT]) -> Optional[PluginOptionsT]:
+    def get(self, plugin_type: Type[TOptionsValidator]) -> Optional[TOptionsValidator]:
         """
         Parameters
         ----------
@@ -277,7 +277,7 @@ class Preset(_PresetShell):
 
     def __validate_and_get_downloader_options(
         self, downloader_source: str, downloader: Type[BaseDownloader]
-    ) -> BaseDownloaderValidator:
+    ) -> DownloaderValidator:
         # Remove the download_strategy key before validating it against the downloader options
         # TODO: make this cleaner
         del self._dict[downloader_source]["download_strategy"]
@@ -287,9 +287,9 @@ class Preset(_PresetShell):
 
     def __validate_and_get_downloader_and_options(
         self,
-    ) -> Tuple[Type[BaseDownloader], BaseDownloaderValidator]:
+    ) -> Tuple[Type[BaseDownloader], DownloaderValidator]:
         downloader: Optional[Type[BaseDownloader]] = None
-        download_options: Optional[BaseDownloaderValidator] = None
+        download_options: Optional[DownloaderValidator] = None
         downloader_sources = DownloadStrategyMapping.sources()
 
         for key in self._keys:

src/ytdl_sub/config/preset_options.py

@@ -3,6 +3,7 @@ from typing import Any
 from typing import Dict
 from typing import List
 from typing import Optional
+from typing import TypeVar
 
 from yt_dlp.utils import sanitize_filename
 
@@ -71,6 +72,9 @@ class OptionsValidator(Validator, ABC):
         return None
 
 
+TOptionsValidator = TypeVar("TOptionsValidator", bound=OptionsValidator)
+
+
 class OptionsDictValidator(StrictDictValidator, OptionsValidator, ABC):
     pass
 

src/ytdl_sub/downloaders/base_downloader.py

@@ -4,21 +4,18 @@ from typing import Generic
 from typing import Iterable
 from typing import List
 from typing import Type
-from typing import TypeVar
 
-from ytdl_sub.config.preset_options import OptionsDictValidator
 from ytdl_sub.config.preset_options import Overrides
+from ytdl_sub.config.preset_options import TOptionsValidator
+from ytdl_sub.downloaders.downloader_validator import TDownloaderValidator
 from ytdl_sub.downloaders.ytdl_options_builder import YTDLOptionsBuilder
 from ytdl_sub.entries.entry import Entry
 from ytdl_sub.plugins.plugin import Plugin
 from ytdl_sub.ytdl_additions.enhanced_download_archive import DownloadArchiver
 from ytdl_sub.ytdl_additions.enhanced_download_archive import EnhancedDownloadArchive
 
-BaseDownloaderValidator = OptionsDictValidator
-BaseDownloaderOptionsT = TypeVar("BaseDownloaderOptionsT", bound=BaseDownloaderValidator)
 
-
+class BaseDownloaderPlugin(Plugin[TDownloaderValidator], ABC):
-class BaseDownloaderPlugin(Plugin[BaseDownloaderOptionsT], ABC):
     """
     Plugins that get added automatically by using a downloader. Downloader options
     are the plugin options.
@@ -26,7 +23,7 @@ class BaseDownloaderPlugin(Plugin[BaseDownloaderOptionsT], ABC):
 
     def __init__(
         self,
-        downloader_options: BaseDownloaderOptionsT,
+        downloader_options: TDownloaderValidator,
         overrides: Overrides,
         enhanced_download_archive: EnhancedDownloadArchive,
     ):
@@ -38,12 +35,12 @@ class BaseDownloaderPlugin(Plugin[BaseDownloaderOptionsT], ABC):
         )
 
 
-class BaseDownloader(DownloadArchiver, Generic[BaseDownloaderOptionsT], ABC):
+class BaseDownloader(DownloadArchiver, Generic[TOptionsValidator], ABC):
-    downloader_options_type: Type[BaseDownloaderOptionsT]
+    downloader_options_type: Type[TOptionsValidator]
 
     def __init__(
         self,
-        download_options: BaseDownloaderOptionsT,
+        download_options: TOptionsValidator,
         enhanced_download_archive: EnhancedDownloadArchive,
         download_ytdl_options: YTDLOptionsBuilder,
         metadata_ytdl_options: YTDLOptionsBuilder,
@@ -67,7 +64,7 @@ class BaseDownloader(DownloadArchiver, Generic[BaseDownloaderOptionsT], ABC):
     @classmethod
     def added_plugins(
         cls,
-        downloader_options: BaseDownloaderOptionsT,
+        downloader_options: TOptionsValidator,
         enhanced_download_archive: EnhancedDownloadArchive,
         overrides: Overrides,
     ) -> List[BaseDownloaderPlugin]:

src/ytdl_sub/downloaders/downloader_validator.py

@@ -0,0 +1,45 @@
+import abc
+from abc import ABC
+from typing import Dict
+from typing import List
+from typing import TypeVar
+
+from ytdl_sub.config.preset_options import OptionsValidator
+from ytdl_sub.downloaders.url.validators import MultiUrlValidator
+
+
+class DownloaderValidator(OptionsValidator, ABC):
+    """
+    Placeholder class to define downloader options
+    """
+
+    @property
+    @abc.abstractmethod
+    def collection_validator(self) -> MultiUrlValidator:
+        """
+        Returns
+        -------
+        MultiUrlValidator
+            To determine how the entries are downloaded
+        """
+
+    def added_source_variables(self) -> List[str]:
+        """
+        Returns
+        -------
+        Added source variables on the collection
+        """
+        return self.collection_validator.added_source_variables()
+
+    def validate_with_variables(
+        self, source_variables: List[str], override_variables: Dict[str, str]
+    ) -> None:
+        """
+        Validates any source variables added by the collection
+        """
+        self.collection_validator.validate_with_variables(
+            source_variables=source_variables, override_variables=override_variables
+        )
+
+
+TDownloaderValidator = TypeVar("TDownloaderValidator", bound=DownloaderValidator)

src/ytdl_sub/downloaders/info_json/info_json_downloader.py

@@ -5,10 +5,9 @@ from typing import Dict
 from typing import Iterable
 from typing import List
 
+from ytdl_sub.config.preset_options import OptionsDictValidator
 from ytdl_sub.config.preset_options import Overrides
 from ytdl_sub.downloaders.base_downloader import BaseDownloader
-from ytdl_sub.downloaders.base_downloader import BaseDownloaderOptionsT
-from ytdl_sub.downloaders.base_downloader import BaseDownloaderValidator
 from ytdl_sub.downloaders.ytdl_options_builder import YTDLOptionsBuilder
 from ytdl_sub.entries.entry import Entry
 from ytdl_sub.utils.exceptions import ValidationException
@@ -18,7 +17,7 @@ from ytdl_sub.ytdl_additions.enhanced_download_archive import DownloadMapping
 from ytdl_sub.ytdl_additions.enhanced_download_archive import EnhancedDownloadArchive
 
 
-class InfoJsonDownloaderOptions(BaseDownloaderValidator):
+class InfoJsonDownloaderOptions(OptionsDictValidator):
     _optional_keys = {"no-op"}
 
 
@@ -27,7 +26,7 @@ class InfoJsonDownloader(BaseDownloader[InfoJsonDownloaderOptions]):
 
     def __init__(
         self,
-        download_options: BaseDownloaderOptionsT,
+        download_options: InfoJsonDownloaderOptions,
         enhanced_download_archive: EnhancedDownloadArchive,
         download_ytdl_options: YTDLOptionsBuilder,
         metadata_ytdl_options: YTDLOptionsBuilder,

src/ytdl_sub/downloaders/url/downloader.py

@@ -1,4 +1,3 @@
-import abc
 import contextlib
 import os
 from abc import ABC
@@ -15,9 +14,9 @@ from yt_dlp.utils import RejectedVideoReached
 
 from ytdl_sub.config.preset_options import Overrides
 from ytdl_sub.downloaders.base_downloader import BaseDownloader
-from ytdl_sub.downloaders.base_downloader import BaseDownloaderOptionsT
 from ytdl_sub.downloaders.base_downloader import BaseDownloaderPlugin
-from ytdl_sub.downloaders.base_downloader import BaseDownloaderValidator
+from ytdl_sub.downloaders.downloader_validator import DownloaderValidator
+from ytdl_sub.downloaders.downloader_validator import TDownloaderValidator
 from ytdl_sub.downloaders.url.validators import MultiUrlValidator
 from ytdl_sub.downloaders.url.validators import UrlThumbnailListValidator
 from ytdl_sub.downloaders.url.validators import UrlValidator
@@ -45,40 +44,6 @@ from ytdl_sub.ytdl_additions.enhanced_download_archive import EnhancedDownloadAr
 download_logger = Logger.get(name="downloader")
 
 
-class DownloaderValidator(BaseDownloaderValidator, ABC):
-    """
-    Placeholder class to define downloader options
-    """
-
-    @property
-    @abc.abstractmethod
-    def collection_validator(self) -> MultiUrlValidator:
-        """
-        Returns
-        -------
-        MultiUrlValidator
-            To determine how the entries are downloaded
-        """
-
-    def added_source_variables(self) -> List[str]:
-        """
-        Returns
-        -------
-        Added source variables on the collection
-        """
-        return self.collection_validator.added_source_variables()
-
-    def validate_with_variables(
-        self, source_variables: List[str], override_variables: Dict[str, str]
-    ) -> None:
-        """
-        Validates any source variables added by the collection
-        """
-        self.collection_validator.validate_with_variables(
-            source_variables=source_variables, override_variables=override_variables
-        )
-
-
 class URLDownloadState:
     def __init__(self, entries_total: int):
         self.entries_total = entries_total
@@ -221,7 +186,7 @@ class UrlDownloaderCollectionVariablePlugin(BaseDownloaderPlugin):
         return entry
 
 
-class BaseUrlDownloader(BaseDownloader[BaseDownloaderOptionsT], ABC):
+class BaseUrlDownloader(BaseDownloader[TDownloaderValidator], ABC):
     """
     Class that interacts with ytdl to perform the download of metadata and content,
     and should translate that to list of Entry objects.
@@ -230,7 +195,7 @@ class BaseUrlDownloader(BaseDownloader[BaseDownloaderOptionsT], ABC):
     @classmethod
     def added_plugins(
         cls,
-        downloader_options: BaseDownloaderOptionsT,
+        downloader_options: TDownloaderValidator,
         enhanced_download_archive: EnhancedDownloadArchive,
         overrides: Overrides,
     ) -> List[Plugin]:
@@ -264,7 +229,7 @@ class BaseUrlDownloader(BaseDownloader[BaseDownloaderOptionsT], ABC):
 
     def __init__(
         self,
-        download_options: BaseDownloaderOptionsT,
+        download_options: TDownloaderValidator,
         enhanced_download_archive: EnhancedDownloadArchive,
         download_ytdl_options: YTDLOptionsBuilder,
         metadata_ytdl_options: YTDLOptionsBuilder,

src/ytdl_sub/downloaders/url/multi_url.py

@@ -1,8 +1,8 @@
 from typing import Dict
 from typing import List
 
+from ytdl_sub.downloaders.downloader_validator import DownloaderValidator
 from ytdl_sub.downloaders.url.downloader import BaseUrlDownloader
-from ytdl_sub.downloaders.url.downloader import DownloaderValidator
 from ytdl_sub.downloaders.url.validators import MultiUrlValidator
 
 

src/ytdl_sub/downloaders/url/url.py

@@ -1,5 +1,5 @@
+from ytdl_sub.downloaders.downloader_validator import DownloaderValidator
 from ytdl_sub.downloaders.url.downloader import BaseUrlDownloader
-from ytdl_sub.downloaders.url.downloader import DownloaderValidator
 from ytdl_sub.downloaders.url.validators import MultiUrlValidator
 from ytdl_sub.downloaders.url.validators import UrlValidator
 

src/ytdl_sub/plugins/plugin.py

@@ -5,10 +5,9 @@ from typing import List
 from typing import Optional
 from typing import Tuple
 from typing import Type
-from typing import TypeVar
 
-from ytdl_sub.config.preset_options import OptionsValidator
 from ytdl_sub.config.preset_options import Overrides
+from ytdl_sub.config.preset_options import TOptionsValidator
 from ytdl_sub.entries.entry import Entry
 from ytdl_sub.utils.file_handler import FileMetadata
 from ytdl_sub.utils.logger import Logger
@@ -43,15 +42,12 @@ class PluginPriority:
         return self.modify_entry >= PluginPriority.MODIFY_ENTRY_AFTER_SPLIT
 
 
-PluginOptionsT = TypeVar("PluginOptionsT", bound=OptionsValidator)
+class Plugin(DownloadArchiver, Generic[TOptionsValidator], ABC):
-
-
-class Plugin(DownloadArchiver, Generic[PluginOptionsT], ABC):
     """
     Class to define the new plugin functionality
     """
 
-    plugin_options_type: Type[PluginOptionsT] = NotImplemented
+    plugin_options_type: Type[TOptionsValidator] = NotImplemented
     priority: PluginPriority = PluginPriority()
 
     # If the plugin creates multiple entries from a single entry
@@ -59,7 +55,7 @@ class Plugin(DownloadArchiver, Generic[PluginOptionsT], ABC):
 
     def __init__(
         self,
-        plugin_options: PluginOptionsT,
+        plugin_options: TOptionsValidator,
         overrides: Overrides,
         enhanced_download_archive: EnhancedDownloadArchive,
     ):

src/ytdl_sub/subscriptions/base_subscription.py

@@ -9,7 +9,7 @@ from ytdl_sub.config.preset_options import OutputOptions
 from ytdl_sub.config.preset_options import Overrides
 from ytdl_sub.config.preset_options import YTDLOptions
 from ytdl_sub.downloaders.base_downloader import BaseDownloader
-from ytdl_sub.downloaders.base_downloader import BaseDownloaderValidator
+from ytdl_sub.downloaders.downloader_validator import DownloaderValidator
 from ytdl_sub.ytdl_additions.enhanced_download_archive import EnhancedDownloadArchive
 
 
@@ -61,7 +61,7 @@ class BaseSubscription(ABC):
         return self._preset_options.downloader
 
     @property
-    def downloader_options(self) -> BaseDownloaderValidator:
+    def downloader_options(self) -> DownloaderValidator:
         """
         Returns
         -------