[REFACTOR] BaseDownloader (#542)

* [REFACTOR] BaseDownloader

* doh

* fix mock

src/ytdl_sub/config/preset.py

@@ -17,8 +17,8 @@ from ytdl_sub.config.preset_class_mappings import PluginMapping
 from ytdl_sub.config.preset_options import OutputOptions
 from ytdl_sub.config.preset_options import Overrides
 from ytdl_sub.config.preset_options import YTDLOptions
-from ytdl_sub.downloaders.downloader import BaseDownloader
+from ytdl_sub.downloaders.base_downloader import BaseDownloader
-from ytdl_sub.downloaders.downloader import DownloaderValidator
+from ytdl_sub.downloaders.base_downloader import BaseDownloaderValidator
 from ytdl_sub.entries.entry import Entry
 from ytdl_sub.plugins.plugin import Plugin
 from ytdl_sub.plugins.plugin import PluginOptions
@@ -250,7 +250,7 @@ class Preset(_PresetShell):
 
     def __validate_and_get_downloader_options(
         self, downloader_source: str, downloader: Type[BaseDownloader]
-    ) -> DownloaderValidator:
+    ) -> BaseDownloaderValidator:
         # Remove the download_strategy key before validating it against the downloader options
         # TODO: make this cleaner
         del self._dict[downloader_source]["download_strategy"]
@@ -260,9 +260,9 @@ class Preset(_PresetShell):
 
     def __validate_and_get_downloader_and_options(
         self,
-    ) -> Tuple[Type[BaseDownloader], DownloaderValidator]:
+    ) -> Tuple[Type[BaseDownloader], BaseDownloaderValidator]:
         downloader: Optional[Type[BaseDownloader]] = None
-        download_options: Optional[DownloaderValidator] = None
+        download_options: Optional[BaseDownloaderValidator] = None
         downloader_sources = DownloadStrategyMapping.sources()
 
         for key in self._keys:

src/ytdl_sub/config/preset_class_mappings.py

@@ -2,9 +2,9 @@ from typing import Dict
 from typing import List
 from typing import Type
 
-from ytdl_sub.downloaders.downloader import BaseDownloader
+from ytdl_sub.downloaders.base_downloader import BaseDownloader
-from ytdl_sub.downloaders.generic.multi_url import MultiUrlDownloader
+from ytdl_sub.downloaders.url.multi_url import MultiUrlDownloader
-from ytdl_sub.downloaders.generic.url import UrlDownloader
+from ytdl_sub.downloaders.url.url import UrlDownloader
 from ytdl_sub.plugins.audio_extract import AudioExtractPlugin
 from ytdl_sub.plugins.chapters import ChaptersPlugin
 from ytdl_sub.plugins.date_range import DateRangePlugin

src/ytdl_sub/downloaders/base_downloader.py

@@ -0,0 +1,76 @@
+import abc
+from abc import ABC
+from typing import Generic
+from typing import Iterable
+from typing import List
+from typing import Type
+from typing import TypeVar
+
+from ytdl_sub.config.preset_options import AddsVariablesMixin
+from ytdl_sub.config.preset_options import Overrides
+from ytdl_sub.downloaders.ytdl_options_builder import YTDLOptionsBuilder
+from ytdl_sub.entries.entry import Entry
+from ytdl_sub.plugins.plugin import Plugin
+from ytdl_sub.plugins.plugin import PluginOptions
+from ytdl_sub.validators.strict_dict_validator import StrictDictValidator
+from ytdl_sub.ytdl_additions.enhanced_download_archive import DownloadArchiver
+from ytdl_sub.ytdl_additions.enhanced_download_archive import EnhancedDownloadArchive
+
+
+class BaseDownloaderValidator(StrictDictValidator, AddsVariablesMixin, ABC):
+    pass
+
+
+BaseDownloaderOptionsT = TypeVar("BaseDownloaderOptionsT", bound=BaseDownloaderValidator)
+
+
+class BaseDownloaderPluginOptions(PluginOptions):
+    _optional_keys = {"no-op"}
+
+
+class BaseDownloaderPlugin(Plugin[BaseDownloaderPluginOptions], ABC):
+    def __init__(
+        self,
+        overrides: Overrides,
+        enhanced_download_archive: EnhancedDownloadArchive,
+    ):
+        super().__init__(
+            # Downloader plugins do not have exposed YAML options, so keep it blank.
+            # Use init instead.
+            plugin_options=BaseDownloaderPluginOptions(name=self.__class__.__name__, value={}),
+            overrides=overrides,
+            enhanced_download_archive=enhanced_download_archive,
+        )
+
+
+class BaseDownloader(DownloadArchiver, Generic[BaseDownloaderOptionsT], ABC):
+    downloader_options_type: Type[BaseDownloaderOptionsT]
+
+    def __init__(
+        self,
+        download_options: BaseDownloaderOptionsT,
+        enhanced_download_archive: EnhancedDownloadArchive,
+        download_ytdl_options: YTDLOptionsBuilder,
+        metadata_ytdl_options: YTDLOptionsBuilder,
+        overrides: Overrides,
+    ):
+        super().__init__(enhanced_download_archive=enhanced_download_archive)
+        self.download_options = download_options
+        self.overrides = overrides
+        self._download_ytdl_options_builder = download_ytdl_options
+        self._metadata_ytdl_options_builder = metadata_ytdl_options
+
+    @abc.abstractmethod
+    def download_metadata(self) -> Iterable[Entry]:
+        """Gathers metadata of all entries to download"""
+
+    @abc.abstractmethod
+    def download(self, entry: Entry) -> Entry:
+        """The function to perform the download of all media entries"""
+
+    # pylint: disable=no-self-use
+    def added_plugins(self) -> List[BaseDownloaderPlugin]:
+        """Add these plugins from the Downloader to the subscription"""
+        return []
+
+    # pylint: enable=no-self-use

src/ytdl_sub/downloaders/url/downloader.py

@@ -4,21 +4,21 @@ import os
 from abc import ABC
 from pathlib import Path
 from typing import Dict
-from typing import Generic
 from typing import Iterable
 from typing import Iterator
 from typing import List
 from typing import Optional
 from typing import Set
 from typing import Tuple
-from typing import Type
-from typing import TypeVar
 
-from ytdl_sub.config.preset_options import AddsVariablesMixin
 from ytdl_sub.config.preset_options import Overrides
-from ytdl_sub.downloaders.generic.validators import MultiUrlValidator
+from ytdl_sub.downloaders.base_downloader import BaseDownloader
-from ytdl_sub.downloaders.generic.validators import UrlThumbnailListValidator
+from ytdl_sub.downloaders.base_downloader import BaseDownloaderOptionsT
-from ytdl_sub.downloaders.generic.validators import UrlValidator
+from ytdl_sub.downloaders.base_downloader import BaseDownloaderPlugin
+from ytdl_sub.downloaders.base_downloader import BaseDownloaderValidator
+from ytdl_sub.downloaders.url.validators import MultiUrlValidator
+from ytdl_sub.downloaders.url.validators import UrlThumbnailListValidator
+from ytdl_sub.downloaders.url.validators import UrlValidator
 from ytdl_sub.downloaders.ytdl_options_builder import YTDLOptionsBuilder
 from ytdl_sub.downloaders.ytdlp import YTDLP
 from ytdl_sub.entries.entry import Entry
@@ -32,14 +32,11 @@ from ytdl_sub.entries.variables.kwargs import SOURCE_ENTRY
 from ytdl_sub.entries.variables.kwargs import SPONSORBLOCK_CHAPTERS
 from ytdl_sub.entries.variables.kwargs import UPLOAD_DATE_INDEX
 from ytdl_sub.plugins.plugin import Plugin
-from ytdl_sub.plugins.plugin import PluginOptions
 from ytdl_sub.utils.file_handler import FileHandler
 from ytdl_sub.utils.logger import Logger
 from ytdl_sub.utils.thumbnail import ThumbnailTypes
 from ytdl_sub.utils.thumbnail import convert_download_thumbnail
 from ytdl_sub.utils.thumbnail import download_and_convert_url_thumbnail
-from ytdl_sub.validators.strict_dict_validator import StrictDictValidator
-from ytdl_sub.ytdl_additions.enhanced_download_archive import DownloadArchiver
 from ytdl_sub.ytdl_additions.enhanced_download_archive import EnhancedDownloadArchive
 
 # pylint: disable=too-many-instance-attributes
@@ -47,7 +44,7 @@ from ytdl_sub.ytdl_additions.enhanced_download_archive import EnhancedDownloadAr
 download_logger = Logger.get(name="downloader")
 
 
-class DownloaderValidator(StrictDictValidator, AddsVariablesMixin, ABC):
+class DownloaderValidator(BaseDownloaderValidator, ABC):
     """
     Placeholder class to define downloader options
     """
@@ -81,68 +78,13 @@ class DownloaderValidator(StrictDictValidator, AddsVariablesMixin, ABC):
         )
 
 
-DownloaderOptionsT = TypeVar("DownloaderOptionsT", bound=DownloaderValidator)
-
-
 class URLDownloadState:
     def __init__(self, entries_total: int):
         self.entries_total = entries_total
         self.entries_downloaded = 0
 
 
-class EmptyPluginOptions(PluginOptions):
+class UrlDownloaderThumbnailPlugin(BaseDownloaderPlugin):
-    _optional_keys = {"no-op"}
-
-
-class BaseDownloaderPlugin(Plugin[EmptyPluginOptions], ABC):
-    def __init__(
-        self,
-        overrides: Overrides,
-        enhanced_download_archive: EnhancedDownloadArchive,
-    ):
-        super().__init__(
-            # Downloader plugins do not have exposed YAML options, so keep it blank.
-            # Use init instead.
-            plugin_options=EmptyPluginOptions(name=self.__class__.__name__, value={}),
-            overrides=overrides,
-            enhanced_download_archive=enhanced_download_archive,
-        )
-
-
-class BaseDownloader(DownloadArchiver, Generic[DownloaderOptionsT], ABC):
-    downloader_options_type: Type[DownloaderValidator] = DownloaderValidator
-
-    def __init__(
-        self,
-        download_options: DownloaderOptionsT,
-        enhanced_download_archive: EnhancedDownloadArchive,
-        download_ytdl_options: YTDLOptionsBuilder,
-        metadata_ytdl_options: YTDLOptionsBuilder,
-        overrides: Overrides,
-    ):
-        super().__init__(enhanced_download_archive=enhanced_download_archive)
-        self.download_options = download_options
-        self.overrides = overrides
-        self._download_ytdl_options_builder = download_ytdl_options
-        self._metadata_ytdl_options_builder = metadata_ytdl_options
-
-    @abc.abstractmethod
-    def download_metadata(self) -> Iterable[Entry]:
-        """Gathers metadata of all entries to download"""
-
-    @abc.abstractmethod
-    def download(self, entry: Entry) -> Entry:
-        """The function to perform the download of all media entries"""
-
-    # pylint: disable=no-self-use
-    def added_plugins(self) -> List[BaseDownloaderPlugin]:
-        """Add these plugins from the Downloader to the subscription"""
-        return []
-
-    # pylint: enable=no-self-use
-
-
-class YtDlpThumbnailPlugin(BaseDownloaderPlugin):
     def __init__(
         self,
         overrides: Overrides,
@@ -240,7 +182,7 @@ class YtDlpThumbnailPlugin(BaseDownloaderPlugin):
         return entry
 
 
-class YtDlpCollectionVariablePlugin(BaseDownloaderPlugin):
+class UrlDownloaderCollectionVariablePlugin(BaseDownloaderPlugin):
     def __init__(
         self,
         overrides: Overrides,
@@ -270,7 +212,7 @@ class YtDlpCollectionVariablePlugin(BaseDownloaderPlugin):
         return entry
 
 
-class YtDlpDownloader(BaseDownloader[DownloaderOptionsT], ABC):
+class BaseUrlDownloader(BaseDownloader[BaseDownloaderOptionsT], ABC):
     """
     Class that interacts with ytdl to perform the download of metadata and content,
     and should translate that to list of Entry objects.
@@ -283,12 +225,12 @@ class YtDlpDownloader(BaseDownloader[DownloaderOptionsT], ABC):
         2. Collection variable plugin to add to each entry
         """
         return [
-            YtDlpThumbnailPlugin(
+            UrlDownloaderThumbnailPlugin(
                 overrides=self.overrides,
                 enhanced_download_archive=self._enhanced_download_archive,
                 collection_urls=self.collection.urls.list,
             ),
-            YtDlpCollectionVariablePlugin(
+            UrlDownloaderCollectionVariablePlugin(
                 overrides=self.overrides,
                 enhanced_download_archive=self._enhanced_download_archive,
                 collection_urls=self.collection.urls.list,
@@ -307,7 +249,7 @@ class YtDlpDownloader(BaseDownloader[DownloaderOptionsT], ABC):
 
     def __init__(
         self,
-        download_options: DownloaderOptionsT,
+        download_options: BaseDownloaderOptionsT,
         enhanced_download_archive: EnhancedDownloadArchive,
         download_ytdl_options: YTDLOptionsBuilder,
         metadata_ytdl_options: YTDLOptionsBuilder,

src/ytdl_sub/downloaders/url/multi_url.py

@@ -1,6 +1,6 @@
-from ytdl_sub.downloaders.downloader import DownloaderValidator
+from ytdl_sub.downloaders.url.downloader import BaseUrlDownloader
-from ytdl_sub.downloaders.downloader import YtDlpDownloader
+from ytdl_sub.downloaders.url.downloader import DownloaderValidator
-from ytdl_sub.downloaders.generic.validators import MultiUrlValidator
+from ytdl_sub.downloaders.url.validators import MultiUrlValidator
 
 
 class MultiUrlDownloadOptions(MultiUrlValidator, DownloaderValidator):
@@ -44,5 +44,5 @@ class MultiUrlDownloadOptions(MultiUrlValidator, DownloaderValidator):
         return self
 
 
-class MultiUrlDownloader(YtDlpDownloader[MultiUrlDownloadOptions]):
+class MultiUrlDownloader(BaseUrlDownloader[MultiUrlDownloadOptions]):
     downloader_options_type = MultiUrlDownloadOptions

src/ytdl_sub/downloaders/url/url.py

@@ -1,7 +1,7 @@
-from ytdl_sub.downloaders.downloader import DownloaderValidator
+from ytdl_sub.downloaders.url.downloader import BaseUrlDownloader
-from ytdl_sub.downloaders.downloader import YtDlpDownloader
+from ytdl_sub.downloaders.url.downloader import DownloaderValidator
-from ytdl_sub.downloaders.generic.validators import MultiUrlValidator
+from ytdl_sub.downloaders.url.validators import MultiUrlValidator
-from ytdl_sub.downloaders.generic.validators import UrlValidator
+from ytdl_sub.downloaders.url.validators import UrlValidator
 
 
 class UrlDownloadOptions(UrlValidator, DownloaderValidator):
@@ -36,5 +36,5 @@ class UrlDownloadOptions(UrlValidator, DownloaderValidator):
         )
 
 
-class UrlDownloader(YtDlpDownloader[UrlDownloadOptions]):
+class UrlDownloader(BaseUrlDownloader[UrlDownloadOptions]):
     downloader_options_type = UrlDownloadOptions

src/ytdl_sub/subscriptions/base_subscription.py

@@ -8,8 +8,8 @@ from ytdl_sub.config.preset import PresetPlugins
 from ytdl_sub.config.preset_options import OutputOptions
 from ytdl_sub.config.preset_options import Overrides
 from ytdl_sub.config.preset_options import YTDLOptions
-from ytdl_sub.downloaders.downloader import BaseDownloader
+from ytdl_sub.downloaders.base_downloader import BaseDownloader
-from ytdl_sub.downloaders.downloader import DownloaderValidator
+from ytdl_sub.downloaders.base_downloader import BaseDownloaderValidator
 from ytdl_sub.ytdl_additions.enhanced_download_archive import EnhancedDownloadArchive
 
 
@@ -61,7 +61,7 @@ class BaseSubscription(ABC):
         return self._preset_options.downloader
 
     @property
-    def downloader_options(self) -> DownloaderValidator:
+    def downloader_options(self) -> BaseDownloaderValidator:
         """
         Returns
         -------

src/ytdl_sub/subscriptions/subscription_ytdl_options.py

@@ -6,7 +6,7 @@ from typing import Type
 from typing import TypeVar
 
 from ytdl_sub.config.preset import Preset
-from ytdl_sub.downloaders.downloader import BaseDownloader
+from ytdl_sub.downloaders.base_downloader import BaseDownloader
 from ytdl_sub.downloaders.ytdl_options_builder import YTDLOptionsBuilder
 from ytdl_sub.plugins.audio_extract import AudioExtractPlugin
 from ytdl_sub.plugins.chapters import ChaptersPlugin

tests/e2e/bandcamp/test_bandcamp.py

@@ -3,7 +3,6 @@ from conftest import assert_logs
 from expected_download import assert_expected_downloads
 from expected_transaction_log import assert_transaction_log_matches
 
-import ytdl_sub.downloaders.downloader
 from ytdl_sub.downloaders.ytdlp import YTDLP
 from ytdl_sub.subscriptions.subscription import Subscription
 

tests/e2e/plugins/test_date_range.py

@@ -6,7 +6,6 @@ from conftest import assert_logs
 from expected_download import assert_expected_downloads
 from expected_transaction_log import assert_transaction_log_matches
 
-import ytdl_sub.downloaders.downloader
 from ytdl_sub.downloaders.ytdlp import YTDLP
 from ytdl_sub.subscriptions.subscription import Subscription
 

tests/e2e/youtube/test_playlist.py

@@ -4,7 +4,6 @@ from e2e.conftest import mock_run_from_cli
 from expected_download import assert_expected_downloads
 from expected_transaction_log import assert_transaction_log_matches
 
-import ytdl_sub.downloaders.downloader
 from ytdl_sub.downloaders.ytdlp import YTDLP
 from ytdl_sub.subscriptions.subscription import Subscription
 

tests/unit/prebuilt_presets/conftest.py

@@ -1,6 +1,5 @@
 import contextlib
 import os
-import tempfile
 from pathlib import Path
 from typing import Callable
 from typing import Dict
@@ -11,7 +10,7 @@ import pytest
 from resources import copy_file_fixture
 
 from ytdl_sub.config.config_file import ConfigFile
-from ytdl_sub.downloaders.downloader import YtDlpDownloader
+from ytdl_sub.downloaders.url.downloader import BaseUrlDownloader
 from ytdl_sub.downloaders.ytdlp import YTDLP
 from ytdl_sub.entries.variables.kwargs import DESCRIPTION
 from ytdl_sub.entries.variables.kwargs import EPOCH
@@ -115,7 +114,7 @@ def mock_download_collection_thumbnail(mock_downloaded_file_path):
         return False
 
     with patch(
-        "ytdl_sub.downloaders.downloader.download_and_convert_url_thumbnail",
+        "ytdl_sub.downloaders.url.downloader.download_and_convert_url_thumbnail",
         new=_mock_download_and_convert_url_thumbnail,
     ):
         yield  # TODO: create file here
@@ -202,7 +201,7 @@ def mock_download_collection_entries(
         with patch.object(
             YTDLP, "extract_info_via_info_json", new=_write_entries_to_working_dir
         ), patch.object(
-            YtDlpDownloader, "_extract_entry_info_with_retry", new=lambda _, entry: entry
+            BaseUrlDownloader, "_extract_entry_info_with_retry", new=lambda _, entry: entry
         ):
             # Stub out metadata. TODO: update this if we do metadata plugins
             yield