diff --git a/src/ytdl_sub/config/preset.py b/src/ytdl_sub/config/preset.py index 5b365502..7cb5fcec 100644 --- a/src/ytdl_sub/config/preset.py +++ b/src/ytdl_sub/config/preset.py @@ -17,8 +17,8 @@ from ytdl_sub.config.preset_class_mappings import PluginMapping from ytdl_sub.config.preset_options import OutputOptions from ytdl_sub.config.preset_options import Overrides from ytdl_sub.config.preset_options import YTDLOptions -from ytdl_sub.downloaders.downloader import BaseDownloader -from ytdl_sub.downloaders.downloader import DownloaderValidator +from ytdl_sub.downloaders.base_downloader import BaseDownloader +from ytdl_sub.downloaders.base_downloader import BaseDownloaderValidator from ytdl_sub.entries.entry import Entry from ytdl_sub.plugins.plugin import Plugin from ytdl_sub.plugins.plugin import PluginOptions @@ -250,7 +250,7 @@ class Preset(_PresetShell): def __validate_and_get_downloader_options( self, downloader_source: str, downloader: Type[BaseDownloader] - ) -> DownloaderValidator: + ) -> BaseDownloaderValidator: # Remove the download_strategy key before validating it against the downloader options # TODO: make this cleaner del self._dict[downloader_source]["download_strategy"] @@ -260,9 +260,9 @@ class Preset(_PresetShell): def __validate_and_get_downloader_and_options( self, - ) -> Tuple[Type[BaseDownloader], DownloaderValidator]: + ) -> Tuple[Type[BaseDownloader], BaseDownloaderValidator]: downloader: Optional[Type[BaseDownloader]] = None - download_options: Optional[DownloaderValidator] = None + download_options: Optional[BaseDownloaderValidator] = None downloader_sources = DownloadStrategyMapping.sources() for key in self._keys: diff --git a/src/ytdl_sub/config/preset_class_mappings.py b/src/ytdl_sub/config/preset_class_mappings.py index 46d3439a..dac9e996 100644 --- a/src/ytdl_sub/config/preset_class_mappings.py +++ b/src/ytdl_sub/config/preset_class_mappings.py @@ -2,9 +2,9 @@ from typing import Dict from typing import List from typing import Type -from ytdl_sub.downloaders.downloader import BaseDownloader -from ytdl_sub.downloaders.generic.multi_url import MultiUrlDownloader -from ytdl_sub.downloaders.generic.url import UrlDownloader +from ytdl_sub.downloaders.base_downloader import BaseDownloader +from ytdl_sub.downloaders.url.multi_url import MultiUrlDownloader +from ytdl_sub.downloaders.url.url import UrlDownloader from ytdl_sub.plugins.audio_extract import AudioExtractPlugin from ytdl_sub.plugins.chapters import ChaptersPlugin from ytdl_sub.plugins.date_range import DateRangePlugin diff --git a/src/ytdl_sub/downloaders/base_downloader.py b/src/ytdl_sub/downloaders/base_downloader.py new file mode 100644 index 00000000..5df45a5c --- /dev/null +++ b/src/ytdl_sub/downloaders/base_downloader.py @@ -0,0 +1,76 @@ +import abc +from abc import ABC +from typing import Generic +from typing import Iterable +from typing import List +from typing import Type +from typing import TypeVar + +from ytdl_sub.config.preset_options import AddsVariablesMixin +from ytdl_sub.config.preset_options import Overrides +from ytdl_sub.downloaders.ytdl_options_builder import YTDLOptionsBuilder +from ytdl_sub.entries.entry import Entry +from ytdl_sub.plugins.plugin import Plugin +from ytdl_sub.plugins.plugin import PluginOptions +from ytdl_sub.validators.strict_dict_validator import StrictDictValidator +from ytdl_sub.ytdl_additions.enhanced_download_archive import DownloadArchiver +from ytdl_sub.ytdl_additions.enhanced_download_archive import EnhancedDownloadArchive + + +class BaseDownloaderValidator(StrictDictValidator, AddsVariablesMixin, ABC): + pass + + +BaseDownloaderOptionsT = TypeVar("BaseDownloaderOptionsT", bound=BaseDownloaderValidator) + + +class BaseDownloaderPluginOptions(PluginOptions): + _optional_keys = {"no-op"} + + +class BaseDownloaderPlugin(Plugin[BaseDownloaderPluginOptions], ABC): + def __init__( + self, + overrides: Overrides, + enhanced_download_archive: EnhancedDownloadArchive, + ): + super().__init__( + # Downloader plugins do not have exposed YAML options, so keep it blank. + # Use init instead. + plugin_options=BaseDownloaderPluginOptions(name=self.__class__.__name__, value={}), + overrides=overrides, + enhanced_download_archive=enhanced_download_archive, + ) + + +class BaseDownloader(DownloadArchiver, Generic[BaseDownloaderOptionsT], ABC): + downloader_options_type: Type[BaseDownloaderOptionsT] + + def __init__( + self, + download_options: BaseDownloaderOptionsT, + enhanced_download_archive: EnhancedDownloadArchive, + download_ytdl_options: YTDLOptionsBuilder, + metadata_ytdl_options: YTDLOptionsBuilder, + overrides: Overrides, + ): + super().__init__(enhanced_download_archive=enhanced_download_archive) + self.download_options = download_options + self.overrides = overrides + self._download_ytdl_options_builder = download_ytdl_options + self._metadata_ytdl_options_builder = metadata_ytdl_options + + @abc.abstractmethod + def download_metadata(self) -> Iterable[Entry]: + """Gathers metadata of all entries to download""" + + @abc.abstractmethod + def download(self, entry: Entry) -> Entry: + """The function to perform the download of all media entries""" + + # pylint: disable=no-self-use + def added_plugins(self) -> List[BaseDownloaderPlugin]: + """Add these plugins from the Downloader to the subscription""" + return [] + + # pylint: enable=no-self-use diff --git a/src/ytdl_sub/downloaders/generic/__init__.py b/src/ytdl_sub/downloaders/url/__init__.py similarity index 100% rename from src/ytdl_sub/downloaders/generic/__init__.py rename to src/ytdl_sub/downloaders/url/__init__.py diff --git a/src/ytdl_sub/downloaders/downloader.py b/src/ytdl_sub/downloaders/url/downloader.py similarity index 87% rename from src/ytdl_sub/downloaders/downloader.py rename to src/ytdl_sub/downloaders/url/downloader.py index 9afeb594..9372308f 100644 --- a/src/ytdl_sub/downloaders/downloader.py +++ b/src/ytdl_sub/downloaders/url/downloader.py @@ -4,21 +4,21 @@ import os from abc import ABC from pathlib import Path from typing import Dict -from typing import Generic from typing import Iterable from typing import Iterator from typing import List from typing import Optional from typing import Set from typing import Tuple -from typing import Type -from typing import TypeVar -from ytdl_sub.config.preset_options import AddsVariablesMixin from ytdl_sub.config.preset_options import Overrides -from ytdl_sub.downloaders.generic.validators import MultiUrlValidator -from ytdl_sub.downloaders.generic.validators import UrlThumbnailListValidator -from ytdl_sub.downloaders.generic.validators import UrlValidator +from ytdl_sub.downloaders.base_downloader import BaseDownloader +from ytdl_sub.downloaders.base_downloader import BaseDownloaderOptionsT +from ytdl_sub.downloaders.base_downloader import BaseDownloaderPlugin +from ytdl_sub.downloaders.base_downloader import BaseDownloaderValidator +from ytdl_sub.downloaders.url.validators import MultiUrlValidator +from ytdl_sub.downloaders.url.validators import UrlThumbnailListValidator +from ytdl_sub.downloaders.url.validators import UrlValidator from ytdl_sub.downloaders.ytdl_options_builder import YTDLOptionsBuilder from ytdl_sub.downloaders.ytdlp import YTDLP from ytdl_sub.entries.entry import Entry @@ -32,14 +32,11 @@ from ytdl_sub.entries.variables.kwargs import SOURCE_ENTRY from ytdl_sub.entries.variables.kwargs import SPONSORBLOCK_CHAPTERS from ytdl_sub.entries.variables.kwargs import UPLOAD_DATE_INDEX from ytdl_sub.plugins.plugin import Plugin -from ytdl_sub.plugins.plugin import PluginOptions from ytdl_sub.utils.file_handler import FileHandler from ytdl_sub.utils.logger import Logger from ytdl_sub.utils.thumbnail import ThumbnailTypes from ytdl_sub.utils.thumbnail import convert_download_thumbnail from ytdl_sub.utils.thumbnail import download_and_convert_url_thumbnail -from ytdl_sub.validators.strict_dict_validator import StrictDictValidator -from ytdl_sub.ytdl_additions.enhanced_download_archive import DownloadArchiver from ytdl_sub.ytdl_additions.enhanced_download_archive import EnhancedDownloadArchive # pylint: disable=too-many-instance-attributes @@ -47,7 +44,7 @@ from ytdl_sub.ytdl_additions.enhanced_download_archive import EnhancedDownloadAr download_logger = Logger.get(name="downloader") -class DownloaderValidator(StrictDictValidator, AddsVariablesMixin, ABC): +class DownloaderValidator(BaseDownloaderValidator, ABC): """ Placeholder class to define downloader options """ @@ -81,68 +78,13 @@ class DownloaderValidator(StrictDictValidator, AddsVariablesMixin, ABC): ) -DownloaderOptionsT = TypeVar("DownloaderOptionsT", bound=DownloaderValidator) - - class URLDownloadState: def __init__(self, entries_total: int): self.entries_total = entries_total self.entries_downloaded = 0 -class EmptyPluginOptions(PluginOptions): - _optional_keys = {"no-op"} - - -class BaseDownloaderPlugin(Plugin[EmptyPluginOptions], ABC): - def __init__( - self, - overrides: Overrides, - enhanced_download_archive: EnhancedDownloadArchive, - ): - super().__init__( - # Downloader plugins do not have exposed YAML options, so keep it blank. - # Use init instead. - plugin_options=EmptyPluginOptions(name=self.__class__.__name__, value={}), - overrides=overrides, - enhanced_download_archive=enhanced_download_archive, - ) - - -class BaseDownloader(DownloadArchiver, Generic[DownloaderOptionsT], ABC): - downloader_options_type: Type[DownloaderValidator] = DownloaderValidator - - def __init__( - self, - download_options: DownloaderOptionsT, - enhanced_download_archive: EnhancedDownloadArchive, - download_ytdl_options: YTDLOptionsBuilder, - metadata_ytdl_options: YTDLOptionsBuilder, - overrides: Overrides, - ): - super().__init__(enhanced_download_archive=enhanced_download_archive) - self.download_options = download_options - self.overrides = overrides - self._download_ytdl_options_builder = download_ytdl_options - self._metadata_ytdl_options_builder = metadata_ytdl_options - - @abc.abstractmethod - def download_metadata(self) -> Iterable[Entry]: - """Gathers metadata of all entries to download""" - - @abc.abstractmethod - def download(self, entry: Entry) -> Entry: - """The function to perform the download of all media entries""" - - # pylint: disable=no-self-use - def added_plugins(self) -> List[BaseDownloaderPlugin]: - """Add these plugins from the Downloader to the subscription""" - return [] - - # pylint: enable=no-self-use - - -class YtDlpThumbnailPlugin(BaseDownloaderPlugin): +class UrlDownloaderThumbnailPlugin(BaseDownloaderPlugin): def __init__( self, overrides: Overrides, @@ -240,7 +182,7 @@ class YtDlpThumbnailPlugin(BaseDownloaderPlugin): return entry -class YtDlpCollectionVariablePlugin(BaseDownloaderPlugin): +class UrlDownloaderCollectionVariablePlugin(BaseDownloaderPlugin): def __init__( self, overrides: Overrides, @@ -270,7 +212,7 @@ class YtDlpCollectionVariablePlugin(BaseDownloaderPlugin): return entry -class YtDlpDownloader(BaseDownloader[DownloaderOptionsT], ABC): +class BaseUrlDownloader(BaseDownloader[BaseDownloaderOptionsT], ABC): """ Class that interacts with ytdl to perform the download of metadata and content, and should translate that to list of Entry objects. @@ -283,12 +225,12 @@ class YtDlpDownloader(BaseDownloader[DownloaderOptionsT], ABC): 2. Collection variable plugin to add to each entry """ return [ - YtDlpThumbnailPlugin( + UrlDownloaderThumbnailPlugin( overrides=self.overrides, enhanced_download_archive=self._enhanced_download_archive, collection_urls=self.collection.urls.list, ), - YtDlpCollectionVariablePlugin( + UrlDownloaderCollectionVariablePlugin( overrides=self.overrides, enhanced_download_archive=self._enhanced_download_archive, collection_urls=self.collection.urls.list, @@ -307,7 +249,7 @@ class YtDlpDownloader(BaseDownloader[DownloaderOptionsT], ABC): def __init__( self, - download_options: DownloaderOptionsT, + download_options: BaseDownloaderOptionsT, enhanced_download_archive: EnhancedDownloadArchive, download_ytdl_options: YTDLOptionsBuilder, metadata_ytdl_options: YTDLOptionsBuilder, diff --git a/src/ytdl_sub/downloaders/generic/multi_url.py b/src/ytdl_sub/downloaders/url/multi_url.py similarity index 83% rename from src/ytdl_sub/downloaders/generic/multi_url.py rename to src/ytdl_sub/downloaders/url/multi_url.py index ce0ed70f..43599781 100644 --- a/src/ytdl_sub/downloaders/generic/multi_url.py +++ b/src/ytdl_sub/downloaders/url/multi_url.py @@ -1,6 +1,6 @@ -from ytdl_sub.downloaders.downloader import DownloaderValidator -from ytdl_sub.downloaders.downloader import YtDlpDownloader -from ytdl_sub.downloaders.generic.validators import MultiUrlValidator +from ytdl_sub.downloaders.url.downloader import BaseUrlDownloader +from ytdl_sub.downloaders.url.downloader import DownloaderValidator +from ytdl_sub.downloaders.url.validators import MultiUrlValidator class MultiUrlDownloadOptions(MultiUrlValidator, DownloaderValidator): @@ -44,5 +44,5 @@ class MultiUrlDownloadOptions(MultiUrlValidator, DownloaderValidator): return self -class MultiUrlDownloader(YtDlpDownloader[MultiUrlDownloadOptions]): +class MultiUrlDownloader(BaseUrlDownloader[MultiUrlDownloadOptions]): downloader_options_type = MultiUrlDownloadOptions diff --git a/src/ytdl_sub/downloaders/generic/url.py b/src/ytdl_sub/downloaders/url/url.py similarity index 73% rename from src/ytdl_sub/downloaders/generic/url.py rename to src/ytdl_sub/downloaders/url/url.py index ed1091a2..53eeef60 100644 --- a/src/ytdl_sub/downloaders/generic/url.py +++ b/src/ytdl_sub/downloaders/url/url.py @@ -1,7 +1,7 @@ -from ytdl_sub.downloaders.downloader import DownloaderValidator -from ytdl_sub.downloaders.downloader import YtDlpDownloader -from ytdl_sub.downloaders.generic.validators import MultiUrlValidator -from ytdl_sub.downloaders.generic.validators import UrlValidator +from ytdl_sub.downloaders.url.downloader import BaseUrlDownloader +from ytdl_sub.downloaders.url.downloader import DownloaderValidator +from ytdl_sub.downloaders.url.validators import MultiUrlValidator +from ytdl_sub.downloaders.url.validators import UrlValidator class UrlDownloadOptions(UrlValidator, DownloaderValidator): @@ -36,5 +36,5 @@ class UrlDownloadOptions(UrlValidator, DownloaderValidator): ) -class UrlDownloader(YtDlpDownloader[UrlDownloadOptions]): +class UrlDownloader(BaseUrlDownloader[UrlDownloadOptions]): downloader_options_type = UrlDownloadOptions diff --git a/src/ytdl_sub/downloaders/generic/validators.py b/src/ytdl_sub/downloaders/url/validators.py similarity index 100% rename from src/ytdl_sub/downloaders/generic/validators.py rename to src/ytdl_sub/downloaders/url/validators.py diff --git a/src/ytdl_sub/subscriptions/base_subscription.py b/src/ytdl_sub/subscriptions/base_subscription.py index cf314d1d..2a6bef23 100644 --- a/src/ytdl_sub/subscriptions/base_subscription.py +++ b/src/ytdl_sub/subscriptions/base_subscription.py @@ -8,8 +8,8 @@ from ytdl_sub.config.preset import PresetPlugins from ytdl_sub.config.preset_options import OutputOptions from ytdl_sub.config.preset_options import Overrides from ytdl_sub.config.preset_options import YTDLOptions -from ytdl_sub.downloaders.downloader import BaseDownloader -from ytdl_sub.downloaders.downloader import DownloaderValidator +from ytdl_sub.downloaders.base_downloader import BaseDownloader +from ytdl_sub.downloaders.base_downloader import BaseDownloaderValidator from ytdl_sub.ytdl_additions.enhanced_download_archive import EnhancedDownloadArchive @@ -61,7 +61,7 @@ class BaseSubscription(ABC): return self._preset_options.downloader @property - def downloader_options(self) -> DownloaderValidator: + def downloader_options(self) -> BaseDownloaderValidator: """ Returns ------- diff --git a/src/ytdl_sub/subscriptions/subscription_ytdl_options.py b/src/ytdl_sub/subscriptions/subscription_ytdl_options.py index 9acb2866..961c6544 100644 --- a/src/ytdl_sub/subscriptions/subscription_ytdl_options.py +++ b/src/ytdl_sub/subscriptions/subscription_ytdl_options.py @@ -6,7 +6,7 @@ from typing import Type from typing import TypeVar from ytdl_sub.config.preset import Preset -from ytdl_sub.downloaders.downloader import BaseDownloader +from ytdl_sub.downloaders.base_downloader import BaseDownloader from ytdl_sub.downloaders.ytdl_options_builder import YTDLOptionsBuilder from ytdl_sub.plugins.audio_extract import AudioExtractPlugin from ytdl_sub.plugins.chapters import ChaptersPlugin diff --git a/tests/e2e/bandcamp/test_bandcamp.py b/tests/e2e/bandcamp/test_bandcamp.py index b3d86034..2b1fb3f0 100644 --- a/tests/e2e/bandcamp/test_bandcamp.py +++ b/tests/e2e/bandcamp/test_bandcamp.py @@ -3,7 +3,6 @@ from conftest import assert_logs from expected_download import assert_expected_downloads from expected_transaction_log import assert_transaction_log_matches -import ytdl_sub.downloaders.downloader from ytdl_sub.downloaders.ytdlp import YTDLP from ytdl_sub.subscriptions.subscription import Subscription diff --git a/tests/e2e/plugins/test_date_range.py b/tests/e2e/plugins/test_date_range.py index 2e2f7cf2..47bf1b73 100644 --- a/tests/e2e/plugins/test_date_range.py +++ b/tests/e2e/plugins/test_date_range.py @@ -6,7 +6,6 @@ from conftest import assert_logs from expected_download import assert_expected_downloads from expected_transaction_log import assert_transaction_log_matches -import ytdl_sub.downloaders.downloader from ytdl_sub.downloaders.ytdlp import YTDLP from ytdl_sub.subscriptions.subscription import Subscription diff --git a/tests/e2e/youtube/test_playlist.py b/tests/e2e/youtube/test_playlist.py index 5ec58cee..8235ce2c 100644 --- a/tests/e2e/youtube/test_playlist.py +++ b/tests/e2e/youtube/test_playlist.py @@ -4,7 +4,6 @@ from e2e.conftest import mock_run_from_cli from expected_download import assert_expected_downloads from expected_transaction_log import assert_transaction_log_matches -import ytdl_sub.downloaders.downloader from ytdl_sub.downloaders.ytdlp import YTDLP from ytdl_sub.subscriptions.subscription import Subscription diff --git a/tests/unit/prebuilt_presets/conftest.py b/tests/unit/prebuilt_presets/conftest.py index eea16c51..5c6c0f54 100644 --- a/tests/unit/prebuilt_presets/conftest.py +++ b/tests/unit/prebuilt_presets/conftest.py @@ -1,6 +1,5 @@ import contextlib import os -import tempfile from pathlib import Path from typing import Callable from typing import Dict @@ -11,7 +10,7 @@ import pytest from resources import copy_file_fixture from ytdl_sub.config.config_file import ConfigFile -from ytdl_sub.downloaders.downloader import YtDlpDownloader +from ytdl_sub.downloaders.url.downloader import BaseUrlDownloader from ytdl_sub.downloaders.ytdlp import YTDLP from ytdl_sub.entries.variables.kwargs import DESCRIPTION from ytdl_sub.entries.variables.kwargs import EPOCH @@ -115,7 +114,7 @@ def mock_download_collection_thumbnail(mock_downloaded_file_path): return False with patch( - "ytdl_sub.downloaders.downloader.download_and_convert_url_thumbnail", + "ytdl_sub.downloaders.url.downloader.download_and_convert_url_thumbnail", new=_mock_download_and_convert_url_thumbnail, ): yield # TODO: create file here @@ -202,7 +201,7 @@ def mock_download_collection_entries( with patch.object( YTDLP, "extract_info_via_info_json", new=_write_entries_to_working_dir ), patch.object( - YtDlpDownloader, "_extract_entry_info_with_retry", new=lambda _, entry: entry + BaseUrlDownloader, "_extract_entry_info_with_retry", new=lambda _, entry: entry ): # Stub out metadata. TODO: update this if we do metadata plugins yield