danvics/ytdl-sub
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

[REFACTOR] BaseDownloader (#542)

Browse source 
* [REFACTOR] BaseDownloader

* doh

* fix mock
This commit is contained in:
Jesse Bannon 2023-03-14 23:42:50 -07:00 committed by GitHub
parent f4807aa9a4
commit 7c692cda6e
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
14 changed files with 114 additions and 100 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

10
src/ytdl_sub/config/preset.py
View file

@ -17,8 +17,8 @@ from ytdl_sub.config.preset_class_mappings import PluginMapping
from ytdl_sub.config.preset_options import OutputOptions
from ytdl_sub.config.preset_options import Overrides
from ytdl_sub.config.preset_options import YTDLOptions
from ytdl_sub.downloaders.downloader import BaseDownloader
from ytdl_sub.downloaders.downloader import DownloaderValidator
from ytdl_sub.downloaders.base_downloader import BaseDownloader
from ytdl_sub.downloaders.base_downloader import BaseDownloaderValidator
from ytdl_sub.entries.entry import Entry
from ytdl_sub.plugins.plugin import Plugin
from ytdl_sub.plugins.plugin import PluginOptions
@ -250,7 +250,7 @@ class Preset(_PresetShell):
def __validate_and_get_downloader_options(
self, downloader_source: str, downloader: Type[BaseDownloader]
) -> DownloaderValidator:
) -> BaseDownloaderValidator:
# Remove the download_strategy key before validating it against the downloader options
# TODO: make this cleaner
del self._dict[downloader_source]["download_strategy"]
@ -260,9 +260,9 @@ class Preset(_PresetShell):
def __validate_and_get_downloader_and_options(
self,
) -> Tuple[Type[BaseDownloader], DownloaderValidator]:
) -> Tuple[Type[BaseDownloader], BaseDownloaderValidator]:
downloader: Optional[Type[BaseDownloader]] = None
download_options: Optional[DownloaderValidator] = None
download_options: Optional[BaseDownloaderValidator] = None
downloader_sources = DownloadStrategyMapping.sources()
for key in self._keys:

6
src/ytdl_sub/config/preset_class_mappings.py
View file

@ -2,9 +2,9 @@ from typing import Dict
from typing import List
from typing import Type
from ytdl_sub.downloaders.downloader import BaseDownloader
from ytdl_sub.downloaders.generic.multi_url import MultiUrlDownloader
from ytdl_sub.downloaders.generic.url import UrlDownloader
from ytdl_sub.downloaders.base_downloader import BaseDownloader
from ytdl_sub.downloaders.url.multi_url import MultiUrlDownloader
from ytdl_sub.downloaders.url.url import UrlDownloader
from ytdl_sub.plugins.audio_extract import AudioExtractPlugin
from ytdl_sub.plugins.chapters import ChaptersPlugin
from ytdl_sub.plugins.date_range import DateRangePlugin

76
src/ytdl_sub/downloaders/base_downloader.py Normal file
View file

@ -0,0 +1,76 @@
import abc
from abc import ABC
from typing import Generic
from typing import Iterable
from typing import List
from typing import Type
from typing import TypeVar
from ytdl_sub.config.preset_options import AddsVariablesMixin
from ytdl_sub.config.preset_options import Overrides
from ytdl_sub.downloaders.ytdl_options_builder import YTDLOptionsBuilder
from ytdl_sub.entries.entry import Entry
from ytdl_sub.plugins.plugin import Plugin
from ytdl_sub.plugins.plugin import PluginOptions
from ytdl_sub.validators.strict_dict_validator import StrictDictValidator
from ytdl_sub.ytdl_additions.enhanced_download_archive import DownloadArchiver
from ytdl_sub.ytdl_additions.enhanced_download_archive import EnhancedDownloadArchive
class BaseDownloaderValidator(StrictDictValidator, AddsVariablesMixin, ABC):
pass
BaseDownloaderOptionsT = TypeVar("BaseDownloaderOptionsT", bound=BaseDownloaderValidator)
class BaseDownloaderPluginOptions(PluginOptions):
_optional_keys = {"no-op"}
class BaseDownloaderPlugin(Plugin[BaseDownloaderPluginOptions], ABC):
def __init__(
self,
overrides: Overrides,
enhanced_download_archive: EnhancedDownloadArchive,
):
super().__init__(
# Downloader plugins do not have exposed YAML options, so keep it blank.
# Use init instead.
plugin_options=BaseDownloaderPluginOptions(name=self.__class__.__name__, value={}),
overrides=overrides,
enhanced_download_archive=enhanced_download_archive,
)
class BaseDownloader(DownloadArchiver, Generic[BaseDownloaderOptionsT], ABC):
downloader_options_type: Type[BaseDownloaderOptionsT]
def __init__(
self,
download_options: BaseDownloaderOptionsT,
enhanced_download_archive: EnhancedDownloadArchive,
download_ytdl_options: YTDLOptionsBuilder,
metadata_ytdl_options: YTDLOptionsBuilder,
overrides: Overrides,
):
super().__init__(enhanced_download_archive=enhanced_download_archive)
self.download_options = download_options
self.overrides = overrides
self._download_ytdl_options_builder = download_ytdl_options
self._metadata_ytdl_options_builder = metadata_ytdl_options
@abc.abstractmethod
def download_metadata(self) -> Iterable[Entry]:
"""Gathers metadata of all entries to download"""
@abc.abstractmethod
def download(self, entry: Entry) -> Entry:
"""The function to perform the download of all media entries"""
# pylint: disable=no-self-use
def added_plugins(self) -> List[BaseDownloaderPlugin]:
"""Add these plugins from the Downloader to the subscription"""
return []
# pylint: enable=no-self-use

0
src/ytdl_sub/downloaders/generic/__init__.py → src/ytdl_sub/downloaders/url/__init__.py
View file

86
src/ytdl_sub/downloaders/downloader.py → src/ytdl_sub/downloaders/url/downloader.py
View file

@ -4,21 +4,21 @@ import os
from abc import ABC
from pathlib import Path
from typing import Dict
from typing import Generic
from typing import Iterable
from typing import Iterator
from typing import List
from typing import Optional
from typing import Set
from typing import Tuple
from typing import Type
from typing import TypeVar
from ytdl_sub.config.preset_options import AddsVariablesMixin
from ytdl_sub.config.preset_options import Overrides
from ytdl_sub.downloaders.generic.validators import MultiUrlValidator
from ytdl_sub.downloaders.generic.validators import UrlThumbnailListValidator
from ytdl_sub.downloaders.generic.validators import UrlValidator
from ytdl_sub.downloaders.base_downloader import BaseDownloader
from ytdl_sub.downloaders.base_downloader import BaseDownloaderOptionsT
from ytdl_sub.downloaders.base_downloader import BaseDownloaderPlugin
from ytdl_sub.downloaders.base_downloader import BaseDownloaderValidator
from ytdl_sub.downloaders.url.validators import MultiUrlValidator
from ytdl_sub.downloaders.url.validators import UrlThumbnailListValidator
from ytdl_sub.downloaders.url.validators import UrlValidator
from ytdl_sub.downloaders.ytdl_options_builder import YTDLOptionsBuilder
from ytdl_sub.downloaders.ytdlp import YTDLP
from ytdl_sub.entries.entry import Entry
@ -32,14 +32,11 @@ from ytdl_sub.entries.variables.kwargs import SOURCE_ENTRY
from ytdl_sub.entries.variables.kwargs import SPONSORBLOCK_CHAPTERS
from ytdl_sub.entries.variables.kwargs import UPLOAD_DATE_INDEX
from ytdl_sub.plugins.plugin import Plugin
from ytdl_sub.plugins.plugin import PluginOptions
from ytdl_sub.utils.file_handler import FileHandler
from ytdl_sub.utils.logger import Logger
from ytdl_sub.utils.thumbnail import ThumbnailTypes
from ytdl_sub.utils.thumbnail import convert_download_thumbnail
from ytdl_sub.utils.thumbnail import download_and_convert_url_thumbnail
from ytdl_sub.validators.strict_dict_validator import StrictDictValidator
from ytdl_sub.ytdl_additions.enhanced_download_archive import DownloadArchiver
from ytdl_sub.ytdl_additions.enhanced_download_archive import EnhancedDownloadArchive
# pylint: disable=too-many-instance-attributes
@ -47,7 +44,7 @@ from ytdl_sub.ytdl_additions.enhanced_download_archive import EnhancedDownloadAr
download_logger = Logger.get(name="downloader")
class DownloaderValidator(StrictDictValidator, AddsVariablesMixin, ABC):
class DownloaderValidator(BaseDownloaderValidator, ABC):
"""
Placeholder class to define downloader options
"""
@ -81,68 +78,13 @@ class DownloaderValidator(StrictDictValidator, AddsVariablesMixin, ABC):
)
DownloaderOptionsT = TypeVar("DownloaderOptionsT", bound=DownloaderValidator)
class URLDownloadState:
def __init__(self, entries_total: int):
self.entries_total = entries_total
self.entries_downloaded = 0
class EmptyPluginOptions(PluginOptions):
_optional_keys = {"no-op"}
class BaseDownloaderPlugin(Plugin[EmptyPluginOptions], ABC):
def __init__(
self,
overrides: Overrides,
enhanced_download_archive: EnhancedDownloadArchive,
):
super().__init__(
# Downloader plugins do not have exposed YAML options, so keep it blank.
# Use init instead.
plugin_options=EmptyPluginOptions(name=self.__class__.__name__, value={}),
overrides=overrides,
enhanced_download_archive=enhanced_download_archive,
)
class BaseDownloader(DownloadArchiver, Generic[DownloaderOptionsT], ABC):
downloader_options_type: Type[DownloaderValidator] = DownloaderValidator
def __init__(
self,
download_options: DownloaderOptionsT,
enhanced_download_archive: EnhancedDownloadArchive,
download_ytdl_options: YTDLOptionsBuilder,
metadata_ytdl_options: YTDLOptionsBuilder,
overrides: Overrides,
):
super().__init__(enhanced_download_archive=enhanced_download_archive)
self.download_options = download_options
self.overrides = overrides
self._download_ytdl_options_builder = download_ytdl_options
self._metadata_ytdl_options_builder = metadata_ytdl_options
@abc.abstractmethod
def download_metadata(self) -> Iterable[Entry]:
"""Gathers metadata of all entries to download"""
@abc.abstractmethod
def download(self, entry: Entry) -> Entry:
"""The function to perform the download of all media entries"""
# pylint: disable=no-self-use
def added_plugins(self) -> List[BaseDownloaderPlugin]:
"""Add these plugins from the Downloader to the subscription"""
return []
# pylint: enable=no-self-use
class YtDlpThumbnailPlugin(BaseDownloaderPlugin):
class UrlDownloaderThumbnailPlugin(BaseDownloaderPlugin):
def __init__(
self,
overrides: Overrides,
@ -240,7 +182,7 @@ class YtDlpThumbnailPlugin(BaseDownloaderPlugin):
return entry
class YtDlpCollectionVariablePlugin(BaseDownloaderPlugin):
class UrlDownloaderCollectionVariablePlugin(BaseDownloaderPlugin):
def __init__(
self,
overrides: Overrides,
@ -270,7 +212,7 @@ class YtDlpCollectionVariablePlugin(BaseDownloaderPlugin):
return entry
class YtDlpDownloader(BaseDownloader[DownloaderOptionsT], ABC):
class BaseUrlDownloader(BaseDownloader[BaseDownloaderOptionsT], ABC):
"""
Class that interacts with ytdl to perform the download of metadata and content,
and should translate that to list of Entry objects.
@ -283,12 +225,12 @@ class YtDlpDownloader(BaseDownloader[DownloaderOptionsT], ABC):
2. Collection variable plugin to add to each entry
"""
return [
YtDlpThumbnailPlugin(
UrlDownloaderThumbnailPlugin(
overrides=self.overrides,
enhanced_download_archive=self._enhanced_download_archive,
collection_urls=self.collection.urls.list,
),
YtDlpCollectionVariablePlugin(
UrlDownloaderCollectionVariablePlugin(
overrides=self.overrides,
enhanced_download_archive=self._enhanced_download_archive,
collection_urls=self.collection.urls.list,
@ -307,7 +249,7 @@ class YtDlpDownloader(BaseDownloader[DownloaderOptionsT], ABC):
def __init__(
self,
download_options: DownloaderOptionsT,
download_options: BaseDownloaderOptionsT,
enhanced_download_archive: EnhancedDownloadArchive,
download_ytdl_options: YTDLOptionsBuilder,
metadata_ytdl_options: YTDLOptionsBuilder,

8
src/ytdl_sub/downloaders/generic/multi_url.py → src/ytdl_sub/downloaders/url/multi_url.py
View file

@ -1,6 +1,6 @@
from ytdl_sub.downloaders.downloader import DownloaderValidator
from ytdl_sub.downloaders.downloader import YtDlpDownloader
from ytdl_sub.downloaders.generic.validators import MultiUrlValidator
from ytdl_sub.downloaders.url.downloader import BaseUrlDownloader
from ytdl_sub.downloaders.url.downloader import DownloaderValidator
from ytdl_sub.downloaders.url.validators import MultiUrlValidator
class MultiUrlDownloadOptions(MultiUrlValidator, DownloaderValidator):
@ -44,5 +44,5 @@ class MultiUrlDownloadOptions(MultiUrlValidator, DownloaderValidator):
return self
class MultiUrlDownloader(YtDlpDownloader[MultiUrlDownloadOptions]):
class MultiUrlDownloader(BaseUrlDownloader[MultiUrlDownloadOptions]):
downloader_options_type = MultiUrlDownloadOptions

10
src/ytdl_sub/downloaders/generic/url.py → src/ytdl_sub/downloaders/url/url.py
View file

@ -1,7 +1,7 @@
from ytdl_sub.downloaders.downloader import DownloaderValidator
from ytdl_sub.downloaders.downloader import YtDlpDownloader
from ytdl_sub.downloaders.generic.validators import MultiUrlValidator
from ytdl_sub.downloaders.generic.validators import UrlValidator
from ytdl_sub.downloaders.url.downloader import BaseUrlDownloader
from ytdl_sub.downloaders.url.downloader import DownloaderValidator
from ytdl_sub.downloaders.url.validators import MultiUrlValidator
from ytdl_sub.downloaders.url.validators import UrlValidator
class UrlDownloadOptions(UrlValidator, DownloaderValidator):
@ -36,5 +36,5 @@ class UrlDownloadOptions(UrlValidator, DownloaderValidator):
)
class UrlDownloader(YtDlpDownloader[UrlDownloadOptions]):
class UrlDownloader(BaseUrlDownloader[UrlDownloadOptions]):
downloader_options_type = UrlDownloadOptions

0
src/ytdl_sub/downloaders/generic/validators.py → src/ytdl_sub/downloaders/url/validators.py
View file

6
src/ytdl_sub/subscriptions/base_subscription.py
View file

@ -8,8 +8,8 @@ from ytdl_sub.config.preset import PresetPlugins
from ytdl_sub.config.preset_options import OutputOptions
from ytdl_sub.config.preset_options import Overrides
from ytdl_sub.config.preset_options import YTDLOptions
from ytdl_sub.downloaders.downloader import BaseDownloader
from ytdl_sub.downloaders.downloader import DownloaderValidator
from ytdl_sub.downloaders.base_downloader import BaseDownloader
from ytdl_sub.downloaders.base_downloader import BaseDownloaderValidator
from ytdl_sub.ytdl_additions.enhanced_download_archive import EnhancedDownloadArchive
@ -61,7 +61,7 @@ class BaseSubscription(ABC):
return self._preset_options.downloader
@property
def downloader_options(self) -> DownloaderValidator:
def downloader_options(self) -> BaseDownloaderValidator:
"""
Returns
-------

2
src/ytdl_sub/subscriptions/subscription_ytdl_options.py
View file

@ -6,7 +6,7 @@ from typing import Type
from typing import TypeVar
from ytdl_sub.config.preset import Preset
from ytdl_sub.downloaders.downloader import BaseDownloader
from ytdl_sub.downloaders.base_downloader import BaseDownloader
from ytdl_sub.downloaders.ytdl_options_builder import YTDLOptionsBuilder
from ytdl_sub.plugins.audio_extract import AudioExtractPlugin
from ytdl_sub.plugins.chapters import ChaptersPlugin

1
tests/e2e/bandcamp/test_bandcamp.py
View file

@ -3,7 +3,6 @@ from conftest import assert_logs
from expected_download import assert_expected_downloads
from expected_transaction_log import assert_transaction_log_matches
import ytdl_sub.downloaders.downloader
from ytdl_sub.downloaders.ytdlp import YTDLP
from ytdl_sub.subscriptions.subscription import Subscription

1
tests/e2e/plugins/test_date_range.py
View file

@ -6,7 +6,6 @@ from conftest import assert_logs
from expected_download import assert_expected_downloads
from expected_transaction_log import assert_transaction_log_matches
import ytdl_sub.downloaders.downloader
from ytdl_sub.downloaders.ytdlp import YTDLP
from ytdl_sub.subscriptions.subscription import Subscription

1
tests/e2e/youtube/test_playlist.py
View file

@ -4,7 +4,6 @@ from e2e.conftest import mock_run_from_cli
from expected_download import assert_expected_downloads
from expected_transaction_log import assert_transaction_log_matches
import ytdl_sub.downloaders.downloader
from ytdl_sub.downloaders.ytdlp import YTDLP
from ytdl_sub.subscriptions.subscription import Subscription

7
tests/unit/prebuilt_presets/conftest.py
View file

@ -1,6 +1,5 @@
import contextlib
import os
import tempfile
from pathlib import Path
from typing import Callable
from typing import Dict
@ -11,7 +10,7 @@ import pytest
from resources import copy_file_fixture
from ytdl_sub.config.config_file import ConfigFile
from ytdl_sub.downloaders.downloader import YtDlpDownloader
from ytdl_sub.downloaders.url.downloader import BaseUrlDownloader
from ytdl_sub.downloaders.ytdlp import YTDLP
from ytdl_sub.entries.variables.kwargs import DESCRIPTION
from ytdl_sub.entries.variables.kwargs import EPOCH
@ -115,7 +114,7 @@ def mock_download_collection_thumbnail(mock_downloaded_file_path):
return False
with patch(
"ytdl_sub.downloaders.downloader.download_and_convert_url_thumbnail",
"ytdl_sub.downloaders.url.downloader.download_and_convert_url_thumbnail",
new=_mock_download_and_convert_url_thumbnail,
):
yield # TODO: create file here
@ -202,7 +201,7 @@ def mock_download_collection_entries(
with patch.object(
YTDLP, "extract_info_via_info_json", new=_write_entries_to_working_dir
), patch.object(
YtDlpDownloader, "_extract_entry_info_with_retry", new=lambda _, entry: entry
BaseUrlDownloader, "_extract_entry_info_with_retry", new=lambda _, entry: entry
):
# Stub out metadata. TODO: update this if we do metadata plugins
yield