danvics/ytdl-sub
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Merge branch 'master' into jesse/reformat

Browse source
This commit is contained in:
Jesse Bannon 2023-03-14 23:43:08 -07:00
commit 353574401e
16 changed files with 164 additions and 103 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

10
src/ytdl_sub/config/preset.py
View file

@ -17,8 +17,8 @@ from ytdl_sub.config.preset_class_mappings import PluginMapping
from ytdl_sub.config.preset_options import OutputOptions
from ytdl_sub.config.preset_options import Overrides
from ytdl_sub.config.preset_options import YTDLOptions
from ytdl_sub.downloaders.downloader import BaseDownloader
from ytdl_sub.downloaders.downloader import DownloaderValidator
from ytdl_sub.downloaders.base_downloader import BaseDownloader
from ytdl_sub.downloaders.base_downloader import BaseDownloaderValidator
from ytdl_sub.entries.entry import Entry
from ytdl_sub.plugins.plugin import Plugin
from ytdl_sub.plugins.plugin import PluginOptions
@ -250,7 +250,7 @@ class Preset(_PresetShell):
def __validate_and_get_downloader_options(
self, downloader_source: str, downloader: Type[BaseDownloader]
) -> DownloaderValidator:
) -> BaseDownloaderValidator:
# Remove the download_strategy key before validating it against the downloader options
# TODO: make this cleaner
del self._dict[downloader_source]["download_strategy"]
@ -260,9 +260,9 @@ class Preset(_PresetShell):
def __validate_and_get_downloader_and_options(
self,
) -> Tuple[Type[BaseDownloader], DownloaderValidator]:
) -> Tuple[Type[BaseDownloader], BaseDownloaderValidator]:
downloader: Optional[Type[BaseDownloader]] = None
download_options: Optional[DownloaderValidator] = None
download_options: Optional[BaseDownloaderValidator] = None
downloader_sources = DownloadStrategyMapping.sources()
for key in self._keys:

6
src/ytdl_sub/config/preset_class_mappings.py
View file

@ -2,9 +2,9 @@ from typing import Dict
from typing import List
from typing import Type
from ytdl_sub.downloaders.downloader import BaseDownloader
from ytdl_sub.downloaders.generic.multi_url import MultiUrlDownloader
from ytdl_sub.downloaders.generic.url import UrlDownloader
from ytdl_sub.downloaders.base_downloader import BaseDownloader
from ytdl_sub.downloaders.url.multi_url import MultiUrlDownloader
from ytdl_sub.downloaders.url.url import UrlDownloader
from ytdl_sub.plugins.audio_extract import AudioExtractPlugin
from ytdl_sub.plugins.chapters import ChaptersPlugin
from ytdl_sub.plugins.date_range import DateRangePlugin

76
src/ytdl_sub/downloaders/base_downloader.py Normal file
View file

@ -0,0 +1,76 @@
import abc
from abc import ABC
from typing import Generic
from typing import Iterable
from typing import List
from typing import Type
from typing import TypeVar
from ytdl_sub.config.preset_options import AddsVariablesMixin
from ytdl_sub.config.preset_options import Overrides
from ytdl_sub.downloaders.ytdl_options_builder import YTDLOptionsBuilder
from ytdl_sub.entries.entry import Entry
from ytdl_sub.plugins.plugin import Plugin
from ytdl_sub.plugins.plugin import PluginOptions
from ytdl_sub.validators.strict_dict_validator import StrictDictValidator
from ytdl_sub.ytdl_additions.enhanced_download_archive import DownloadArchiver
from ytdl_sub.ytdl_additions.enhanced_download_archive import EnhancedDownloadArchive
class BaseDownloaderValidator(StrictDictValidator, AddsVariablesMixin, ABC):
pass
BaseDownloaderOptionsT = TypeVar("BaseDownloaderOptionsT", bound=BaseDownloaderValidator)
class BaseDownloaderPluginOptions(PluginOptions):
_optional_keys = {"no-op"}
class BaseDownloaderPlugin(Plugin[BaseDownloaderPluginOptions], ABC):
def __init__(
self,
overrides: Overrides,
enhanced_download_archive: EnhancedDownloadArchive,
):
super().__init__(
# Downloader plugins do not have exposed YAML options, so keep it blank.
# Use init instead.
plugin_options=BaseDownloaderPluginOptions(name=self.__class__.__name__, value={}),
overrides=overrides,
enhanced_download_archive=enhanced_download_archive,
)
class BaseDownloader(DownloadArchiver, Generic[BaseDownloaderOptionsT], ABC):
downloader_options_type: Type[BaseDownloaderOptionsT]
def __init__(
self,
download_options: BaseDownloaderOptionsT,
enhanced_download_archive: EnhancedDownloadArchive,
download_ytdl_options: YTDLOptionsBuilder,
metadata_ytdl_options: YTDLOptionsBuilder,
overrides: Overrides,
):
super().__init__(enhanced_download_archive=enhanced_download_archive)
self.download_options = download_options
self.overrides = overrides
self._download_ytdl_options_builder = download_ytdl_options
self._metadata_ytdl_options_builder = metadata_ytdl_options
@abc.abstractmethod
def download_metadata(self) -> Iterable[Entry]:
"""Gathers metadata of all entries to download"""
@abc.abstractmethod
def download(self, entry: Entry) -> Entry:
"""The function to perform the download of all media entries"""
# pylint: disable=no-self-use
def added_plugins(self) -> List[BaseDownloaderPlugin]:
"""Add these plugins from the Downloader to the subscription"""
return []
# pylint: enable=no-self-use

0
src/ytdl_sub/downloaders/generic/__init__.py → src/ytdl_sub/downloaders/url/__init__.py
View file

86
src/ytdl_sub/downloaders/downloader.py → src/ytdl_sub/downloaders/url/downloader.py
View file

@ -4,21 +4,21 @@ import os
from abc import ABC
from pathlib import Path
from typing import Dict
from typing import Generic
from typing import Iterable
from typing import Iterator
from typing import List
from typing import Optional
from typing import Set
from typing import Tuple
from typing import Type
from typing import TypeVar
from ytdl_sub.config.preset_options import AddsVariablesMixin
from ytdl_sub.config.preset_options import Overrides
from ytdl_sub.downloaders.generic.validators import MultiUrlValidator
from ytdl_sub.downloaders.generic.validators import UrlThumbnailListValidator
from ytdl_sub.downloaders.generic.validators import UrlValidator
from ytdl_sub.downloaders.base_downloader import BaseDownloader
from ytdl_sub.downloaders.base_downloader import BaseDownloaderOptionsT
from ytdl_sub.downloaders.base_downloader import BaseDownloaderPlugin
from ytdl_sub.downloaders.base_downloader import BaseDownloaderValidator
from ytdl_sub.downloaders.url.validators import MultiUrlValidator
from ytdl_sub.downloaders.url.validators import UrlThumbnailListValidator
from ytdl_sub.downloaders.url.validators import UrlValidator
from ytdl_sub.downloaders.ytdl_options_builder import YTDLOptionsBuilder
from ytdl_sub.downloaders.ytdlp import YTDLP
from ytdl_sub.entries.entry import Entry
@ -32,14 +32,11 @@ from ytdl_sub.entries.variables.kwargs import SOURCE_ENTRY
from ytdl_sub.entries.variables.kwargs import SPONSORBLOCK_CHAPTERS
from ytdl_sub.entries.variables.kwargs import UPLOAD_DATE_INDEX
from ytdl_sub.plugins.plugin import Plugin
from ytdl_sub.plugins.plugin import PluginOptions
from ytdl_sub.utils.file_handler import FileHandler
from ytdl_sub.utils.logger import Logger
from ytdl_sub.utils.thumbnail import ThumbnailTypes
from ytdl_sub.utils.thumbnail import convert_download_thumbnail
from ytdl_sub.utils.thumbnail import download_and_convert_url_thumbnail
from ytdl_sub.validators.strict_dict_validator import StrictDictValidator
from ytdl_sub.ytdl_additions.enhanced_download_archive import DownloadArchiver
from ytdl_sub.ytdl_additions.enhanced_download_archive import EnhancedDownloadArchive
# pylint: disable=too-many-instance-attributes
@ -47,7 +44,7 @@ from ytdl_sub.ytdl_additions.enhanced_download_archive import EnhancedDownloadAr
download_logger = Logger.get(name="downloader")
class DownloaderValidator(StrictDictValidator, AddsVariablesMixin, ABC):
class DownloaderValidator(BaseDownloaderValidator, ABC):
"""
Placeholder class to define downloader options
"""
@ -81,68 +78,13 @@ class DownloaderValidator(StrictDictValidator, AddsVariablesMixin, ABC):
)
DownloaderOptionsT = TypeVar("DownloaderOptionsT", bound=DownloaderValidator)
class URLDownloadState:
def __init__(self, entries_total: int):
self.entries_total = entries_total
self.entries_downloaded = 0
class EmptyPluginOptions(PluginOptions):
_optional_keys = {"no-op"}
class BaseDownloaderPlugin(Plugin[EmptyPluginOptions], ABC):
def __init__(
self,
overrides: Overrides,
enhanced_download_archive: EnhancedDownloadArchive,
):
super().__init__(
# Downloader plugins do not have exposed YAML options, so keep it blank.
# Use init instead.
plugin_options=EmptyPluginOptions(name=self.__class__.__name__, value={}),
overrides=overrides,
enhanced_download_archive=enhanced_download_archive,
)
class BaseDownloader(DownloadArchiver, Generic[DownloaderOptionsT], ABC):
downloader_options_type: Type[DownloaderValidator] = DownloaderValidator
def __init__(
self,
download_options: DownloaderOptionsT,
enhanced_download_archive: EnhancedDownloadArchive,
download_ytdl_options: YTDLOptionsBuilder,
metadata_ytdl_options: YTDLOptionsBuilder,
overrides: Overrides,
):
super().__init__(enhanced_download_archive=enhanced_download_archive)
self.download_options = download_options
self.overrides = overrides
self._download_ytdl_options_builder = download_ytdl_options
self._metadata_ytdl_options_builder = metadata_ytdl_options
@abc.abstractmethod
def download_metadata(self) -> Iterable[Entry]:
"""Gathers metadata of all entries to download"""
@abc.abstractmethod
def download(self, entry: Entry) -> Entry:
"""The function to perform the download of all media entries"""
# pylint: disable=no-self-use
def added_plugins(self) -> List[BaseDownloaderPlugin]:
"""Add these plugins from the Downloader to the subscription"""
return []
# pylint: enable=no-self-use
class YtDlpThumbnailPlugin(BaseDownloaderPlugin):
class UrlDownloaderThumbnailPlugin(BaseDownloaderPlugin):
def __init__(
self,
overrides: Overrides,
@ -240,7 +182,7 @@ class YtDlpThumbnailPlugin(BaseDownloaderPlugin):
return entry
class YtDlpCollectionVariablePlugin(BaseDownloaderPlugin):
class UrlDownloaderCollectionVariablePlugin(BaseDownloaderPlugin):
def __init__(
self,
overrides: Overrides,
@ -270,7 +212,7 @@ class YtDlpCollectionVariablePlugin(BaseDownloaderPlugin):
return entry
class YtDlpDownloader(BaseDownloader[DownloaderOptionsT], ABC):
class BaseUrlDownloader(BaseDownloader[BaseDownloaderOptionsT], ABC):
"""
Class that interacts with ytdl to perform the download of metadata and content,
and should translate that to list of Entry objects.
@ -283,12 +225,12 @@ class YtDlpDownloader(BaseDownloader[DownloaderOptionsT], ABC):
2. Collection variable plugin to add to each entry
"""
return [
YtDlpThumbnailPlugin(
UrlDownloaderThumbnailPlugin(
overrides=self.overrides,
enhanced_download_archive=self._enhanced_download_archive,
collection_urls=self.collection.urls.list,
),
YtDlpCollectionVariablePlugin(
UrlDownloaderCollectionVariablePlugin(
overrides=self.overrides,
enhanced_download_archive=self._enhanced_download_archive,
collection_urls=self.collection.urls.list,
@ -307,7 +249,7 @@ class YtDlpDownloader(BaseDownloader[DownloaderOptionsT], ABC):
def __init__(
self,
download_options: DownloaderOptionsT,
download_options: BaseDownloaderOptionsT,
enhanced_download_archive: EnhancedDownloadArchive,
download_ytdl_options: YTDLOptionsBuilder,
metadata_ytdl_options: YTDLOptionsBuilder,

8
src/ytdl_sub/downloaders/generic/multi_url.py → src/ytdl_sub/downloaders/url/multi_url.py
View file

@ -1,6 +1,6 @@
from ytdl_sub.downloaders.downloader import DownloaderValidator
from ytdl_sub.downloaders.downloader import YtDlpDownloader
from ytdl_sub.downloaders.generic.validators import MultiUrlValidator
from ytdl_sub.downloaders.url.downloader import BaseUrlDownloader
from ytdl_sub.downloaders.url.downloader import DownloaderValidator
from ytdl_sub.downloaders.url.validators import MultiUrlValidator
class MultiUrlDownloadOptions(MultiUrlValidator, DownloaderValidator):
@ -44,5 +44,5 @@ class MultiUrlDownloadOptions(MultiUrlValidator, DownloaderValidator):
return self
class MultiUrlDownloader(YtDlpDownloader[MultiUrlDownloadOptions]):
class MultiUrlDownloader(BaseUrlDownloader[MultiUrlDownloadOptions]):
downloader_options_type = MultiUrlDownloadOptions

10
src/ytdl_sub/downloaders/generic/url.py → src/ytdl_sub/downloaders/url/url.py
View file

@ -1,7 +1,7 @@
from ytdl_sub.downloaders.downloader import DownloaderValidator
from ytdl_sub.downloaders.downloader import YtDlpDownloader
from ytdl_sub.downloaders.generic.validators import MultiUrlValidator
from ytdl_sub.downloaders.generic.validators import UrlValidator
from ytdl_sub.downloaders.url.downloader import BaseUrlDownloader
from ytdl_sub.downloaders.url.downloader import DownloaderValidator
from ytdl_sub.downloaders.url.validators import MultiUrlValidator
from ytdl_sub.downloaders.url.validators import UrlValidator
class UrlDownloadOptions(UrlValidator, DownloaderValidator):
@ -36,5 +36,5 @@ class UrlDownloadOptions(UrlValidator, DownloaderValidator):
)
class UrlDownloader(YtDlpDownloader[UrlDownloadOptions]):
class UrlDownloader(BaseUrlDownloader[UrlDownloadOptions]):
downloader_options_type = UrlDownloadOptions

0
src/ytdl_sub/downloaders/generic/validators.py → src/ytdl_sub/downloaders/url/validators.py
View file

6
src/ytdl_sub/subscriptions/base_subscription.py
View file

@ -8,8 +8,8 @@ from ytdl_sub.config.preset import PresetPlugins
from ytdl_sub.config.preset_options import OutputOptions
from ytdl_sub.config.preset_options import Overrides
from ytdl_sub.config.preset_options import YTDLOptions
from ytdl_sub.downloaders.downloader import BaseDownloader
from ytdl_sub.downloaders.downloader import DownloaderValidator
from ytdl_sub.downloaders.base_downloader import BaseDownloader
from ytdl_sub.downloaders.base_downloader import BaseDownloaderValidator
from ytdl_sub.ytdl_additions.enhanced_download_archive import EnhancedDownloadArchive
@ -61,7 +61,7 @@ class BaseSubscription(ABC):
return self._preset_options.downloader
@property
def downloader_options(self) -> DownloaderValidator:
def downloader_options(self) -> BaseDownloaderValidator:
"""
Returns
-------

2
src/ytdl_sub/subscriptions/subscription_ytdl_options.py
View file

@ -6,7 +6,7 @@ from typing import Type
from typing import TypeVar
from ytdl_sub.config.preset import Preset
from ytdl_sub.downloaders.downloader import BaseDownloader
from ytdl_sub.downloaders.base_downloader import BaseDownloader
from ytdl_sub.downloaders.ytdl_options_builder import YTDLOptionsBuilder
from ytdl_sub.plugins.audio_extract import AudioExtractPlugin
from ytdl_sub.plugins.chapters import ChaptersPlugin

7
src/ytdl_sub/utils/yaml.py
View file

@ -36,13 +36,18 @@ def load_yaml(file_path: str | Path) -> Dict:
try:
with open(file_path, "r", encoding="utf-8") as file:
return yaml.safe_load(file)
output = yaml.safe_load(file)
except YAMLError as yaml_exception:
logger.debug(yaml_exception)
raise InvalidYamlException(
f"'{file_path}' has invalid YAML, copy-paste it into a YAML checker to find the issue."
) from yaml_exception
if not isinstance(output, dict):
raise InvalidYamlException(f"'{file_path}' was specified but does not contain any YAML.")
return output
def dump_yaml(to_dump: Dict) -> str:
"""

1
tests/e2e/bandcamp/test_bandcamp.py
View file

@ -3,7 +3,6 @@ from conftest import assert_logs
from expected_download import assert_expected_downloads
from expected_transaction_log import assert_transaction_log_matches
import ytdl_sub.downloaders.downloader
from ytdl_sub.downloaders.ytdlp import YTDLP
from ytdl_sub.subscriptions.subscription import Subscription

1
tests/e2e/plugins/test_date_range.py
View file

@ -6,7 +6,6 @@ from conftest import assert_logs
from expected_download import assert_expected_downloads
from expected_transaction_log import assert_transaction_log_matches
import ytdl_sub.downloaders.downloader
from ytdl_sub.downloaders.ytdlp import YTDLP
from ytdl_sub.subscriptions.subscription import Subscription

1
tests/e2e/youtube/test_playlist.py
View file

@ -4,7 +4,6 @@ from e2e.conftest import mock_run_from_cli
from expected_download import assert_expected_downloads
from expected_transaction_log import assert_transaction_log_matches
import ytdl_sub.downloaders.downloader
from ytdl_sub.downloaders.ytdlp import YTDLP
from ytdl_sub.subscriptions.subscription import Subscription

7
tests/unit/prebuilt_presets/conftest.py
View file

@ -1,6 +1,5 @@
import contextlib
import os
import tempfile
from pathlib import Path
from typing import Callable
from typing import Dict
@ -11,7 +10,7 @@ import pytest
from resources import copy_file_fixture
from ytdl_sub.config.config_file import ConfigFile
from ytdl_sub.downloaders.downloader import YtDlpDownloader
from ytdl_sub.downloaders.url.downloader import BaseUrlDownloader
from ytdl_sub.downloaders.ytdlp import YTDLP
from ytdl_sub.entries.variables.kwargs import DESCRIPTION
from ytdl_sub.entries.variables.kwargs import EPOCH
@ -115,7 +114,7 @@ def mock_download_collection_thumbnail(mock_downloaded_file_path):
return False
with patch(
"ytdl_sub.downloaders.downloader.download_and_convert_url_thumbnail",
"ytdl_sub.downloaders.url.downloader.download_and_convert_url_thumbnail",
new=_mock_download_and_convert_url_thumbnail,
):
yield # TODO: create file here
@ -202,7 +201,7 @@ def mock_download_collection_entries(
with patch.object(
YTDLP, "extract_info_via_info_json", new=_write_entries_to_working_dir
), patch.object(
YtDlpDownloader, "_extract_entry_info_with_retry", new=lambda _, entry: entry
BaseUrlDownloader, "_extract_entry_info_with_retry", new=lambda _, entry: entry
):
# Stub out metadata. TODO: update this if we do metadata plugins
yield

46
tests/unit/utils/test_yaml.py
View file

@ -35,17 +35,59 @@ def bad_yaml_file_path(bad_yaml) -> str:
FileHandler.delete(tmp_file.name)
@pytest.fixture
def single_int_file_path(bad_yaml) -> str:
# Do not delete the file in the context manager - for Windows compatibility
with tempfile.NamedTemporaryFile(suffix=".yaml", delete=False) as tmp_file:
tmp_file.write("0".encode("utf-8"))
tmp_file.flush()
try:
yield tmp_file.name
finally:
FileHandler.delete(tmp_file.name)
@pytest.fixture
def empty_yaml_file_path(bad_yaml) -> str:
# Do not delete the file in the context manager - for Windows compatibility
with tempfile.NamedTemporaryFile(suffix=".yaml", delete=False) as tmp_file:
pass
try:
yield tmp_file.name
finally:
FileHandler.delete(tmp_file.name)
def test_load_yaml_file_not_found():
file_path = "does_not_exist.yaml"
with pytest.raises(FileNotFoundException, match=f"The file '{file_path}' does not exist."):
load_yaml(file_path=file_path)
def test_load_yaml_invalid_syntax(bad_yaml_file_path):
def test_load_yaml_invalid_syntax(bad_yaml_file_path: str):
with pytest.raises(
InvalidYamlException,
match=re.escape(
f"'{bad_yaml_file_path}' has invalid YAML, copy-paste it into a YAML checker to find the issue."
f"'{bad_yaml_file_path}' has invalid YAML, "
f"copy-paste it into a YAML checker to find the issue."
),
):
load_yaml(file_path=bad_yaml_file_path)
def test_empty_yaml(empty_yaml_file_path: str):
with pytest.raises(
InvalidYamlException,
match=re.escape(f"'{empty_yaml_file_path}' was specified but does not contain any YAML."),
):
load_yaml(file_path=empty_yaml_file_path)
def test_empty_yaml_only_int(single_int_file_path: str):
with pytest.raises(
InvalidYamlException,
match=re.escape(f"'{single_int_file_path}' was specified but does not contain any YAML."),
):
load_yaml(file_path=single_int_file_path)