danvics/ytdl-sub
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

[BACKEND] Only have single entry type

Browse source
This commit is contained in:
Jesse Bannon 2022-09-27 19:04:57 -07:00
parent 3a37138ce6
commit db99eda41f
14 changed files with 29 additions and 221 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

21
docs/config.rst
View file

@ -50,8 +50,6 @@ _______
:member-order: bysource
:exclude-members: get_date_range
.. autofunction:: ytdl_sub.downloaders.youtube.channel.YoutubeChannelDownloader.added_override_variables()
.. autofunction:: ytdl_sub.downloaders.youtube.channel.YoutubeChannelDownloader.ytdl_option_defaults()
-------------------------------------------------------------------------------
@ -64,8 +62,6 @@ ________
:members: playlist_url
:member-order: bysource
.. autofunction:: ytdl_sub.downloaders.youtube.playlist.YoutubePlaylistDownloader.added_override_variables()
.. autofunction:: ytdl_sub.downloaders.youtube.playlist.YoutubePlaylistDownloader.ytdl_option_defaults()
-------------------------------------------------------------------------------
@ -300,26 +296,9 @@ Source Variables
----------------
.. autoclass:: ytdl_sub.entries.variables.entry_variables.EntryVariables
.. _youtube-variables:
Youtube Variables
^^^^^^^^^^^^^^^^^
.. automodule:: ytdl_sub.entries.variables.youtube_variables
:members:
:inherited-members:
:undoc-members:
:exclude-members: source_variables
.. _soundcloud-variables:
Soundcloud Variables
^^^^^^^^^^^^^^^^^^^^
.. automodule:: ytdl_sub.entries.variables.soundcloud_variables
:members:
:inherited-members:
:undoc-members:
:exclude-members: source_variables
-------------------------------------------------------------------------------

3
src/ytdl_sub/config/preset.py
View file

@ -19,6 +19,7 @@ from ytdl_sub.config.preset_options import Overrides
from ytdl_sub.config.preset_options import YTDLOptions
from ytdl_sub.downloaders.downloader import Downloader
from ytdl_sub.downloaders.downloader import DownloaderValidator
from ytdl_sub.entries.entry import Entry
from ytdl_sub.plugins.plugin import Plugin
from ytdl_sub.plugins.plugin import PluginOptions
from ytdl_sub.utils.yaml import load_yaml
@ -129,7 +130,7 @@ class Preset(StrictDictValidator):
@property
def _source_variables(self) -> List[str]:
return self.downloader.downloader_entry_type.source_variables()
return Entry.source_variables()
def __validate_and_get_downloader(self, downloader_source: str) -> Type[Downloader]:
return self._validate_key(key=downloader_source, validator=DownloadStrategyValidator).get(

6
src/ytdl_sub/downloaders/downloader.py
View file

@ -86,17 +86,15 @@ class DownloaderValidator(StrictDictValidator, AddsVariablesMixin, ABC):
DownloaderOptionsT = TypeVar("DownloaderOptionsT", bound=DownloaderValidator)
DownloaderEntryT = TypeVar("DownloaderEntryT", bound=Entry)
class Downloader(DownloadArchiver, Generic[DownloaderOptionsT, DownloaderEntryT], ABC):
class Downloader(DownloadArchiver, Generic[DownloaderOptionsT], ABC):
"""
Class that interacts with ytdl to perform the download of metadata and content,
and should translate that to list of Entry objects.
"""
downloader_options_type: Type[DownloaderValidator] = DownloaderValidator
downloader_entry_type: Type[Entry] = Entry
supports_download_archive: bool = True
supports_subtitles: bool = True
@ -473,7 +471,7 @@ class Downloader(DownloadArchiver, Generic[DownloaderOptionsT, DownloaderEntryT]
def download(
self,
) -> Iterable[DownloaderEntryT] | Iterable[Tuple[DownloaderEntryT, FileMetadata]]:
) -> Iterable[Entry] | Iterable[Tuple[Entry, FileMetadata]]:
"""The function to perform the download of all media entries"""
# download the bottom-most urls first since they are top-priority
for collection_url in reversed(self.collection.collection_urls.list):

4
src/ytdl_sub/downloaders/generic/collection.py
View file

@ -1,7 +1,6 @@
from ytdl_sub.downloaders.downloader import Downloader
from ytdl_sub.downloaders.downloader import DownloaderValidator
from ytdl_sub.downloaders.generic.collection_validator import CollectionValidator
from ytdl_sub.entries.entry import Entry
class CollectionDownloadOptions(CollectionValidator, DownloaderValidator):
@ -41,6 +40,5 @@ class CollectionDownloadOptions(CollectionValidator, DownloaderValidator):
return self
class CollectionDownloader(Downloader[CollectionDownloadOptions, Entry]):
class CollectionDownloader(Downloader[CollectionDownloadOptions]):
downloader_options_type = CollectionDownloadOptions
downloader_entry_type = Entry

4
src/ytdl_sub/downloaders/soundcloud/albums_and_singles.py
View file

@ -89,9 +89,7 @@ class SoundcloudAlbumsAndSinglesDownloadOptions(DownloaderValidator):
return self._url
class SoundcloudAlbumsAndSinglesDownloader(
Downloader[SoundcloudAlbumsAndSinglesDownloadOptions, Entry]
):
class SoundcloudAlbumsAndSinglesDownloader(Downloader[SoundcloudAlbumsAndSinglesDownloadOptions]):
downloader_options_type = SoundcloudAlbumsAndSinglesDownloadOptions
downloader_entry_type = Entry

29
src/ytdl_sub/downloaders/youtube/abc.py
View file

@ -1,29 +0,0 @@
from abc import ABC
from typing import Generic
from typing import TypeVar
from ytdl_sub.downloaders.downloader import Downloader
from ytdl_sub.downloaders.downloader import DownloaderValidator
from ytdl_sub.entries.youtube import YoutubeVideo
class YoutubeDownloaderOptions(DownloaderValidator, ABC):
"""
Abstract source validator for all soundcloud sources.
"""
YoutubeDownloaderOptionsT = TypeVar("YoutubeDownloaderOptionsT", bound=YoutubeDownloaderOptions)
YoutubeVideoT = TypeVar("YoutubeVideoT", bound=YoutubeVideo)
class YoutubeDownloader(
Downloader[YoutubeDownloaderOptionsT, YoutubeVideoT],
Generic[YoutubeDownloaderOptionsT, YoutubeVideoT],
ABC,
):
"""
Class that handles downloading youtube entries via ytdl and converting them into
YoutubeVideo like objects. Reserved for any future logic that is shared amongst all YT
downloaders.
"""

18
src/ytdl_sub/downloaders/youtube/channel.py
View file

@ -1,17 +1,15 @@
from typing import Dict
from typing import Generator
from typing import List
from typing import Optional
from ytdl_sub.downloaders.downloader import Downloader
from ytdl_sub.downloaders.downloader import DownloaderValidator
from ytdl_sub.downloaders.generic.collection_validator import CollectionValidator
from ytdl_sub.downloaders.youtube.abc import YoutubeDownloader
from ytdl_sub.downloaders.youtube.abc import YoutubeDownloaderOptions
from ytdl_sub.entries.youtube import YoutubeVideo
from ytdl_sub.validators.string_formatter_validators import OverridesStringFormatterValidator
from ytdl_sub.validators.url_validator import YoutubeChannelUrlValidator
class YoutubeChannelDownloaderOptions(YoutubeDownloaderOptions):
class YoutubeChannelDownloaderOptions(DownloaderValidator):
"""
Downloads all videos from a youtube channel.
@ -104,9 +102,8 @@ class YoutubeChannelDownloaderOptions(YoutubeDownloaderOptions):
return self._channel_banner_path
class YoutubeChannelDownloader(YoutubeDownloader[YoutubeChannelDownloaderOptions, YoutubeVideo]):
class YoutubeChannelDownloader(Downloader[YoutubeChannelDownloaderOptions]):
downloader_options_type = YoutubeChannelDownloaderOptions
downloader_entry_type = YoutubeVideo
# pylint: disable=line-too-long
@classmethod
@ -128,10 +125,3 @@ class YoutubeChannelDownloader(YoutubeDownloader[YoutubeChannelDownloaderOptions
)
# pylint: enable=line-too-long
def download(self) -> Generator[YoutubeVideo, None, None]:
"""
Downloads all videos from a channel
"""
for entry in super().download():
yield entry.to_type(YoutubeVideo)

17
src/ytdl_sub/downloaders/youtube/merge_playlist.py
View file

@ -3,9 +3,9 @@ from typing import List
from typing import Optional
from typing import Tuple
from ytdl_sub.downloaders.youtube.abc import YoutubeDownloader
from ytdl_sub.downloaders.downloader import Downloader
from ytdl_sub.downloaders.youtube.playlist import YoutubePlaylistDownloaderOptions
from ytdl_sub.entries.youtube import YoutubeVideo
from ytdl_sub.entries.entry import Entry
from ytdl_sub.utils.chapters import Chapters
from ytdl_sub.utils.chapters import Timestamp
from ytdl_sub.utils.ffmpeg import set_ffmpeg_metadata_chapters
@ -58,11 +58,8 @@ class YoutubeMergePlaylistDownloaderOptions(YoutubePlaylistDownloaderOptions):
return self._add_chapters
class YoutubeMergePlaylistDownloader(
YoutubeDownloader[YoutubeMergePlaylistDownloaderOptions, YoutubeVideo]
):
class YoutubeMergePlaylistDownloader(Downloader[YoutubeMergePlaylistDownloaderOptions]):
downloader_options_type = YoutubeMergePlaylistDownloaderOptions
downloader_entry_type = YoutubeVideo
supports_download_archive = False
supports_subtitles = False
supports_chapters = False
@ -102,7 +99,7 @@ class YoutubeMergePlaylistDownloader(
},
)
def _get_chapters(self, merged_video: YoutubeVideo, add_chapters: bool) -> FileMetadata:
def _get_chapters(self, merged_video: Entry, add_chapters: bool) -> FileMetadata:
titles: List[str] = []
timestamps: List[Timestamp] = []
@ -124,7 +121,7 @@ class YoutubeMergePlaylistDownloader(
return chapters.to_file_metadata(title="Timestamps of playlist videos in the merged file")
def _to_merged_video(self, entry_dict: Dict) -> YoutubeVideo:
def _to_merged_video(self, entry_dict: Dict) -> Entry:
"""
Adds a few entries not included in a playlist entry to make it look like a merged video
entry_dict
@ -143,9 +140,9 @@ class YoutubeMergePlaylistDownloader(
)
entry_dict["webpage_url"] = self.download_options.playlist_url
return YoutubeVideo(entry_dict=entry_dict, working_directory=self.working_directory)
return Entry(entry_dict=entry_dict, working_directory=self.working_directory)
def download(self) -> List[Tuple[YoutubeVideo, FileMetadata]]:
def download(self) -> List[Tuple[Entry, FileMetadata]]:
"""Download a single Youtube video, then split it into multiple videos"""
entry_dict = self.extract_info(url=self.collection.collection_urls.list[0].url)
merged_video = self._to_merged_video(entry_dict=entry_dict)

25
src/ytdl_sub/downloaders/youtube/playlist.py
View file

@ -1,15 +1,12 @@
from typing import Dict
from typing import Generator
from ytdl_sub.downloaders.downloader import Downloader
from ytdl_sub.downloaders.downloader import DownloaderValidator
from ytdl_sub.downloaders.generic.collection_validator import CollectionValidator
from ytdl_sub.downloaders.youtube.abc import YoutubeDownloader
from ytdl_sub.downloaders.youtube.abc import YoutubeDownloaderOptions
from ytdl_sub.entries.entry_parent import EntryParent
from ytdl_sub.entries.youtube import YoutubeVideo
from ytdl_sub.validators.url_validator import YoutubePlaylistUrlValidator
class YoutubePlaylistDownloaderOptions(YoutubeDownloaderOptions):
class YoutubePlaylistDownloaderOptions(DownloaderValidator):
"""
Downloads all videos from a youtube playlist.
@ -54,9 +51,8 @@ class YoutubePlaylistDownloaderOptions(YoutubeDownloaderOptions):
return self._playlist_url
class YoutubePlaylistDownloader(YoutubeDownloader[YoutubePlaylistDownloaderOptions, YoutubeVideo]):
class YoutubePlaylistDownloader(Downloader[YoutubePlaylistDownloaderOptions]):
downloader_options_type = YoutubePlaylistDownloaderOptions
downloader_entry_type = YoutubeVideo
# pylint: disable=line-too-long
@classmethod
@ -76,16 +72,3 @@ class YoutubePlaylistDownloader(YoutubeDownloader[YoutubePlaylistDownloaderOptio
)
# pylint: enable=line-too-long
@property
def playlist(self) -> EntryParent:
"""Get the playlist parent entry"""
assert len(self.parents) == 1, "Playlist should be the only entry parent"
return self.parents[0]
def download(self) -> Generator[YoutubeVideo, None, None]:
"""
Downloads all videos in a Youtube playlist.
"""
for entry in super().download():
yield entry.to_type(YoutubeVideo)

16
src/ytdl_sub/downloaders/youtube/video.py
View file

@ -1,14 +1,12 @@
from typing import Dict
from typing import List
from ytdl_sub.downloaders.downloader import Downloader
from ytdl_sub.downloaders.downloader import DownloaderValidator
from ytdl_sub.downloaders.generic.collection_validator import CollectionValidator
from ytdl_sub.downloaders.youtube.abc import YoutubeDownloader
from ytdl_sub.downloaders.youtube.abc import YoutubeDownloaderOptions
from ytdl_sub.entries.youtube import YoutubeVideo
from ytdl_sub.validators.url_validator import YoutubeVideoUrlValidator
class YoutubeVideoDownloaderOptions(YoutubeDownloaderOptions):
class YoutubeVideoDownloaderOptions(DownloaderValidator):
"""
Downloads a single youtube video. This download strategy is intended for CLI usage performing
a one-time download of a video, not a subscription.
@ -53,9 +51,8 @@ class YoutubeVideoDownloaderOptions(YoutubeDownloaderOptions):
return self._video_url
class YoutubeVideoDownloader(YoutubeDownloader[YoutubeVideoDownloaderOptions, YoutubeVideo]):
class YoutubeVideoDownloader(Downloader[YoutubeVideoDownloaderOptions]):
downloader_options_type = YoutubeVideoDownloaderOptions
downloader_entry_type = YoutubeVideo
@classmethod
def ytdl_option_defaults(cls) -> Dict:
@ -71,8 +68,3 @@ class YoutubeVideoDownloader(YoutubeDownloader[YoutubeVideoDownloaderOptions, Yo
super().ytdl_option_defaults(),
**{"break_on_existing": True},
)
def download(self) -> List[YoutubeVideo]:
"""Downloads the single video"""
for entry in super().download():
yield entry.to_type(YoutubeVideo)

3
src/ytdl_sub/entries/variables/entry_variables.py
View file

@ -4,7 +4,8 @@ from yt_dlp.utils import sanitize_filename
from ytdl_sub.entries.base_entry import BaseEntry
from ytdl_sub.entries.base_entry import BaseEntryVariables
from ytdl_sub.entries.variables.kwargs import EXT, CHANNEL
from ytdl_sub.entries.variables.kwargs import CHANNEL
from ytdl_sub.entries.variables.kwargs import EXT
from ytdl_sub.entries.variables.kwargs import PLAYLIST_COUNT
from ytdl_sub.entries.variables.kwargs import PLAYLIST_DESCRIPTION
from ytdl_sub.entries.variables.kwargs import PLAYLIST_INDEX

2
tests/unit/entries/conftest.py
View file

@ -65,6 +65,8 @@ def mock_entry_to_dict(
"title_sanitized": "entry ｛title｝",
"ext": ext,
"description": "",
"channel": "abc123",
"channel_sanitized": "abc123",
"extractor": extractor,
"uploader": "abc123",
"uploader_id": "abc123",

43
tests/unit/entries/test_soundcloud_entries.py
View file

@ -1,43 +0,0 @@
import pytest
from ytdl_sub.entries.soundcloud import SoundcloudTrack
@pytest.fixture
def url():
return "soundcloud.com/artist/track-asdfasdf"
@pytest.fixture
def mock_soundcloud_track_to_dict(mock_entry_to_dict):
return dict(
mock_entry_to_dict,
**{
"track_number": 1,
"track_number_padded": "01",
"album": mock_entry_to_dict["title"],
"album_sanitized": mock_entry_to_dict["title_sanitized"],
"album_year": mock_entry_to_dict["upload_year"],
"track_count": 1,
}
)
@pytest.fixture
def mock_soundcloud_track_kwargs(mock_entry_kwargs, url):
return dict(mock_entry_kwargs, **{"url": url})
@pytest.fixture
def mock_soundcloud_track(mock_soundcloud_track_kwargs):
return SoundcloudTrack(entry_dict=mock_soundcloud_track_kwargs, working_directory=".")
class TestSoundcloudTrack(object):
def test_to_dict(self, mock_soundcloud_track, mock_soundcloud_track_to_dict):
assert mock_soundcloud_track.to_dict() == mock_soundcloud_track_to_dict
def test_soundcloud_dict_contains_valid_formatters(
self, mock_soundcloud_track, validate_entry_dict_contains_valid_formatters
):
assert validate_entry_dict_contains_valid_formatters(mock_soundcloud_track)

59
tests/unit/entries/test_youtube_entries.py
View file

@ -1,59 +0,0 @@
import pytest
from ytdl_sub.entries.soundcloud import SoundcloudTrack
from ytdl_sub.entries.youtube import YoutubeVideo
@pytest.fixture
def playlist_index():
return 1
@pytest.fixture
def playlist_count():
return 1
@pytest.fixture
def channel():
return "the channel"
@pytest.fixture
def mock_youtube_video_to_dict(mock_entry_to_dict, playlist_index, playlist_count, channel):
return dict(
mock_entry_to_dict,
**{
"playlist_index": playlist_index,
"playlist_count": playlist_count,
"channel": channel,
"channel_sanitized": channel,
}
)
@pytest.fixture
def mock_youtube_video_kwargs(mock_entry_kwargs, playlist_index, playlist_count, channel):
return dict(
mock_entry_kwargs,
**{
"playlist_index": playlist_index,
"playlist_size": playlist_count,
"channel": channel,
}
)
@pytest.fixture
def mock_youtube_video(mock_youtube_video_kwargs):
return YoutubeVideo(entry_dict=mock_youtube_video_kwargs, working_directory=".")
class TestYoutubeVideo(object):
def test_to_dict(self, mock_youtube_video, mock_youtube_video_to_dict):
assert mock_youtube_video.to_dict() == mock_youtube_video_to_dict
def test_youtube_dict_contains_valid_formatters(
self, mock_youtube_video, validate_entry_dict_contains_valid_formatters
):
assert validate_entry_dict_contains_valid_formatters(mock_youtube_video)