danvics/ytdl-sub
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

[BACKEND] Only have single entry type (#248)

Browse source
This commit is contained in:
Jesse Bannon 2022-09-27 20:17:10 -07:00 committed by GitHub
parent 8edca7af36
commit ebdd6c8aec
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
22 changed files with 63 additions and 354 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

21
docs/config.rst
View file

@ -50,8 +50,6 @@ _______
:member-order: bysource
:exclude-members: get_date_range
.. autofunction:: ytdl_sub.downloaders.youtube.channel.YoutubeChannelDownloader.added_override_variables()
.. autofunction:: ytdl_sub.downloaders.youtube.channel.YoutubeChannelDownloader.ytdl_option_defaults()
-------------------------------------------------------------------------------
@ -64,8 +62,6 @@ ________
:members: playlist_url
:member-order: bysource
.. autofunction:: ytdl_sub.downloaders.youtube.playlist.YoutubePlaylistDownloader.added_override_variables()
.. autofunction:: ytdl_sub.downloaders.youtube.playlist.YoutubePlaylistDownloader.ytdl_option_defaults()
-------------------------------------------------------------------------------
@ -300,26 +296,9 @@ Source Variables
----------------
.. autoclass:: ytdl_sub.entries.variables.entry_variables.EntryVariables
.. _youtube-variables:
Youtube Variables
^^^^^^^^^^^^^^^^^
.. automodule:: ytdl_sub.entries.variables.youtube_variables
:members:
:inherited-members:
:undoc-members:
:exclude-members: source_variables
.. _soundcloud-variables:
Soundcloud Variables
^^^^^^^^^^^^^^^^^^^^
.. automodule:: ytdl_sub.entries.variables.soundcloud_variables
:members:
:inherited-members:
:undoc-members:
:exclude-members: source_variables
-------------------------------------------------------------------------------

3
src/ytdl_sub/config/preset.py
View file

@ -19,6 +19,7 @@ from ytdl_sub.config.preset_options import Overrides
from ytdl_sub.config.preset_options import YTDLOptions
from ytdl_sub.downloaders.downloader import Downloader
from ytdl_sub.downloaders.downloader import DownloaderValidator
from ytdl_sub.entries.entry import Entry
from ytdl_sub.plugins.plugin import Plugin
from ytdl_sub.plugins.plugin import PluginOptions
from ytdl_sub.utils.yaml import load_yaml
@ -129,7 +130,7 @@ class Preset(StrictDictValidator):
@property
def _source_variables(self) -> List[str]:
return self.downloader.downloader_entry_type.source_variables()
return Entry.source_variables()
def __validate_and_get_downloader(self, downloader_source: str) -> Type[Downloader]:
return self._validate_key(key=downloader_source, validator=DownloadStrategyValidator).get(

6
src/ytdl_sub/downloaders/downloader.py
View file

@ -86,17 +86,15 @@ class DownloaderValidator(StrictDictValidator, AddsVariablesMixin, ABC):
DownloaderOptionsT = TypeVar("DownloaderOptionsT", bound=DownloaderValidator)
DownloaderEntryT = TypeVar("DownloaderEntryT", bound=Entry)
class Downloader(DownloadArchiver, Generic[DownloaderOptionsT, DownloaderEntryT], ABC):
class Downloader(DownloadArchiver, Generic[DownloaderOptionsT], ABC):
"""
Class that interacts with ytdl to perform the download of metadata and content,
and should translate that to list of Entry objects.
"""
downloader_options_type: Type[DownloaderValidator] = DownloaderValidator
downloader_entry_type: Type[Entry] = Entry
supports_download_archive: bool = True
supports_subtitles: bool = True
@ -473,7 +471,7 @@ class Downloader(DownloadArchiver, Generic[DownloaderOptionsT, DownloaderEntryT]
def download(
self,
) -> Iterable[DownloaderEntryT] | Iterable[Tuple[DownloaderEntryT, FileMetadata]]:
) -> Iterable[Entry] | Iterable[Tuple[Entry, FileMetadata]]:
"""The function to perform the download of all media entries"""
# download the bottom-most urls first since they are top-priority
for collection_url in reversed(self.collection.collection_urls.list):

4
src/ytdl_sub/downloaders/generic/collection.py
View file

@ -1,7 +1,6 @@
from ytdl_sub.downloaders.downloader import Downloader
from ytdl_sub.downloaders.downloader import DownloaderValidator
from ytdl_sub.downloaders.generic.collection_validator import CollectionValidator
from ytdl_sub.entries.entry import Entry
class CollectionDownloadOptions(CollectionValidator, DownloaderValidator):
@ -41,6 +40,5 @@ class CollectionDownloadOptions(CollectionValidator, DownloaderValidator):
return self
class CollectionDownloader(Downloader[CollectionDownloadOptions, Entry]):
class CollectionDownloader(Downloader[CollectionDownloadOptions]):
downloader_options_type = CollectionDownloadOptions
downloader_entry_type = Entry

4
src/ytdl_sub/downloaders/soundcloud/albums_and_singles.py
View file

@ -89,9 +89,7 @@ class SoundcloudAlbumsAndSinglesDownloadOptions(DownloaderValidator):
return self._url
class SoundcloudAlbumsAndSinglesDownloader(
Downloader[SoundcloudAlbumsAndSinglesDownloadOptions, Entry]
):
class SoundcloudAlbumsAndSinglesDownloader(Downloader[SoundcloudAlbumsAndSinglesDownloadOptions]):
downloader_options_type = SoundcloudAlbumsAndSinglesDownloadOptions
downloader_entry_type = Entry

29
src/ytdl_sub/downloaders/youtube/abc.py
View file

@ -1,29 +0,0 @@
from abc import ABC
from typing import Generic
from typing import TypeVar
from ytdl_sub.downloaders.downloader import Downloader
from ytdl_sub.downloaders.downloader import DownloaderValidator
from ytdl_sub.entries.youtube import YoutubeVideo
class YoutubeDownloaderOptions(DownloaderValidator, ABC):
"""
Abstract source validator for all soundcloud sources.
"""
YoutubeDownloaderOptionsT = TypeVar("YoutubeDownloaderOptionsT", bound=YoutubeDownloaderOptions)
YoutubeVideoT = TypeVar("YoutubeVideoT", bound=YoutubeVideo)
class YoutubeDownloader(
Downloader[YoutubeDownloaderOptionsT, YoutubeVideoT],
Generic[YoutubeDownloaderOptionsT, YoutubeVideoT],
ABC,
):
"""
Class that handles downloading youtube entries via ytdl and converting them into
YoutubeVideo like objects. Reserved for any future logic that is shared amongst all YT
downloaders.
"""

18
src/ytdl_sub/downloaders/youtube/channel.py
View file

@ -1,17 +1,15 @@
from typing import Dict
from typing import Generator
from typing import List
from typing import Optional
from ytdl_sub.downloaders.downloader import Downloader
from ytdl_sub.downloaders.downloader import DownloaderValidator
from ytdl_sub.downloaders.generic.collection_validator import CollectionValidator
from ytdl_sub.downloaders.youtube.abc import YoutubeDownloader
from ytdl_sub.downloaders.youtube.abc import YoutubeDownloaderOptions
from ytdl_sub.entries.youtube import YoutubeVideo
from ytdl_sub.validators.string_formatter_validators import OverridesStringFormatterValidator
from ytdl_sub.validators.url_validator import YoutubeChannelUrlValidator
class YoutubeChannelDownloaderOptions(YoutubeDownloaderOptions):
class YoutubeChannelDownloaderOptions(DownloaderValidator):
"""
Downloads all videos from a youtube channel.
@ -104,9 +102,8 @@ class YoutubeChannelDownloaderOptions(YoutubeDownloaderOptions):
return self._channel_banner_path
class YoutubeChannelDownloader(YoutubeDownloader[YoutubeChannelDownloaderOptions, YoutubeVideo]):
class YoutubeChannelDownloader(Downloader[YoutubeChannelDownloaderOptions]):
downloader_options_type = YoutubeChannelDownloaderOptions
downloader_entry_type = YoutubeVideo
# pylint: disable=line-too-long
@classmethod
@ -128,10 +125,3 @@ class YoutubeChannelDownloader(YoutubeDownloader[YoutubeChannelDownloaderOptions
)
# pylint: enable=line-too-long
def download(self) -> Generator[YoutubeVideo, None, None]:
"""
Downloads all videos from a channel
"""
for entry in super().download():
yield entry.to_type(YoutubeVideo)

17
src/ytdl_sub/downloaders/youtube/merge_playlist.py
View file

@ -3,9 +3,9 @@ from typing import List
from typing import Optional
from typing import Tuple
from ytdl_sub.downloaders.youtube.abc import YoutubeDownloader
from ytdl_sub.downloaders.downloader import Downloader
from ytdl_sub.downloaders.youtube.playlist import YoutubePlaylistDownloaderOptions
from ytdl_sub.entries.youtube import YoutubeVideo
from ytdl_sub.entries.entry import Entry
from ytdl_sub.utils.chapters import Chapters
from ytdl_sub.utils.chapters import Timestamp
from ytdl_sub.utils.ffmpeg import set_ffmpeg_metadata_chapters
@ -58,11 +58,8 @@ class YoutubeMergePlaylistDownloaderOptions(YoutubePlaylistDownloaderOptions):
return self._add_chapters
class YoutubeMergePlaylistDownloader(
YoutubeDownloader[YoutubeMergePlaylistDownloaderOptions, YoutubeVideo]
):
class YoutubeMergePlaylistDownloader(Downloader[YoutubeMergePlaylistDownloaderOptions]):
downloader_options_type = YoutubeMergePlaylistDownloaderOptions
downloader_entry_type = YoutubeVideo
supports_download_archive = False
supports_subtitles = False
supports_chapters = False
@ -102,7 +99,7 @@ class YoutubeMergePlaylistDownloader(
},
)
def _get_chapters(self, merged_video: YoutubeVideo, add_chapters: bool) -> FileMetadata:
def _get_chapters(self, merged_video: Entry, add_chapters: bool) -> FileMetadata:
titles: List[str] = []
timestamps: List[Timestamp] = []
@ -124,7 +121,7 @@ class YoutubeMergePlaylistDownloader(
return chapters.to_file_metadata(title="Timestamps of playlist videos in the merged file")
def _to_merged_video(self, entry_dict: Dict) -> YoutubeVideo:
def _to_merged_video(self, entry_dict: Dict) -> Entry:
"""
Adds a few entries not included in a playlist entry to make it look like a merged video
entry_dict
@ -143,9 +140,9 @@ class YoutubeMergePlaylistDownloader(
)
entry_dict["webpage_url"] = self.download_options.playlist_url
return YoutubeVideo(entry_dict=entry_dict, working_directory=self.working_directory)
return Entry(entry_dict=entry_dict, working_directory=self.working_directory)
def download(self) -> List[Tuple[YoutubeVideo, FileMetadata]]:
def download(self) -> List[Tuple[Entry, FileMetadata]]:
"""Download a single Youtube video, then split it into multiple videos"""
entry_dict = self.extract_info(url=self.collection.collection_urls.list[0].url)
merged_video = self._to_merged_video(entry_dict=entry_dict)

25
src/ytdl_sub/downloaders/youtube/playlist.py
View file

@ -1,15 +1,12 @@
from typing import Dict
from typing import Generator
from ytdl_sub.downloaders.downloader import Downloader
from ytdl_sub.downloaders.downloader import DownloaderValidator
from ytdl_sub.downloaders.generic.collection_validator import CollectionValidator
from ytdl_sub.downloaders.youtube.abc import YoutubeDownloader
from ytdl_sub.downloaders.youtube.abc import YoutubeDownloaderOptions
from ytdl_sub.entries.entry_parent import EntryParent
from ytdl_sub.entries.youtube import YoutubeVideo
from ytdl_sub.validators.url_validator import YoutubePlaylistUrlValidator
class YoutubePlaylistDownloaderOptions(YoutubeDownloaderOptions):
class YoutubePlaylistDownloaderOptions(DownloaderValidator):
"""
Downloads all videos from a youtube playlist.
@ -54,9 +51,8 @@ class YoutubePlaylistDownloaderOptions(YoutubeDownloaderOptions):
return self._playlist_url
class YoutubePlaylistDownloader(YoutubeDownloader[YoutubePlaylistDownloaderOptions, YoutubeVideo]):
class YoutubePlaylistDownloader(Downloader[YoutubePlaylistDownloaderOptions]):
downloader_options_type = YoutubePlaylistDownloaderOptions
downloader_entry_type = YoutubeVideo
# pylint: disable=line-too-long
@classmethod
@ -76,16 +72,3 @@ class YoutubePlaylistDownloader(YoutubeDownloader[YoutubePlaylistDownloaderOptio
)
# pylint: enable=line-too-long
@property
def playlist(self) -> EntryParent:
"""Get the playlist parent entry"""
assert len(self.parents) == 1, "Playlist should be the only entry parent"
return self.parents[0]
def download(self) -> Generator[YoutubeVideo, None, None]:
"""
Downloads all videos in a Youtube playlist.
"""
for entry in super().download():
yield entry.to_type(YoutubeVideo)

16
src/ytdl_sub/downloaders/youtube/video.py
View file

@ -1,14 +1,12 @@
from typing import Dict
from typing import List
from ytdl_sub.downloaders.downloader import Downloader
from ytdl_sub.downloaders.downloader import DownloaderValidator
from ytdl_sub.downloaders.generic.collection_validator import CollectionValidator
from ytdl_sub.downloaders.youtube.abc import YoutubeDownloader
from ytdl_sub.downloaders.youtube.abc import YoutubeDownloaderOptions
from ytdl_sub.entries.youtube import YoutubeVideo
from ytdl_sub.validators.url_validator import YoutubeVideoUrlValidator
class YoutubeVideoDownloaderOptions(YoutubeDownloaderOptions):
class YoutubeVideoDownloaderOptions(DownloaderValidator):
"""
Downloads a single youtube video. This download strategy is intended for CLI usage performing
a one-time download of a video, not a subscription.
@ -53,9 +51,8 @@ class YoutubeVideoDownloaderOptions(YoutubeDownloaderOptions):
return self._video_url
class YoutubeVideoDownloader(YoutubeDownloader[YoutubeVideoDownloaderOptions, YoutubeVideo]):
class YoutubeVideoDownloader(Downloader[YoutubeVideoDownloaderOptions]):
downloader_options_type = YoutubeVideoDownloaderOptions
downloader_entry_type = YoutubeVideo
@classmethod
def ytdl_option_defaults(cls) -> Dict:
@ -71,8 +68,3 @@ class YoutubeVideoDownloader(YoutubeDownloader[YoutubeVideoDownloaderOptions, Yo
super().ytdl_option_defaults(),
**{"break_on_existing": True},
)
def download(self) -> List[YoutubeVideo]:
"""Downloads the single video"""
for entry in super().download():
yield entry.to_type(YoutubeVideo)

12
src/ytdl_sub/entries/entry.py
View file

@ -15,6 +15,18 @@ class Entry(EntryVariables, BaseEntry):
Entry object to represent a single media object returned from yt-dlp.
"""
@property
def ext(self) -> str:
"""
With ffmpeg installed, yt-dlp will sometimes merge the file into an mkv file.
This is not reflected in the entry. See if the mkv file exists and return "mkv" if so,
otherwise, return the original extension.
"""
mkv_file_path = str(Path(self.working_directory()) / f"{self.uid}.mkv")
if os.path.isfile(mkv_file_path):
return "mkv"
return super().ext
def get_download_file_name(self) -> str:
"""
Returns

11
src/ytdl_sub/entries/soundcloud.py
View file

@ -1,11 +0,0 @@
from ytdl_sub.entries.entry import Entry
from ytdl_sub.entries.variables.soundcloud_variables import SoundcloudVariables
# TODO: Delete since not used
class SoundcloudTrack(SoundcloudVariables, Entry):
"""
Entry object to represent a Soundcloud track yt-dlp that is a single, which implies
it does not belong to an album.
"""

21
src/ytdl_sub/entries/variables/entry_variables.py
View file

@ -4,6 +4,7 @@ from yt_dlp.utils import sanitize_filename
from ytdl_sub.entries.base_entry import BaseEntry
from ytdl_sub.entries.base_entry import BaseEntryVariables
from ytdl_sub.entries.variables.kwargs import CHANNEL
from ytdl_sub.entries.variables.kwargs import EXT
from ytdl_sub.entries.variables.kwargs import PLAYLIST_COUNT
from ytdl_sub.entries.variables.kwargs import PLAYLIST_DESCRIPTION
@ -304,6 +305,26 @@ class EntryVariables(BaseEntryVariables):
"""
return self.kwargs_get(SOURCE_UPLOADER_URL, self.source_webpage_url)
@property
def channel(self: Self) -> str:
"""
Returns
-------
str
The channel name if it exists, otherwise returns the uploader.
"""
return self.kwargs_get(CHANNEL, self.uploader)
@property
def channel_sanitized(self: Self) -> str:
"""
Returns
-------
str
The channel name, sanitized.
"""
return sanitize_filename(self.channel)
@property
def ext(self: Self) -> str:
"""

1
src/ytdl_sub/entries/variables/kwargs.py
View file

@ -41,6 +41,7 @@ PLAYLIST_UPLOADER_ID = _("playlist_uploader_id")
PLAYLIST_UPLOADER_URL = _("playlist_uploader_url")
UID = _("id")
CHANNEL = _("channel")
EXT = _("ext")
TITLE = _("title")
DESCRIPTION = _("description")

69
src/ytdl_sub/entries/variables/soundcloud_variables.py
View file

@ -1,69 +0,0 @@
from yt_dlp.utils import sanitize_filename
from ytdl_sub.entries.variables.entry_variables import EntryVariables
# This file contains mixins to a BaseEntry subclass. Ignore pylint's "no kwargs member" suggestion
# pylint: disable=no-member
class SoundcloudVariables(EntryVariables):
@property
def track_number(self) -> int:
"""
Returns
-------
int
The entry's track number within an album. For singles, it will always be 1.
"""
return 1
@property
def track_number_padded(self) -> str:
"""
Returns
-------
str
The entry's track number, padded two digits.
"""
return f"{self.track_number:02d}"
@property
def track_count(self) -> int:
"""
Returns
-------
int
The total tracks in album. For singles, it will always be 1.
"""
return 1
@property
def album(self) -> str:
"""
Returns
-------
str
The entry's album name. For singles, it will be the same as the title.
"""
return self.title
@property
def album_sanitized(self) -> str:
"""
Returns
-------
str
The entry's sanitized album name, which is safe to use for Unix and Windows file names.
"""
return sanitize_filename(self.album)
@property
def album_year(self) -> int:
"""
Returns
-------
int
The entry's album year, which is determined by the latest year amongst all tracks in the
album.
"""
return self.upload_year

29
src/ytdl_sub/entries/variables/youtube_variables.py
View file

@ -1,29 +0,0 @@
from yt_dlp.utils import sanitize_filename
from ytdl_sub.entries.base_entry import BaseEntry
from ytdl_sub.entries.variables.entry_variables import EntryVariables
# This file contains mixins to a BaseEntry subclass. Ignore pylint's "no kwargs member" suggestion
# pylint: disable=no-member
class YoutubeVideoVariables(EntryVariables):
@property
def channel(self: BaseEntry) -> str:
"""
Returns
-------
str
The channel name.
"""
return self.kwargs("channel")
@property
def channel_sanitized(self) -> str:
"""
Returns
-------
str
The channel name, sanitized.
"""
return sanitize_filename(self.channel)

23
src/ytdl_sub/entries/youtube.py
View file

@ -1,23 +0,0 @@
import os.path
from pathlib import Path
from ytdl_sub.entries.entry import Entry
from ytdl_sub.entries.variables.youtube_variables import YoutubeVideoVariables
class YoutubeVideo(YoutubeVideoVariables, Entry):
"""
Entry object to represent a Youtube video. Reserved for shared Youtube entry logic.
"""
@property
def ext(self) -> str:
"""
With ffmpeg installed, yt-dlp will sometimes merge the file into an mkv file.
This is not reflected in the entry. See if the mkv file exists and return "mkv" if so,
otherwise, return the original extension.
"""
mkv_file_path = str(Path(self.working_directory()) / f"{self.uid}.mkv")
if os.path.isfile(mkv_file_path):
return "mkv"
return super().ext

2
tests/e2e/expected_download.py
View file

@ -115,7 +115,7 @@ def assert_expected_downloads(
dry_run: bool,
expected_download_summary_file_name: str,
ignore_md5_hashes_for: Optional[List[str]] = None,
regenerate_expected_download_summary: bool = True,
regenerate_expected_download_summary: bool = False,
):
if dry_run:
output_directory_contents = list(Path(output_directory).rglob("*"))

2
tests/e2e/expected_transaction_log.py
View file

@ -10,7 +10,7 @@ def assert_transaction_log_matches(
output_directory: str,
transaction_log: FileHandlerTransactionLog,
transaction_log_summary_file_name: str,
regenerate_transaction_log: bool = True,
regenerate_transaction_log: bool = False,
):
"""
Parameters

2
tests/unit/entries/conftest.py
View file

@ -65,6 +65,8 @@ def mock_entry_to_dict(
"title_sanitized": "entry ｛title｝",
"ext": ext,
"description": "",
"channel": "abc123",
"channel_sanitized": "abc123",
"extractor": extractor,
"uploader": "abc123",
"uploader_id": "abc123",

43
tests/unit/entries/test_soundcloud_entries.py
View file

@ -1,43 +0,0 @@
import pytest
from ytdl_sub.entries.soundcloud import SoundcloudTrack
@pytest.fixture
def url():
return "soundcloud.com/artist/track-asdfasdf"
@pytest.fixture
def mock_soundcloud_track_to_dict(mock_entry_to_dict):
return dict(
mock_entry_to_dict,
**{
"track_number": 1,
"track_number_padded": "01",
"album": mock_entry_to_dict["title"],
"album_sanitized": mock_entry_to_dict["title_sanitized"],
"album_year": mock_entry_to_dict["upload_year"],
"track_count": 1,
}
)
@pytest.fixture
def mock_soundcloud_track_kwargs(mock_entry_kwargs, url):
return dict(mock_entry_kwargs, **{"url": url})
@pytest.fixture
def mock_soundcloud_track(mock_soundcloud_track_kwargs):
return SoundcloudTrack(entry_dict=mock_soundcloud_track_kwargs, working_directory=".")
class TestSoundcloudTrack(object):
def test_to_dict(self, mock_soundcloud_track, mock_soundcloud_track_to_dict):
assert mock_soundcloud_track.to_dict() == mock_soundcloud_track_to_dict
def test_soundcloud_dict_contains_valid_formatters(
self, mock_soundcloud_track, validate_entry_dict_contains_valid_formatters
):
assert validate_entry_dict_contains_valid_formatters(mock_soundcloud_track)

59
tests/unit/entries/test_youtube_entries.py
View file

@ -1,59 +0,0 @@
import pytest
from ytdl_sub.entries.soundcloud import SoundcloudTrack
from ytdl_sub.entries.youtube import YoutubeVideo
@pytest.fixture
def playlist_index():
return 1
@pytest.fixture
def playlist_count():
return 1
@pytest.fixture
def channel():
return "the channel"
@pytest.fixture
def mock_youtube_video_to_dict(mock_entry_to_dict, playlist_index, playlist_count, channel):
return dict(
mock_entry_to_dict,
**{
"playlist_index": playlist_index,
"playlist_count": playlist_count,
"channel": channel,
"channel_sanitized": channel,
}
)
@pytest.fixture
def mock_youtube_video_kwargs(mock_entry_kwargs, playlist_index, playlist_count, channel):
return dict(
mock_entry_kwargs,
**{
"playlist_index": playlist_index,
"playlist_size": playlist_count,
"channel": channel,
}
)
@pytest.fixture
def mock_youtube_video(mock_youtube_video_kwargs):
return YoutubeVideo(entry_dict=mock_youtube_video_kwargs, working_directory=".")
class TestYoutubeVideo(object):
def test_to_dict(self, mock_youtube_video, mock_youtube_video_to_dict):
assert mock_youtube_video.to_dict() == mock_youtube_video_to_dict
def test_youtube_dict_contains_valid_formatters(
self, mock_youtube_video, validate_entry_dict_contains_valid_formatters
):
assert validate_entry_dict_contains_valid_formatters(mock_youtube_video)