danvics/ytdl-sub
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

[REFACTOR] Move downloaders to separate files (#91)

Browse source
This commit is contained in:
Jesse Bannon 2022-07-02 21:48:41 -07:00 committed by GitHub
parent 9df40497e2
commit 607ad9f9de
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
11 changed files with 267 additions and 270 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

16
docs/config.rst
View file

@ -45,13 +45,13 @@ for available source variables to use.
channel
_______
.. autoclass:: ytdl_sub.downloaders.youtube_downloader.YoutubeChannelDownloaderOptions()
.. autoclass:: ytdl_sub.downloaders.youtube.channel.YoutubeChannelDownloaderOptions()
:members:
:member-order: bysource
:inherited-members:
:exclude-members: get_date_range
.. autofunction:: ytdl_sub.downloaders.youtube_downloader.YoutubeChannelDownloader.ytdl_option_defaults()
.. autofunction:: ytdl_sub.downloaders.youtube.channel.YoutubeChannelDownloader.ytdl_option_defaults()
-------------------------------------------------------------------------------
@ -59,23 +59,23 @@ _______
playlist
________
.. autoclass:: ytdl_sub.downloaders.youtube_downloader.YoutubePlaylistDownloaderOptions()
.. autoclass:: ytdl_sub.downloaders.youtube.playlist.YoutubePlaylistDownloaderOptions()
:members:
:member-order: bysource
:inherited-members:
.. autofunction:: ytdl_sub.downloaders.youtube_downloader.YoutubePlaylistDownloader.ytdl_option_defaults()
.. autofunction:: ytdl_sub.downloaders.youtube.playlist.YoutubePlaylistDownloader.ytdl_option_defaults()
-------------------------------------------------------------------------------
video
_____
.. autoclass:: ytdl_sub.downloaders.youtube_downloader.YoutubeVideoDownloaderOptions()
.. autoclass:: ytdl_sub.downloaders.youtube.video.YoutubeVideoDownloaderOptions()
:members:
:member-order: bysource
:inherited-members:
.. autofunction:: ytdl_sub.downloaders.youtube_downloader.YoutubeVideoDownloader.ytdl_option_defaults()
.. autofunction:: ytdl_sub.downloaders.youtube.video.YoutubeVideoDownloader.ytdl_option_defaults()
-------------------------------------------------------------------------------
@ -109,12 +109,12 @@ for available source variables to use.
albums_and_singles
__________________
.. autoclass:: ytdl_sub.downloaders.soundcloud_downloader.SoundcloudAlbumsAndSinglesDownloadOptions()
.. autoclass:: ytdl_sub.downloaders.soundcloud.albums_and_singles.SoundcloudAlbumsAndSinglesDownloadOptions()
:members:
:member-order: bysource
:inherited-members:
.. autofunction:: ytdl_sub.downloaders.soundcloud_downloader.SoundcloudAlbumsAndSinglesDownloader.ytdl_option_defaults()
.. autofunction:: ytdl_sub.downloaders.soundcloud.albums_and_singles.SoundcloudAlbumsAndSinglesDownloader.ytdl_option_defaults()
-------------------------------------------------------------------------------

8
src/ytdl_sub/config/preset_class_mappings.py
View file

@ -3,12 +3,12 @@ from typing import List
from typing import Type
from ytdl_sub.downloaders.downloader import Downloader
from ytdl_sub.downloaders.soundcloud_downloader import SoundcloudAlbumsAndSinglesDownloader
from ytdl_sub.downloaders.soundcloud.albums_and_singles import SoundcloudAlbumsAndSinglesDownloader
from ytdl_sub.downloaders.youtube.channel import YoutubeChannelDownloader
from ytdl_sub.downloaders.youtube.merge_playlist import YoutubeMergePlaylistDownloader
from ytdl_sub.downloaders.youtube.playlist import YoutubePlaylistDownloader
from ytdl_sub.downloaders.youtube.split_video import YoutubeSplitVideoDownloader
from ytdl_sub.downloaders.youtube_downloader import YoutubeChannelDownloader
from ytdl_sub.downloaders.youtube_downloader import YoutubePlaylistDownloader
from ytdl_sub.downloaders.youtube_downloader import YoutubeVideoDownloader
from ytdl_sub.downloaders.youtube.video import YoutubeVideoDownloader
from ytdl_sub.plugins.music_tags import MusicTagsPlugin
from ytdl_sub.plugins.nfo_tags import NfoTagsPlugin
from ytdl_sub.plugins.output_directory_nfo_tags import OutputDirectoryNfoTagsPlugin

0
src/ytdl_sub/downloaders/soundcloud/__init__.py Normal file
View file

65
src/ytdl_sub/downloaders/soundcloud/abc.py Normal file
View file

@ -0,0 +1,65 @@
from abc import ABC
from typing import Generic
from typing import TypeVar
from ytdl_sub.downloaders.downloader import Downloader
from ytdl_sub.downloaders.downloader import DownloaderValidator
from ytdl_sub.entries.soundcloud import SoundcloudTrack
from ytdl_sub.validators.validators import BoolValidator
class SoundcloudDownloaderOptions(DownloaderValidator, ABC):
"""
Abstract source validator for all soundcloud sources.
"""
_optional_keys = {"skip_premiere_tracks"}
def __init__(self, name: str, value: dict):
super().__init__(name=name, value=value)
self._skip_premiere_tracks = self._validate_key(
"skip_premiere_tracks", BoolValidator, default=True
)
@property
def skip_premiere_tracks(self) -> bool:
"""
Optional. True to skip tracks that require purchasing. False otherwise. Defaults to True.
"""
return self._skip_premiere_tracks.value
SoundcloudDownloaderOptionsT = TypeVar(
"SoundcloudDownloaderOptionsT", bound=SoundcloudDownloaderOptions
)
class SoundcloudDownloader(
Downloader[SoundcloudDownloaderOptionsT, SoundcloudTrack],
Generic[SoundcloudDownloaderOptionsT],
ABC,
):
"""
Class that handles downloading soundcloud entries via ytdl and converting them into
SoundcloudTrack / SoundcloudAlbumTrack objects
"""
downloader_entry_type = SoundcloudTrack
@classmethod
def artist_albums_url(cls, artist_url: str) -> str:
"""
Returns
-------
Full artist album url
"""
return f"{artist_url}/albums"
@classmethod
def artist_tracks_url(cls, artist_url: str) -> str:
"""
Returns
-------
Full artist tracks url
"""
return f"{artist_url}/tracks"

72
src/ytdl_sub/downloaders/soundcloud_downloader.py → src/ytdl_sub/downloaders/soundcloud/albums_and_singles.py
View file

@ -1,79 +1,11 @@
from abc import ABC
from typing import Dict
from typing import Generic
from typing import List
from typing import TypeVar
from ytdl_sub.downloaders.downloader import Downloader
from ytdl_sub.downloaders.downloader import DownloaderValidator
from ytdl_sub.downloaders.soundcloud.abc import SoundcloudDownloader
from ytdl_sub.downloaders.soundcloud.abc import SoundcloudDownloaderOptions
from ytdl_sub.entries.soundcloud import SoundcloudAlbum
from ytdl_sub.entries.soundcloud import SoundcloudTrack
from ytdl_sub.validators.url_validator import SoundcloudUsernameUrlValidator
from ytdl_sub.validators.validators import BoolValidator
###############################################################################
# Abstract Soundcloud downloader + options
class SoundcloudDownloaderOptions(DownloaderValidator, ABC):
"""
Abstract source validator for all soundcloud sources.
"""
_optional_keys = {"skip_premiere_tracks"}
def __init__(self, name: str, value: dict):
super().__init__(name=name, value=value)
self._skip_premiere_tracks = self._validate_key(
"skip_premiere_tracks", BoolValidator, default=True
)
@property
def skip_premiere_tracks(self) -> bool:
"""
Optional. True to skip tracks that require purchasing. False otherwise. Defaults to True.
"""
return self._skip_premiere_tracks.value
SoundcloudDownloaderOptionsT = TypeVar(
"SoundcloudDownloaderOptionsT", bound=SoundcloudDownloaderOptions
)
class SoundcloudDownloader(
Downloader[SoundcloudDownloaderOptionsT, SoundcloudTrack],
Generic[SoundcloudDownloaderOptionsT],
ABC,
):
"""
Class that handles downloading soundcloud entries via ytdl and converting them into
SoundcloudTrack / SoundcloudAlbumTrack objects
"""
downloader_entry_type = SoundcloudTrack
@classmethod
def artist_albums_url(cls, artist_url: str) -> str:
"""
Returns
-------
Full artist album url
"""
return f"{artist_url}/albums"
@classmethod
def artist_tracks_url(cls, artist_url: str) -> str:
"""
Returns
-------
Full artist tracks url
"""
return f"{artist_url}/tracks"
###############################################################################
# Soundcloud albums and singles downloader + options
class SoundcloudAlbumsAndSinglesDownloadOptions(SoundcloudDownloaderOptions):

29
src/ytdl_sub/downloaders/youtube/abc.py Normal file
View file

@ -0,0 +1,29 @@
from abc import ABC
from typing import Generic
from typing import TypeVar
from ytdl_sub.downloaders.downloader import Downloader
from ytdl_sub.downloaders.downloader import DownloaderValidator
from ytdl_sub.entries.youtube import YoutubeVideo
class YoutubeDownloaderOptions(DownloaderValidator, ABC):
"""
Abstract source validator for all soundcloud sources.
"""
YoutubeDownloaderOptionsT = TypeVar("YoutubeDownloaderOptionsT", bound=YoutubeDownloaderOptions)
YoutubeVideoT = TypeVar("YoutubeVideoT", bound=YoutubeVideo)
class YoutubeDownloader(
Downloader[YoutubeDownloaderOptionsT, YoutubeVideoT],
Generic[YoutubeDownloaderOptionsT, YoutubeVideoT],
ABC,
):
"""
Class that handles downloading youtube entries via ytdl and converting them into
YoutubeVideo like objects. Reserved for any future logic that is shared amongst all YT
downloaders.
"""

186
src/ytdl_sub/downloaders/youtube_downloader.py → src/ytdl_sub/downloaders/youtube/channel.py
View file

@ -1,205 +1,23 @@
from abc import ABC
from pathlib import Path
from typing import Dict
from typing import Generic
from typing import List
from typing import Optional
from typing import TypeVar
from ytdl_sub.config.preset_options import Overrides
from ytdl_sub.downloaders.downloader import Downloader
from ytdl_sub.downloaders.downloader import DownloaderOptionsT
from ytdl_sub.downloaders.downloader import DownloaderValidator
from ytdl_sub.downloaders.youtube.abc import YoutubeDownloader
from ytdl_sub.downloaders.youtube.abc import YoutubeDownloaderOptions
from ytdl_sub.entries.youtube import YoutubeChannel
from ytdl_sub.entries.youtube import YoutubePlaylistVideo
from ytdl_sub.entries.youtube import YoutubeVideo
from ytdl_sub.utils.logger import Logger
from ytdl_sub.utils.thumbnail import convert_url_thumbnail
from ytdl_sub.validators.date_range_validator import DateRangeValidator
from ytdl_sub.validators.string_formatter_validators import OverridesStringFormatterValidator
from ytdl_sub.validators.url_validator import YoutubeChannelUrlValidator
from ytdl_sub.validators.url_validator import YoutubePlaylistUrlValidator
from ytdl_sub.validators.url_validator import YoutubeVideoUrlValidator
from ytdl_sub.ytdl_additions.enhanced_download_archive import EnhancedDownloadArchive
logger = Logger.get()
###############################################################################
# Abstract Youtube downloader + options
class YoutubeDownloaderOptions(DownloaderValidator, ABC):
"""
Abstract source validator for all soundcloud sources.
"""
YoutubeDownloaderOptionsT = TypeVar("YoutubeDownloaderOptionsT", bound=YoutubeDownloaderOptions)
YoutubeVideoT = TypeVar("YoutubeVideoT", bound=YoutubeVideo)
class YoutubeDownloader(
Downloader[YoutubeDownloaderOptionsT, YoutubeVideoT],
Generic[YoutubeDownloaderOptionsT, YoutubeVideoT],
ABC,
):
"""
Class that handles downloading youtube entries via ytdl and converting them into
YoutubeVideo like objects. Reserved for any future logic that is shared amongst all YT
downloaders.
"""
###############################################################################
# Youtube single video downloader + options
class YoutubeVideoDownloaderOptions(YoutubeDownloaderOptions):
"""
Downloads a single youtube video. This download strategy is intended for CLI usage performing
a one-time download of a video, not a subscription.
Usage:
.. code-block:: yaml
presets:
example_preset:
youtube:
# required
download_strategy: "video"
video_url: "youtube.com/watch?v=VMAPTo7RVDo"
CLI usage:
.. code-block:: bash
ytdl-sub dl --preset "example_preset" --youtube.video_url "youtube.com/watch?v=VMAPTo7RVDo"
"""
_required_keys = {"video_url"}
def __init__(self, name, value):
super().__init__(name, value)
self._video_url = self._validate_key("video_url", YoutubeVideoUrlValidator).video_url
@property
def video_url(self) -> str:
"""
Required. The url of the video, i.e. ``youtube.com/watch?v=VMAPTo7RVDo``.
"""
return self._video_url
class YoutubeVideoDownloader(YoutubeDownloader[YoutubeVideoDownloaderOptions, YoutubeVideo]):
downloader_options_type = YoutubeVideoDownloaderOptions
downloader_entry_type = YoutubeVideo
@classmethod
def ytdl_option_defaults(cls) -> Dict:
"""
Default `ytdl_options`_ for ``video``
.. code-block:: yaml
ytdl_options:
ignoreerrors: True # ignore errors like hidden videos, age restriction, etc
"""
return dict(
super().ytdl_option_defaults(),
**{"break_on_existing": True},
)
def download(self) -> List[YoutubeVideo]:
"""Download a single Youtube video"""
entry_dict = self.extract_info(url=self.download_options.video_url)
return [YoutubeVideo(entry_dict=entry_dict, working_directory=self.working_directory)]
###############################################################################
# Youtube playlist downloader + options
class YoutubePlaylistDownloaderOptions(YoutubeDownloaderOptions):
"""
Downloads all videos from a youtube playlist.
Usage:
.. code-block:: yaml
presets:
my_example_preset:
youtube:
# required
download_strategy: "playlist"
playlist_url: "https://www.youtube.com/playlist?list=UCsvn_Po0SmunchJYtttWpOxMg"
"""
_required_keys = {"playlist_url"}
def __init__(self, name, value):
super().__init__(name, value)
self._playlist_url = self._validate_key(
"playlist_url", YoutubePlaylistUrlValidator
).playlist_url
@property
def playlist_url(self) -> str:
"""
Required. The playlist's url, i.e.
``https://www.youtube.com/playlist?list=UCsvn_Po0SmunchJYtttWpOxMg``.
"""
return self._playlist_url
class YoutubePlaylistDownloader(
YoutubeDownloader[YoutubePlaylistDownloaderOptions, YoutubePlaylistVideo]
):
downloader_options_type = YoutubePlaylistDownloaderOptions
downloader_entry_type = YoutubePlaylistVideo
# pylint: disable=line-too-long
@classmethod
def ytdl_option_defaults(cls) -> Dict:
"""
Default `ytdl_options`_ for ``playlist``
.. code-block:: yaml
ytdl_options:
ignoreerrors: True # ignore errors like hidden videos, age restriction, etc
break_on_existing: True # stop downloads (newest to oldest) if a video is already downloaded
"""
return dict(
super().ytdl_option_defaults(),
**{"break_on_existing": True},
)
# pylint: enable=line-too-long
def download(self) -> List[YoutubePlaylistVideo]:
"""
Downloads all videos in a Youtube playlist
"""
playlist_videos: List[YoutubePlaylistVideo] = []
entry_dicts = self.extract_info_via_info_json(url=self.download_options.playlist_url)
for entry_dict in entry_dicts:
if entry_dict.get("extractor") == "youtube":
playlist_videos.append(
YoutubePlaylistVideo(
entry_dict=entry_dict, working_directory=self.working_directory
)
)
return playlist_videos
###############################################################################
# Youtube channel downloader + options
class YoutubeChannelDownloaderOptions(YoutubeDownloaderOptions, DateRangeValidator):
"""

4
src/ytdl_sub/downloaders/youtube/merge_playlist.py
View file

@ -2,8 +2,8 @@ from typing import Dict
from typing import List
from typing import Optional
from ytdl_sub.downloaders.youtube_downloader import YoutubeDownloader
from ytdl_sub.downloaders.youtube_downloader import YoutubePlaylistDownloaderOptions
from ytdl_sub.downloaders.youtube.abc import YoutubeDownloader
from ytdl_sub.downloaders.youtube.playlist import YoutubePlaylistDownloaderOptions
from ytdl_sub.entries.youtube import YoutubeVideo
from ytdl_sub.utils.chapters import Chapters
from ytdl_sub.utils.chapters import Timestamp

83
src/ytdl_sub/downloaders/youtube/playlist.py Normal file
View file

@ -0,0 +1,83 @@
from typing import Dict
from typing import List
from ytdl_sub.downloaders.youtube.abc import YoutubeDownloader
from ytdl_sub.downloaders.youtube.abc import YoutubeDownloaderOptions
from ytdl_sub.entries.youtube import YoutubePlaylistVideo
from ytdl_sub.validators.url_validator import YoutubePlaylistUrlValidator
class YoutubePlaylistDownloaderOptions(YoutubeDownloaderOptions):
"""
Downloads all videos from a youtube playlist.
Usage:
.. code-block:: yaml
presets:
my_example_preset:
youtube:
# required
download_strategy: "playlist"
playlist_url: "https://www.youtube.com/playlist?list=UCsvn_Po0SmunchJYtttWpOxMg"
"""
_required_keys = {"playlist_url"}
def __init__(self, name, value):
super().__init__(name, value)
self._playlist_url = self._validate_key(
"playlist_url", YoutubePlaylistUrlValidator
).playlist_url
@property
def playlist_url(self) -> str:
"""
Required. The playlist's url, i.e.
``https://www.youtube.com/playlist?list=UCsvn_Po0SmunchJYtttWpOxMg``.
"""
return self._playlist_url
class YoutubePlaylistDownloader(
YoutubeDownloader[YoutubePlaylistDownloaderOptions, YoutubePlaylistVideo]
):
downloader_options_type = YoutubePlaylistDownloaderOptions
downloader_entry_type = YoutubePlaylistVideo
# pylint: disable=line-too-long
@classmethod
def ytdl_option_defaults(cls) -> Dict:
"""
Default `ytdl_options`_ for ``playlist``
.. code-block:: yaml
ytdl_options:
ignoreerrors: True # ignore errors like hidden videos, age restriction, etc
break_on_existing: True # stop downloads (newest to oldest) if a video is already downloaded
"""
return dict(
super().ytdl_option_defaults(),
**{"break_on_existing": True},
)
# pylint: enable=line-too-long
def download(self) -> List[YoutubePlaylistVideo]:
"""
Downloads all videos in a Youtube playlist
"""
playlist_videos: List[YoutubePlaylistVideo] = []
entry_dicts = self.extract_info_via_info_json(url=self.download_options.playlist_url)
for entry_dict in entry_dicts:
if entry_dict.get("extractor") == "youtube":
playlist_videos.append(
YoutubePlaylistVideo(
entry_dict=entry_dict, working_directory=self.working_directory
)
)
return playlist_videos

4
src/ytdl_sub/downloaders/youtube/split_video.py
View file

@ -4,8 +4,8 @@ from pathlib import Path
from typing import Dict
from typing import List
from ytdl_sub.downloaders.youtube_downloader import YoutubeDownloader
from ytdl_sub.downloaders.youtube_downloader import YoutubeVideoDownloaderOptions
from ytdl_sub.downloaders.youtube.abc import YoutubeDownloader
from ytdl_sub.downloaders.youtube.video import YoutubeVideoDownloaderOptions
from ytdl_sub.entries.youtube import YoutubePlaylistVideo
from ytdl_sub.entries.youtube import YoutubeVideo
from ytdl_sub.utils.chapters import Chapters

70
src/ytdl_sub/downloaders/youtube/video.py Normal file
View file

@ -0,0 +1,70 @@
from typing import Dict
from typing import List
from ytdl_sub.downloaders.youtube.abc import YoutubeDownloader
from ytdl_sub.downloaders.youtube.abc import YoutubeDownloaderOptions
from ytdl_sub.entries.youtube import YoutubeVideo
from ytdl_sub.validators.url_validator import YoutubeVideoUrlValidator
class YoutubeVideoDownloaderOptions(YoutubeDownloaderOptions):
"""
Downloads a single youtube video. This download strategy is intended for CLI usage performing
a one-time download of a video, not a subscription.
Usage:
.. code-block:: yaml
presets:
example_preset:
youtube:
# required
download_strategy: "video"
video_url: "youtube.com/watch?v=VMAPTo7RVDo"
CLI usage:
.. code-block:: bash
ytdl-sub dl --preset "example_preset" --youtube.video_url "youtube.com/watch?v=VMAPTo7RVDo"
"""
_required_keys = {"video_url"}
def __init__(self, name, value):
super().__init__(name, value)
self._video_url = self._validate_key("video_url", YoutubeVideoUrlValidator).video_url
@property
def video_url(self) -> str:
"""
Required. The url of the video, i.e. ``youtube.com/watch?v=VMAPTo7RVDo``.
"""
return self._video_url
class YoutubeVideoDownloader(YoutubeDownloader[YoutubeVideoDownloaderOptions, YoutubeVideo]):
downloader_options_type = YoutubeVideoDownloaderOptions
downloader_entry_type = YoutubeVideo
@classmethod
def ytdl_option_defaults(cls) -> Dict:
"""
Default `ytdl_options`_ for ``video``
.. code-block:: yaml
ytdl_options:
ignoreerrors: True # ignore errors like hidden videos, age restriction, etc
"""
return dict(
super().ytdl_option_defaults(),
**{"break_on_existing": True},
)
def download(self) -> List[YoutubeVideo]:
"""Download a single Youtube video"""
entry_dict = self.extract_info(url=self.download_options.video_url)
return [YoutubeVideo(entry_dict=entry_dict, working_directory=self.working_directory)]