danvics/ytdl-sub
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

downloaders all in one

Browse source
This commit is contained in:
jbannon 2022-04-22 18:14:10 +00:00
parent 87338853bc
commit 6157648183
9 changed files with 164 additions and 146 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

16
ytdl_subscribe/config/download_strategy_validators.py
View file

@ -6,11 +6,11 @@ from typing import Type
from ytdl_subscribe.downloaders.downloader import DownloaderValidator
from ytdl_subscribe.downloaders.soundcloud_downloader import (
SoundcloudAlbumsAndSinglesSourceValidator,
SoundcloudAlbumsAndSinglesDownloadOptions,
)
from ytdl_subscribe.downloaders.youtube_downloader import YoutubeChannelDownloaderValidator
from ytdl_subscribe.downloaders.youtube_downloader import YoutubePlaylistDownloaderValidator
from ytdl_subscribe.downloaders.youtube_downloader import YoutubeVideoDownloaderValidator
from ytdl_subscribe.downloaders.youtube_downloader import YoutubeChannelDownloaderOptions
from ytdl_subscribe.downloaders.youtube_downloader import YoutubePlaylistDownloaderOptions
from ytdl_subscribe.downloaders.youtube_downloader import YoutubeVideoDownloaderOptions
from ytdl_subscribe.validators.strict_dict_validator import StrictDictValidator
from ytdl_subscribe.validators.validators import StringValidator
@ -56,13 +56,13 @@ class DownloadStrategyValidator(StrictDictValidator, ABC):
class SoundcloudDownloadStrategyValidator(DownloadStrategyValidator):
_download_strategy_to_source_mapping = {
"albums_and_singles": SoundcloudAlbumsAndSinglesSourceValidator
"albums_and_singles": SoundcloudAlbumsAndSinglesDownloadOptions
}
class YoutubeDownloadStrategyValidator(DownloadStrategyValidator):
_download_strategy_to_source_mapping = {
"channel": YoutubeChannelDownloaderValidator,
"playlist": YoutubePlaylistDownloaderValidator,
"video": YoutubeVideoDownloaderValidator,
"channel": YoutubeChannelDownloaderOptions,
"playlist": YoutubePlaylistDownloaderOptions,
"video": YoutubeVideoDownloaderOptions,
}

0
ytdl_subscribe/config/preset_downloader.py Normal file
View file

48
ytdl_subscribe/config/subscription.py
View file

@ -2,8 +2,6 @@ import copy
from typing import Any
from typing import Dict
from typing import List
from typing import Optional
from typing import Type
import yaml
from mergedeep import mergedeep
@ -13,17 +11,7 @@ from ytdl_subscribe.config.preset import PRESET_OPTIONAL_KEYS
from ytdl_subscribe.config.preset import PRESET_REQUIRED_KEYS
from ytdl_subscribe.config.preset import Overrides
from ytdl_subscribe.config.preset import PresetValidator
from ytdl_subscribe.downloaders.soundcloud_downloader import (
SoundcloudAlbumsAndSinglesSourceValidator,
)
from ytdl_subscribe.downloaders.youtube_downloader import YoutubeChannelDownloaderValidator
from ytdl_subscribe.downloaders.youtube_downloader import YoutubePlaylistDownloaderValidator
from ytdl_subscribe.downloaders.youtube_downloader import YoutubeVideoDownloaderValidator
from ytdl_subscribe.subscriptions.soundcloud import SoundcloudAlbumsAndSinglesSubscription
from ytdl_subscribe.subscriptions.subscription import Subscription
from ytdl_subscribe.subscriptions.youtube import YoutubeChannelSubscription
from ytdl_subscribe.subscriptions.youtube import YoutubePlaylistSubscription
from ytdl_subscribe.subscriptions.youtube import YoutubeVideoSubscription
from ytdl_subscribe.validators.strict_dict_validator import StrictDictValidator
from ytdl_subscribe.validators.validators import StringValidator
@ -70,23 +58,25 @@ class SubscriptionValidator(StrictDictValidator):
)
def to_subscription(self) -> Subscription:
subscription_class: Optional[Type[Subscription]] = None
if isinstance(self.preset.subscription_source, SoundcloudAlbumsAndSinglesSourceValidator):
subscription_class = SoundcloudAlbumsAndSinglesSubscription
elif isinstance(self.preset.subscription_source, YoutubePlaylistDownloaderValidator):
subscription_class = YoutubePlaylistSubscription
elif isinstance(self.preset.subscription_source, YoutubeVideoDownloaderValidator):
subscription_class = YoutubeVideoSubscription
elif isinstance(self.preset.subscription_source, YoutubeChannelDownloaderValidator):
subscription_class = YoutubeChannelSubscription
if subscription_class is None:
raise ValueError("subscription source class not found")
return subscription_class(
name=self._name,
config_options=self.config.config_options,
preset_options=self.preset,
)
# TODO: Fix this abomination
return None
# subscription_class: Optional[Type[Subscription]] = None
# if isinstance(self.preset.subscription_source, SoundcloudAlbumsAndSinglesDownloadOptions):
# subscription_class = SoundcloudAlbumsAndSinglesSubscription
# elif isinstance(self.preset.subscription_source, YoutubePlaylistDownloaderValidator):
# subscription_class = YoutubePlaylistSubscription
# elif isinstance(self.preset.subscription_source, YoutubeVideoDownloaderValidator):
# subscription_class = YoutubeVideoSubscription
# elif isinstance(self.preset.subscription_source, YoutubeChannelDownloaderValidator):
# subscription_class = YoutubeChannelSubscription
# if subscription_class is None:
# raise ValueError("subscription source class not found")
#
# return subscription_class(
# name=self._name,
# config_options=self.config.config_options,
# preset_options=self.preset,
# )
@classmethod
def from_dict(

17
ytdl_subscribe/downloaders/downloader.py
View file

@ -1,11 +1,16 @@
import abc
from abc import ABC
from contextlib import contextmanager
from pathlib import Path
from typing import Dict
from typing import Generic
from typing import List
from typing import Optional
from typing import TypeVar
import yt_dlp as ytdl
from ytdl_subscribe.entries.entry import Entry
from ytdl_subscribe.validators.strict_dict_validator import StrictDictValidator
@ -15,7 +20,11 @@ class DownloaderValidator(StrictDictValidator, ABC):
"""
class Downloader:
DownloaderOptionsT = TypeVar("DownloaderOptionsT", bound=DownloaderValidator)
DownloaderEntryT = TypeVar("DownloaderEntryT", bound=Entry)
class Downloader(Generic[DownloaderOptionsT, DownloaderEntryT]):
"""
Class that interacts with ytdl to perform the download of metadata and content,
and should translate that to list of Entry objects.
@ -59,10 +68,12 @@ class Downloader:
def __init__(
self,
working_directory: str,
download_options: DownloaderOptionsT,
ytdl_options: Optional[Dict] = None,
download_archive_file_name: Optional[str] = None,
):
self.working_directory = working_directory
self.download_options = download_options
self.ytdl_options = Downloader._configure_ytdl_options(
ytdl_options=ytdl_options,
working_directory=self.working_directory,
@ -88,3 +99,7 @@ class Downloader:
"""
with self.ytdl_downloader(ytdl_options_overrides) as ytdl_downloader:
return ytdl_downloader.extract_info(**kwargs)
@abc.abstractmethod
def download(self) -> List[DownloaderEntryT]:
"""The function to perform the download"""

53
ytdl_subscribe/downloaders/soundcloud_downloader.py
View file

@ -1,6 +1,8 @@
from abc import ABC
from typing import Dict
from typing import Generic
from typing import List
from typing import TypeVar
from ytdl_subscribe.downloaders.downloader import Downloader
from ytdl_subscribe.downloaders.downloader import DownloaderValidator
@ -9,8 +11,11 @@ from ytdl_subscribe.entries.soundcloud import SoundcloudTrack
from ytdl_subscribe.validators.validators import BoolValidator
from ytdl_subscribe.validators.validators import StringValidator
###############################################################################
# Abstract Soundcloud downloader + options
class SoundcloudDownloaderValidator(DownloaderValidator, ABC):
class SoundcloudDownloaderOptions(DownloaderValidator, ABC):
"""
Abstract source validator for all soundcloud sources.
"""
@ -24,7 +29,16 @@ class SoundcloudDownloaderValidator(DownloaderValidator, ABC):
)
class SoundcloudDownloader(Downloader):
SoundcloudDownloaderOptionsT = TypeVar(
"SoundcloudDownloaderOptionsT", bound=SoundcloudDownloaderOptions
)
class SoundcloudDownloader(
Generic[SoundcloudDownloaderOptionsT],
Downloader[SoundcloudDownloaderOptionsT, SoundcloudTrack],
ABC,
):
"""
Class that handles downloading soundcloud entries via ytdl and converting them into
SoundcloudTrack / SoundcloudAlbumTrack objects
@ -68,9 +82,42 @@ class SoundcloudDownloader(Downloader):
]
class SoundcloudAlbumsAndSinglesSourceValidator(SoundcloudDownloaderValidator):
###############################################################################
# Soundcloud albums and singles downloader + options
class SoundcloudAlbumsAndSinglesDownloadOptions(SoundcloudDownloaderOptions):
_required_keys = {"username"}
def __init__(self, name, value):
super().__init__(name, value)
self.username = self._validate_key(key="username", validator=StringValidator)
class SoundcloudAlbumsAndSinglesDownloader(
SoundcloudDownloader[SoundcloudAlbumsAndSinglesDownloadOptions]
):
def download(self) -> List[SoundcloudTrack]:
"""
Soundcloud subscription to download albums and tracks as singles.
"""
tracks: List[SoundcloudTrack] = []
# Get the album info first. This tells us which track ids belong
# to an album. Unfortunately we cannot use download_archive or info.json for this
albums: List[SoundcloudAlbum] = self.download_albums(
artist_name=self.download_options.username.value
)
for album in albums:
tracks += album.album_tracks(
skip_premiere_tracks=self.download_options.skip_premiere_tracks.value
)
# only add tracks that are not part of an album
single_tracks = self.download_tracks(artist_name=self.download_options.username.value)
tracks += [
track for track in single_tracks if not any(album.contains(track) for album in albums)
]
return tracks

86
ytdl_subscribe/downloaders/youtube_downloader.py
View file

@ -2,7 +2,11 @@ import json
import os
from abc import ABC
from pathlib import Path
from typing import Dict
from typing import Generic
from typing import List
from typing import Optional
from typing import TypeVar
from yt_dlp.utils import RejectedVideoReached
@ -12,14 +16,22 @@ from ytdl_subscribe.entries.youtube import YoutubeVideo
from ytdl_subscribe.validators.date_range_validator import DownloadDateRangeSource
from ytdl_subscribe.validators.validators import StringValidator
###############################################################################
# Abstract Youtube downloader + options
class YoutubeDownloaderValidator(DownloaderValidator, ABC):
class YoutubeDownloaderOptions(DownloaderValidator, ABC):
"""
Abstract source validator for all soundcloud sources.
"""
class YoutubeDownloader(Downloader):
YoutubeDownloaderOptionsT = TypeVar("YoutubeDownloaderOptionsT", bound=YoutubeDownloaderOptions)
class YoutubeDownloader(
Generic[YoutubeDownloaderOptionsT], Downloader[YoutubeDownloaderOptionsT, YoutubeVideo], ABC
):
"""
Class that handles downloading youtube entries via ytdl and converting them into
YoutubeVideo objects
@ -40,18 +52,22 @@ class YoutubeDownloader(Downloader):
"""Returns full channel url"""
return f"https://youtube.com/channel/{channel_id}"
def _download_with_metadata(self, url: str) -> None:
def _download_with_metadata(
self, url: str, ytdl_options_overrides: Optional[Dict] = None
) -> None:
"""
Do not get entries from the extract info, let it write to the info.json file and load
that instead. This is because if the video is already downloaded in a playlist, it will
not fetch the metadata (maybe there is a way??)
"""
ytdl_metadata_override = {
ytdl_overrides = {
"writeinfojson": True,
}
if ytdl_options_overrides:
ytdl_overrides = dict(ytdl_overrides, **ytdl_options_overrides)
try:
_ = self.extract_info(ytdl_options_overrides=ytdl_metadata_override, url=url)
_ = self.extract_info(ytdl_options_overrides=ytdl_overrides, url=url)
except RejectedVideoReached:
pass
@ -79,11 +95,15 @@ class YoutubeDownloader(Downloader):
return entries
def download_channel(self, channel_id: str) -> List[YoutubeVideo]:
def download_channel(
self, channel_id: str, ytdl_options_overrides: Optional[Dict] = None
) -> List[YoutubeVideo]:
"""
Downloads all videos from a channel
"""
self._download_with_metadata(url=self.channel_url(channel_id))
self._download_with_metadata(
url=self.channel_url(channel_id), ytdl_options_overrides=ytdl_options_overrides
)
# Load the entries from info.json
entries: List[YoutubeVideo] = []
@ -98,7 +118,29 @@ class YoutubeDownloader(Downloader):
return entries
class YoutubePlaylistDownloaderValidator(YoutubeDownloaderValidator):
###############################################################################
# Youtube single video downloader + options
class YoutubeVideoDownloaderOptions(YoutubeDownloaderOptions):
_required_keys = {"video_id"}
def __init__(self, name, value):
super().__init__(name, value)
self.video_id = self._validate_key("video_id", StringValidator)
class YoutubeVideoDownloader(YoutubeDownloader[YoutubeVideoDownloaderOptions]):
def download(self) -> List[YoutubeVideo]:
video = self.download_video(video_id=self.download_options.video_id.value)
return [video]
###############################################################################
# Youtube playlist downloader + options
class YoutubePlaylistDownloaderOptions(YoutubeDownloaderOptions):
_required_keys = {"playlist_id"}
def __init__(self, name, value):
@ -106,19 +148,33 @@ class YoutubePlaylistDownloaderValidator(YoutubeDownloaderValidator):
self.playlist_id = self._validate_key("playlist_id", StringValidator)
class YoutubeChannelDownloaderValidator(YoutubeDownloaderValidator, DownloadDateRangeSource):
class YoutubePlaylistDownloader(YoutubeDownloader[YoutubePlaylistDownloaderOptions]):
def download(self) -> List[YoutubeVideo]:
return self.download_playlist(playlist_id=self.download_options.playlist_id.value)
###############################################################################
# Youtube channel downloader + options
class YoutubeChannelDownloaderOptions(YoutubeDownloaderOptions, DownloadDateRangeSource):
_required_keys = {"channel_id"}
_optional_keys = {"before", "after"}
def __init__(self, name, value):
YoutubeDownloaderValidator.__init__(self, name, value)
YoutubeDownloaderOptions.__init__(self, name, value)
DownloadDateRangeSource.__init__(self, name, value)
self.channel_id = self._validate_key("channel_id", StringValidator)
class YoutubeVideoDownloaderValidator(YoutubeDownloaderValidator):
_required_keys = {"video_id"}
class YoutubeChannelDownloader(YoutubeDownloader[YoutubeChannelDownloaderOptions]):
def download(self) -> List[YoutubeVideo]:
ytdl_options_overrides = {}
source_date_range = self.download_options.get_date_range()
if source_date_range:
ytdl_options_overrides["daterange"] = source_date_range
def __init__(self, name, value):
super().__init__(name, value)
self.video_id = self._validate_key("video_id", StringValidator)
return self.download_channel(
channel_id=self.download_options.channel_id.value,
ytdl_options_overrides=ytdl_options_overrides,
)

4
ytdl_subscribe/plugins/plugin.py
View file

@ -14,8 +14,6 @@ class PluginValidator(StrictDictValidator):
Class that defines the parameters to a plugin
"""
pass
PluginValidatorT = TypeVar("PluginValidatorT", bound=PluginValidator)
@ -62,10 +60,8 @@ class Plugin(Generic[PluginValidatorT], ABC):
----------
entry: Entry to post process
"""
pass
def post_process_subscription(self):
"""
After all downloaded files have been post-processed, apply a subscription-wide post process
"""
pass

40
ytdl_subscribe/subscriptions/soundcloud.py
View file

@ -1,40 +0,0 @@
from typing import List
from ytdl_subscribe.downloaders.soundcloud_downloader import (
SoundcloudAlbumsAndSinglesSourceValidator,
)
from ytdl_subscribe.downloaders.soundcloud_downloader import SoundcloudDownloader
from ytdl_subscribe.entries.soundcloud import SoundcloudAlbum
from ytdl_subscribe.entries.soundcloud import SoundcloudTrack
from ytdl_subscribe.subscriptions.subscription import Subscription
class SoundcloudAlbumsAndSinglesSubscription(
Subscription[SoundcloudAlbumsAndSinglesSourceValidator, SoundcloudTrack]
):
"""
Soundcloud subscription to download albums and tracks as singles.
"""
def _extract_info(self) -> List[SoundcloudTrack]:
tracks: List[SoundcloudTrack] = []
downloader = self.get_downloader(SoundcloudDownloader)
# Get the album info first. This tells us which track ids belong
# to an album. Unfortunately we cannot use download_archive or info.json for this
albums: List[SoundcloudAlbum] = downloader.download_albums(
artist_name=self.source_options.username.value
)
for album in albums:
tracks += album.album_tracks(
skip_premiere_tracks=self.source_options.skip_premiere_tracks.value
)
# only add tracks that are not part of an album
single_tracks = downloader.download_tracks(artist_name=self.source_options.username.value)
tracks += [
track for track in single_tracks if not any(album.contains(track) for album in albums)
]
return tracks

46
ytdl_subscribe/subscriptions/youtube.py
View file

@ -1,46 +0,0 @@
from typing import List
from ytdl_subscribe.downloaders.youtube_downloader import YoutubeChannelDownloaderValidator
from ytdl_subscribe.downloaders.youtube_downloader import YoutubeDownloader
from ytdl_subscribe.downloaders.youtube_downloader import YoutubePlaylistDownloaderValidator
from ytdl_subscribe.downloaders.youtube_downloader import YoutubeVideoDownloaderValidator
from ytdl_subscribe.entries.youtube import YoutubeVideo
from ytdl_subscribe.subscriptions.subscription import Subscription
class YoutubePlaylistSubscription(Subscription[YoutubePlaylistDownloaderValidator, YoutubeVideo]):
"""
Youtube subscription to download videos from a playlist
"""
def _extract_info(self) -> List[YoutubeVideo]:
return self.get_downloader(YoutubeDownloader).download_playlist(
playlist_id=self.source_options.playlist_id.value
)
class YoutubeChannelSubscription(Subscription[YoutubeChannelDownloaderValidator, YoutubeVideo]):
"""
Youtube subscription to download videos from a channel
"""
def _extract_info(self) -> List[YoutubeVideo]:
source_ytdl_options = {}
source_date_range = self.source_options.get_date_range()
if source_date_range:
source_ytdl_options["daterange"] = source_date_range
downloader = self.get_downloader(YoutubeDownloader, source_ytdl_options=source_ytdl_options)
return downloader.download_channel(channel_id=self.source_options.channel_id.value)
class YoutubeVideoSubscription(Subscription[YoutubeVideoDownloaderValidator, YoutubeVideo]):
"""
Youtube subscription to download a single video
"""
def _extract_info(self) -> List[YoutubeVideo]:
entry = self.get_downloader(YoutubeDownloader).download_video(
video_id=self.source_options.video_id.value
)
return [entry]