diff --git a/ytdl_subscribe/config/download_strategy_validators.py b/ytdl_subscribe/config/download_strategy_validators.py index 8dfc7765..f8e5b98a 100644 --- a/ytdl_subscribe/config/download_strategy_validators.py +++ b/ytdl_subscribe/config/download_strategy_validators.py @@ -6,11 +6,11 @@ from typing import Type from ytdl_subscribe.downloaders.downloader import DownloaderValidator from ytdl_subscribe.downloaders.soundcloud_downloader import ( - SoundcloudAlbumsAndSinglesSourceValidator, + SoundcloudAlbumsAndSinglesDownloadOptions, ) -from ytdl_subscribe.downloaders.youtube_downloader import YoutubeChannelDownloaderValidator -from ytdl_subscribe.downloaders.youtube_downloader import YoutubePlaylistDownloaderValidator -from ytdl_subscribe.downloaders.youtube_downloader import YoutubeVideoDownloaderValidator +from ytdl_subscribe.downloaders.youtube_downloader import YoutubeChannelDownloaderOptions +from ytdl_subscribe.downloaders.youtube_downloader import YoutubePlaylistDownloaderOptions +from ytdl_subscribe.downloaders.youtube_downloader import YoutubeVideoDownloaderOptions from ytdl_subscribe.validators.strict_dict_validator import StrictDictValidator from ytdl_subscribe.validators.validators import StringValidator @@ -56,13 +56,13 @@ class DownloadStrategyValidator(StrictDictValidator, ABC): class SoundcloudDownloadStrategyValidator(DownloadStrategyValidator): _download_strategy_to_source_mapping = { - "albums_and_singles": SoundcloudAlbumsAndSinglesSourceValidator + "albums_and_singles": SoundcloudAlbumsAndSinglesDownloadOptions } class YoutubeDownloadStrategyValidator(DownloadStrategyValidator): _download_strategy_to_source_mapping = { - "channel": YoutubeChannelDownloaderValidator, - "playlist": YoutubePlaylistDownloaderValidator, - "video": YoutubeVideoDownloaderValidator, + "channel": YoutubeChannelDownloaderOptions, + "playlist": YoutubePlaylistDownloaderOptions, + "video": YoutubeVideoDownloaderOptions, } diff --git a/ytdl_subscribe/config/preset_downloader.py b/ytdl_subscribe/config/preset_downloader.py new file mode 100644 index 00000000..e69de29b diff --git a/ytdl_subscribe/config/subscription.py b/ytdl_subscribe/config/subscription.py index 9811dfe1..5dca0d51 100644 --- a/ytdl_subscribe/config/subscription.py +++ b/ytdl_subscribe/config/subscription.py @@ -2,8 +2,6 @@ import copy from typing import Any from typing import Dict from typing import List -from typing import Optional -from typing import Type import yaml from mergedeep import mergedeep @@ -13,17 +11,7 @@ from ytdl_subscribe.config.preset import PRESET_OPTIONAL_KEYS from ytdl_subscribe.config.preset import PRESET_REQUIRED_KEYS from ytdl_subscribe.config.preset import Overrides from ytdl_subscribe.config.preset import PresetValidator -from ytdl_subscribe.downloaders.soundcloud_downloader import ( - SoundcloudAlbumsAndSinglesSourceValidator, -) -from ytdl_subscribe.downloaders.youtube_downloader import YoutubeChannelDownloaderValidator -from ytdl_subscribe.downloaders.youtube_downloader import YoutubePlaylistDownloaderValidator -from ytdl_subscribe.downloaders.youtube_downloader import YoutubeVideoDownloaderValidator -from ytdl_subscribe.subscriptions.soundcloud import SoundcloudAlbumsAndSinglesSubscription from ytdl_subscribe.subscriptions.subscription import Subscription -from ytdl_subscribe.subscriptions.youtube import YoutubeChannelSubscription -from ytdl_subscribe.subscriptions.youtube import YoutubePlaylistSubscription -from ytdl_subscribe.subscriptions.youtube import YoutubeVideoSubscription from ytdl_subscribe.validators.strict_dict_validator import StrictDictValidator from ytdl_subscribe.validators.validators import StringValidator @@ -70,23 +58,25 @@ class SubscriptionValidator(StrictDictValidator): ) def to_subscription(self) -> Subscription: - subscription_class: Optional[Type[Subscription]] = None - if isinstance(self.preset.subscription_source, SoundcloudAlbumsAndSinglesSourceValidator): - subscription_class = SoundcloudAlbumsAndSinglesSubscription - elif isinstance(self.preset.subscription_source, YoutubePlaylistDownloaderValidator): - subscription_class = YoutubePlaylistSubscription - elif isinstance(self.preset.subscription_source, YoutubeVideoDownloaderValidator): - subscription_class = YoutubeVideoSubscription - elif isinstance(self.preset.subscription_source, YoutubeChannelDownloaderValidator): - subscription_class = YoutubeChannelSubscription - if subscription_class is None: - raise ValueError("subscription source class not found") - - return subscription_class( - name=self._name, - config_options=self.config.config_options, - preset_options=self.preset, - ) + # TODO: Fix this abomination + return None + # subscription_class: Optional[Type[Subscription]] = None + # if isinstance(self.preset.subscription_source, SoundcloudAlbumsAndSinglesDownloadOptions): + # subscription_class = SoundcloudAlbumsAndSinglesSubscription + # elif isinstance(self.preset.subscription_source, YoutubePlaylistDownloaderValidator): + # subscription_class = YoutubePlaylistSubscription + # elif isinstance(self.preset.subscription_source, YoutubeVideoDownloaderValidator): + # subscription_class = YoutubeVideoSubscription + # elif isinstance(self.preset.subscription_source, YoutubeChannelDownloaderValidator): + # subscription_class = YoutubeChannelSubscription + # if subscription_class is None: + # raise ValueError("subscription source class not found") + # + # return subscription_class( + # name=self._name, + # config_options=self.config.config_options, + # preset_options=self.preset, + # ) @classmethod def from_dict( diff --git a/ytdl_subscribe/downloaders/downloader.py b/ytdl_subscribe/downloaders/downloader.py index 4bedc31c..55e73d53 100644 --- a/ytdl_subscribe/downloaders/downloader.py +++ b/ytdl_subscribe/downloaders/downloader.py @@ -1,11 +1,16 @@ +import abc from abc import ABC from contextlib import contextmanager from pathlib import Path from typing import Dict +from typing import Generic +from typing import List from typing import Optional +from typing import TypeVar import yt_dlp as ytdl +from ytdl_subscribe.entries.entry import Entry from ytdl_subscribe.validators.strict_dict_validator import StrictDictValidator @@ -15,7 +20,11 @@ class DownloaderValidator(StrictDictValidator, ABC): """ -class Downloader: +DownloaderOptionsT = TypeVar("DownloaderOptionsT", bound=DownloaderValidator) +DownloaderEntryT = TypeVar("DownloaderEntryT", bound=Entry) + + +class Downloader(Generic[DownloaderOptionsT, DownloaderEntryT]): """ Class that interacts with ytdl to perform the download of metadata and content, and should translate that to list of Entry objects. @@ -59,10 +68,12 @@ class Downloader: def __init__( self, working_directory: str, + download_options: DownloaderOptionsT, ytdl_options: Optional[Dict] = None, download_archive_file_name: Optional[str] = None, ): self.working_directory = working_directory + self.download_options = download_options self.ytdl_options = Downloader._configure_ytdl_options( ytdl_options=ytdl_options, working_directory=self.working_directory, @@ -88,3 +99,7 @@ class Downloader: """ with self.ytdl_downloader(ytdl_options_overrides) as ytdl_downloader: return ytdl_downloader.extract_info(**kwargs) + + @abc.abstractmethod + def download(self) -> List[DownloaderEntryT]: + """The function to perform the download""" diff --git a/ytdl_subscribe/downloaders/soundcloud_downloader.py b/ytdl_subscribe/downloaders/soundcloud_downloader.py index 27295268..ea7d1223 100644 --- a/ytdl_subscribe/downloaders/soundcloud_downloader.py +++ b/ytdl_subscribe/downloaders/soundcloud_downloader.py @@ -1,6 +1,8 @@ from abc import ABC from typing import Dict +from typing import Generic from typing import List +from typing import TypeVar from ytdl_subscribe.downloaders.downloader import Downloader from ytdl_subscribe.downloaders.downloader import DownloaderValidator @@ -9,8 +11,11 @@ from ytdl_subscribe.entries.soundcloud import SoundcloudTrack from ytdl_subscribe.validators.validators import BoolValidator from ytdl_subscribe.validators.validators import StringValidator +############################################################################### +# Abstract Soundcloud downloader + options -class SoundcloudDownloaderValidator(DownloaderValidator, ABC): + +class SoundcloudDownloaderOptions(DownloaderValidator, ABC): """ Abstract source validator for all soundcloud sources. """ @@ -24,7 +29,16 @@ class SoundcloudDownloaderValidator(DownloaderValidator, ABC): ) -class SoundcloudDownloader(Downloader): +SoundcloudDownloaderOptionsT = TypeVar( + "SoundcloudDownloaderOptionsT", bound=SoundcloudDownloaderOptions +) + + +class SoundcloudDownloader( + Generic[SoundcloudDownloaderOptionsT], + Downloader[SoundcloudDownloaderOptionsT, SoundcloudTrack], + ABC, +): """ Class that handles downloading soundcloud entries via ytdl and converting them into SoundcloudTrack / SoundcloudAlbumTrack objects @@ -68,9 +82,42 @@ class SoundcloudDownloader(Downloader): ] -class SoundcloudAlbumsAndSinglesSourceValidator(SoundcloudDownloaderValidator): +############################################################################### +# Soundcloud albums and singles downloader + options + + +class SoundcloudAlbumsAndSinglesDownloadOptions(SoundcloudDownloaderOptions): _required_keys = {"username"} def __init__(self, name, value): super().__init__(name, value) self.username = self._validate_key(key="username", validator=StringValidator) + + +class SoundcloudAlbumsAndSinglesDownloader( + SoundcloudDownloader[SoundcloudAlbumsAndSinglesDownloadOptions] +): + def download(self) -> List[SoundcloudTrack]: + """ + Soundcloud subscription to download albums and tracks as singles. + """ + tracks: List[SoundcloudTrack] = [] + + # Get the album info first. This tells us which track ids belong + # to an album. Unfortunately we cannot use download_archive or info.json for this + albums: List[SoundcloudAlbum] = self.download_albums( + artist_name=self.download_options.username.value + ) + + for album in albums: + tracks += album.album_tracks( + skip_premiere_tracks=self.download_options.skip_premiere_tracks.value + ) + + # only add tracks that are not part of an album + single_tracks = self.download_tracks(artist_name=self.download_options.username.value) + tracks += [ + track for track in single_tracks if not any(album.contains(track) for album in albums) + ] + + return tracks diff --git a/ytdl_subscribe/downloaders/youtube_downloader.py b/ytdl_subscribe/downloaders/youtube_downloader.py index 4fc77041..f4ac9908 100644 --- a/ytdl_subscribe/downloaders/youtube_downloader.py +++ b/ytdl_subscribe/downloaders/youtube_downloader.py @@ -2,7 +2,11 @@ import json import os from abc import ABC from pathlib import Path +from typing import Dict +from typing import Generic from typing import List +from typing import Optional +from typing import TypeVar from yt_dlp.utils import RejectedVideoReached @@ -12,14 +16,22 @@ from ytdl_subscribe.entries.youtube import YoutubeVideo from ytdl_subscribe.validators.date_range_validator import DownloadDateRangeSource from ytdl_subscribe.validators.validators import StringValidator +############################################################################### +# Abstract Youtube downloader + options -class YoutubeDownloaderValidator(DownloaderValidator, ABC): + +class YoutubeDownloaderOptions(DownloaderValidator, ABC): """ Abstract source validator for all soundcloud sources. """ -class YoutubeDownloader(Downloader): +YoutubeDownloaderOptionsT = TypeVar("YoutubeDownloaderOptionsT", bound=YoutubeDownloaderOptions) + + +class YoutubeDownloader( + Generic[YoutubeDownloaderOptionsT], Downloader[YoutubeDownloaderOptionsT, YoutubeVideo], ABC +): """ Class that handles downloading youtube entries via ytdl and converting them into YoutubeVideo objects @@ -40,18 +52,22 @@ class YoutubeDownloader(Downloader): """Returns full channel url""" return f"https://youtube.com/channel/{channel_id}" - def _download_with_metadata(self, url: str) -> None: + def _download_with_metadata( + self, url: str, ytdl_options_overrides: Optional[Dict] = None + ) -> None: """ Do not get entries from the extract info, let it write to the info.json file and load that instead. This is because if the video is already downloaded in a playlist, it will not fetch the metadata (maybe there is a way??) """ - ytdl_metadata_override = { + ytdl_overrides = { "writeinfojson": True, } + if ytdl_options_overrides: + ytdl_overrides = dict(ytdl_overrides, **ytdl_options_overrides) try: - _ = self.extract_info(ytdl_options_overrides=ytdl_metadata_override, url=url) + _ = self.extract_info(ytdl_options_overrides=ytdl_overrides, url=url) except RejectedVideoReached: pass @@ -79,11 +95,15 @@ class YoutubeDownloader(Downloader): return entries - def download_channel(self, channel_id: str) -> List[YoutubeVideo]: + def download_channel( + self, channel_id: str, ytdl_options_overrides: Optional[Dict] = None + ) -> List[YoutubeVideo]: """ Downloads all videos from a channel """ - self._download_with_metadata(url=self.channel_url(channel_id)) + self._download_with_metadata( + url=self.channel_url(channel_id), ytdl_options_overrides=ytdl_options_overrides + ) # Load the entries from info.json entries: List[YoutubeVideo] = [] @@ -98,7 +118,29 @@ class YoutubeDownloader(Downloader): return entries -class YoutubePlaylistDownloaderValidator(YoutubeDownloaderValidator): +############################################################################### +# Youtube single video downloader + options + + +class YoutubeVideoDownloaderOptions(YoutubeDownloaderOptions): + _required_keys = {"video_id"} + + def __init__(self, name, value): + super().__init__(name, value) + self.video_id = self._validate_key("video_id", StringValidator) + + +class YoutubeVideoDownloader(YoutubeDownloader[YoutubeVideoDownloaderOptions]): + def download(self) -> List[YoutubeVideo]: + video = self.download_video(video_id=self.download_options.video_id.value) + return [video] + + +############################################################################### +# Youtube playlist downloader + options + + +class YoutubePlaylistDownloaderOptions(YoutubeDownloaderOptions): _required_keys = {"playlist_id"} def __init__(self, name, value): @@ -106,19 +148,33 @@ class YoutubePlaylistDownloaderValidator(YoutubeDownloaderValidator): self.playlist_id = self._validate_key("playlist_id", StringValidator) -class YoutubeChannelDownloaderValidator(YoutubeDownloaderValidator, DownloadDateRangeSource): +class YoutubePlaylistDownloader(YoutubeDownloader[YoutubePlaylistDownloaderOptions]): + def download(self) -> List[YoutubeVideo]: + return self.download_playlist(playlist_id=self.download_options.playlist_id.value) + + +############################################################################### +# Youtube channel downloader + options + + +class YoutubeChannelDownloaderOptions(YoutubeDownloaderOptions, DownloadDateRangeSource): _required_keys = {"channel_id"} _optional_keys = {"before", "after"} def __init__(self, name, value): - YoutubeDownloaderValidator.__init__(self, name, value) + YoutubeDownloaderOptions.__init__(self, name, value) DownloadDateRangeSource.__init__(self, name, value) self.channel_id = self._validate_key("channel_id", StringValidator) -class YoutubeVideoDownloaderValidator(YoutubeDownloaderValidator): - _required_keys = {"video_id"} +class YoutubeChannelDownloader(YoutubeDownloader[YoutubeChannelDownloaderOptions]): + def download(self) -> List[YoutubeVideo]: + ytdl_options_overrides = {} + source_date_range = self.download_options.get_date_range() + if source_date_range: + ytdl_options_overrides["daterange"] = source_date_range - def __init__(self, name, value): - super().__init__(name, value) - self.video_id = self._validate_key("video_id", StringValidator) + return self.download_channel( + channel_id=self.download_options.channel_id.value, + ytdl_options_overrides=ytdl_options_overrides, + ) diff --git a/ytdl_subscribe/plugins/plugin.py b/ytdl_subscribe/plugins/plugin.py index b74f3c62..defb204b 100644 --- a/ytdl_subscribe/plugins/plugin.py +++ b/ytdl_subscribe/plugins/plugin.py @@ -14,8 +14,6 @@ class PluginValidator(StrictDictValidator): Class that defines the parameters to a plugin """ - pass - PluginValidatorT = TypeVar("PluginValidatorT", bound=PluginValidator) @@ -62,10 +60,8 @@ class Plugin(Generic[PluginValidatorT], ABC): ---------- entry: Entry to post process """ - pass def post_process_subscription(self): """ After all downloaded files have been post-processed, apply a subscription-wide post process """ - pass diff --git a/ytdl_subscribe/subscriptions/soundcloud.py b/ytdl_subscribe/subscriptions/soundcloud.py deleted file mode 100644 index 15c9b55f..00000000 --- a/ytdl_subscribe/subscriptions/soundcloud.py +++ /dev/null @@ -1,40 +0,0 @@ -from typing import List - -from ytdl_subscribe.downloaders.soundcloud_downloader import ( - SoundcloudAlbumsAndSinglesSourceValidator, -) -from ytdl_subscribe.downloaders.soundcloud_downloader import SoundcloudDownloader -from ytdl_subscribe.entries.soundcloud import SoundcloudAlbum -from ytdl_subscribe.entries.soundcloud import SoundcloudTrack -from ytdl_subscribe.subscriptions.subscription import Subscription - - -class SoundcloudAlbumsAndSinglesSubscription( - Subscription[SoundcloudAlbumsAndSinglesSourceValidator, SoundcloudTrack] -): - """ - Soundcloud subscription to download albums and tracks as singles. - """ - - def _extract_info(self) -> List[SoundcloudTrack]: - tracks: List[SoundcloudTrack] = [] - downloader = self.get_downloader(SoundcloudDownloader) - - # Get the album info first. This tells us which track ids belong - # to an album. Unfortunately we cannot use download_archive or info.json for this - albums: List[SoundcloudAlbum] = downloader.download_albums( - artist_name=self.source_options.username.value - ) - - for album in albums: - tracks += album.album_tracks( - skip_premiere_tracks=self.source_options.skip_premiere_tracks.value - ) - - # only add tracks that are not part of an album - single_tracks = downloader.download_tracks(artist_name=self.source_options.username.value) - tracks += [ - track for track in single_tracks if not any(album.contains(track) for album in albums) - ] - - return tracks diff --git a/ytdl_subscribe/subscriptions/youtube.py b/ytdl_subscribe/subscriptions/youtube.py deleted file mode 100644 index 5e100341..00000000 --- a/ytdl_subscribe/subscriptions/youtube.py +++ /dev/null @@ -1,46 +0,0 @@ -from typing import List - -from ytdl_subscribe.downloaders.youtube_downloader import YoutubeChannelDownloaderValidator -from ytdl_subscribe.downloaders.youtube_downloader import YoutubeDownloader -from ytdl_subscribe.downloaders.youtube_downloader import YoutubePlaylistDownloaderValidator -from ytdl_subscribe.downloaders.youtube_downloader import YoutubeVideoDownloaderValidator -from ytdl_subscribe.entries.youtube import YoutubeVideo -from ytdl_subscribe.subscriptions.subscription import Subscription - - -class YoutubePlaylistSubscription(Subscription[YoutubePlaylistDownloaderValidator, YoutubeVideo]): - """ - Youtube subscription to download videos from a playlist - """ - - def _extract_info(self) -> List[YoutubeVideo]: - return self.get_downloader(YoutubeDownloader).download_playlist( - playlist_id=self.source_options.playlist_id.value - ) - - -class YoutubeChannelSubscription(Subscription[YoutubeChannelDownloaderValidator, YoutubeVideo]): - """ - Youtube subscription to download videos from a channel - """ - - def _extract_info(self) -> List[YoutubeVideo]: - source_ytdl_options = {} - source_date_range = self.source_options.get_date_range() - if source_date_range: - source_ytdl_options["daterange"] = source_date_range - - downloader = self.get_downloader(YoutubeDownloader, source_ytdl_options=source_ytdl_options) - return downloader.download_channel(channel_id=self.source_options.channel_id.value) - - -class YoutubeVideoSubscription(Subscription[YoutubeVideoDownloaderValidator, YoutubeVideo]): - """ - Youtube subscription to download a single video - """ - - def _extract_info(self) -> List[YoutubeVideo]: - entry = self.get_downloader(YoutubeDownloader).download_video( - video_id=self.source_options.video_id.value - ) - return [entry]