downloaders all in one

ytdl_subscribe/config/download_strategy_validators.py

@@ -6,11 +6,11 @@ from typing import Type
 
 from ytdl_subscribe.downloaders.downloader import DownloaderValidator
 from ytdl_subscribe.downloaders.soundcloud_downloader import (
-    SoundcloudAlbumsAndSinglesSourceValidator,
+    SoundcloudAlbumsAndSinglesDownloadOptions,
 )
-from ytdl_subscribe.downloaders.youtube_downloader import YoutubeChannelDownloaderValidator
+from ytdl_subscribe.downloaders.youtube_downloader import YoutubeChannelDownloaderOptions
-from ytdl_subscribe.downloaders.youtube_downloader import YoutubePlaylistDownloaderValidator
+from ytdl_subscribe.downloaders.youtube_downloader import YoutubePlaylistDownloaderOptions
-from ytdl_subscribe.downloaders.youtube_downloader import YoutubeVideoDownloaderValidator
+from ytdl_subscribe.downloaders.youtube_downloader import YoutubeVideoDownloaderOptions
 from ytdl_subscribe.validators.strict_dict_validator import StrictDictValidator
 from ytdl_subscribe.validators.validators import StringValidator
 
@@ -56,13 +56,13 @@ class DownloadStrategyValidator(StrictDictValidator, ABC):
 
 class SoundcloudDownloadStrategyValidator(DownloadStrategyValidator):
     _download_strategy_to_source_mapping = {
-        "albums_and_singles": SoundcloudAlbumsAndSinglesSourceValidator
+        "albums_and_singles": SoundcloudAlbumsAndSinglesDownloadOptions
     }
 
 
 class YoutubeDownloadStrategyValidator(DownloadStrategyValidator):
     _download_strategy_to_source_mapping = {
-        "channel": YoutubeChannelDownloaderValidator,
+        "channel": YoutubeChannelDownloaderOptions,
-        "playlist": YoutubePlaylistDownloaderValidator,
+        "playlist": YoutubePlaylistDownloaderOptions,
-        "video": YoutubeVideoDownloaderValidator,
+        "video": YoutubeVideoDownloaderOptions,
     }

ytdl_subscribe/config/subscription.py

@@ -2,8 +2,6 @@ import copy
 from typing import Any
 from typing import Dict
 from typing import List
-from typing import Optional
-from typing import Type
 
 import yaml
 from mergedeep import mergedeep
@@ -13,17 +11,7 @@ from ytdl_subscribe.config.preset import PRESET_OPTIONAL_KEYS
 from ytdl_subscribe.config.preset import PRESET_REQUIRED_KEYS
 from ytdl_subscribe.config.preset import Overrides
 from ytdl_subscribe.config.preset import PresetValidator
-from ytdl_subscribe.downloaders.soundcloud_downloader import (
-    SoundcloudAlbumsAndSinglesSourceValidator,
-)
-from ytdl_subscribe.downloaders.youtube_downloader import YoutubeChannelDownloaderValidator
-from ytdl_subscribe.downloaders.youtube_downloader import YoutubePlaylistDownloaderValidator
-from ytdl_subscribe.downloaders.youtube_downloader import YoutubeVideoDownloaderValidator
-from ytdl_subscribe.subscriptions.soundcloud import SoundcloudAlbumsAndSinglesSubscription
 from ytdl_subscribe.subscriptions.subscription import Subscription
-from ytdl_subscribe.subscriptions.youtube import YoutubeChannelSubscription
-from ytdl_subscribe.subscriptions.youtube import YoutubePlaylistSubscription
-from ytdl_subscribe.subscriptions.youtube import YoutubeVideoSubscription
 from ytdl_subscribe.validators.strict_dict_validator import StrictDictValidator
 from ytdl_subscribe.validators.validators import StringValidator
 
@@ -70,23 +58,25 @@ class SubscriptionValidator(StrictDictValidator):
         )
 
     def to_subscription(self) -> Subscription:
-        subscription_class: Optional[Type[Subscription]] = None
+        # TODO: Fix this abomination
-        if isinstance(self.preset.subscription_source, SoundcloudAlbumsAndSinglesSourceValidator):
+        return None
-            subscription_class = SoundcloudAlbumsAndSinglesSubscription
+        # subscription_class: Optional[Type[Subscription]] = None
-        elif isinstance(self.preset.subscription_source, YoutubePlaylistDownloaderValidator):
+        # if isinstance(self.preset.subscription_source, SoundcloudAlbumsAndSinglesDownloadOptions):
-            subscription_class = YoutubePlaylistSubscription
+        #     subscription_class = SoundcloudAlbumsAndSinglesSubscription
-        elif isinstance(self.preset.subscription_source, YoutubeVideoDownloaderValidator):
+        # elif isinstance(self.preset.subscription_source, YoutubePlaylistDownloaderValidator):
-            subscription_class = YoutubeVideoSubscription
+        #     subscription_class = YoutubePlaylistSubscription
-        elif isinstance(self.preset.subscription_source, YoutubeChannelDownloaderValidator):
+        # elif isinstance(self.preset.subscription_source, YoutubeVideoDownloaderValidator):
-            subscription_class = YoutubeChannelSubscription
+        #     subscription_class = YoutubeVideoSubscription
-        if subscription_class is None:
+        # elif isinstance(self.preset.subscription_source, YoutubeChannelDownloaderValidator):
-            raise ValueError("subscription source class not found")
+        #     subscription_class = YoutubeChannelSubscription
-
+        # if subscription_class is None:
-        return subscription_class(
+        #     raise ValueError("subscription source class not found")
-            name=self._name,
+        #
-            config_options=self.config.config_options,
+        # return subscription_class(
-            preset_options=self.preset,
+        #     name=self._name,
-        )
+        #     config_options=self.config.config_options,
+        #     preset_options=self.preset,
+        # )
 
     @classmethod
     def from_dict(

ytdl_subscribe/downloaders/downloader.py

@@ -1,11 +1,16 @@
+import abc
 from abc import ABC
 from contextlib import contextmanager
 from pathlib import Path
 from typing import Dict
+from typing import Generic
+from typing import List
 from typing import Optional
+from typing import TypeVar
 
 import yt_dlp as ytdl
 
+from ytdl_subscribe.entries.entry import Entry
 from ytdl_subscribe.validators.strict_dict_validator import StrictDictValidator
 
 
@@ -15,7 +20,11 @@ class DownloaderValidator(StrictDictValidator, ABC):
     """
 
 
-class Downloader:
+DownloaderOptionsT = TypeVar("DownloaderOptionsT", bound=DownloaderValidator)
+DownloaderEntryT = TypeVar("DownloaderEntryT", bound=Entry)
+
+
+class Downloader(Generic[DownloaderOptionsT, DownloaderEntryT]):
     """
     Class that interacts with ytdl to perform the download of metadata and content,
     and should translate that to list of Entry objects.
@@ -59,10 +68,12 @@ class Downloader:
     def __init__(
         self,
         working_directory: str,
+        download_options: DownloaderOptionsT,
         ytdl_options: Optional[Dict] = None,
         download_archive_file_name: Optional[str] = None,
     ):
         self.working_directory = working_directory
+        self.download_options = download_options
         self.ytdl_options = Downloader._configure_ytdl_options(
             ytdl_options=ytdl_options,
             working_directory=self.working_directory,
@@ -88,3 +99,7 @@ class Downloader:
         """
         with self.ytdl_downloader(ytdl_options_overrides) as ytdl_downloader:
             return ytdl_downloader.extract_info(**kwargs)
+
+    @abc.abstractmethod
+    def download(self) -> List[DownloaderEntryT]:
+        """The function to perform the download"""

ytdl_subscribe/downloaders/soundcloud_downloader.py

@@ -1,6 +1,8 @@
 from abc import ABC
 from typing import Dict
+from typing import Generic
 from typing import List
+from typing import TypeVar
 
 from ytdl_subscribe.downloaders.downloader import Downloader
 from ytdl_subscribe.downloaders.downloader import DownloaderValidator
@@ -9,8 +11,11 @@ from ytdl_subscribe.entries.soundcloud import SoundcloudTrack
 from ytdl_subscribe.validators.validators import BoolValidator
 from ytdl_subscribe.validators.validators import StringValidator
 
+###############################################################################
+# Abstract Soundcloud downloader + options
 
-class SoundcloudDownloaderValidator(DownloaderValidator, ABC):
+
+class SoundcloudDownloaderOptions(DownloaderValidator, ABC):
     """
     Abstract source validator for all soundcloud sources.
     """
@@ -24,7 +29,16 @@ class SoundcloudDownloaderValidator(DownloaderValidator, ABC):
         )
 
 
-class SoundcloudDownloader(Downloader):
+SoundcloudDownloaderOptionsT = TypeVar(
+    "SoundcloudDownloaderOptionsT", bound=SoundcloudDownloaderOptions
+)
+
+
+class SoundcloudDownloader(
+    Generic[SoundcloudDownloaderOptionsT],
+    Downloader[SoundcloudDownloaderOptionsT, SoundcloudTrack],
+    ABC,
+):
     """
     Class that handles downloading soundcloud entries via ytdl and converting them into
     SoundcloudTrack / SoundcloudAlbumTrack objects
@@ -68,9 +82,42 @@ class SoundcloudDownloader(Downloader):
         ]
 
 
-class SoundcloudAlbumsAndSinglesSourceValidator(SoundcloudDownloaderValidator):
+###############################################################################
+# Soundcloud albums and singles downloader + options
+
+
+class SoundcloudAlbumsAndSinglesDownloadOptions(SoundcloudDownloaderOptions):
     _required_keys = {"username"}
 
     def __init__(self, name, value):
         super().__init__(name, value)
         self.username = self._validate_key(key="username", validator=StringValidator)
+
+
+class SoundcloudAlbumsAndSinglesDownloader(
+    SoundcloudDownloader[SoundcloudAlbumsAndSinglesDownloadOptions]
+):
+    def download(self) -> List[SoundcloudTrack]:
+        """
+        Soundcloud subscription to download albums and tracks as singles.
+        """
+        tracks: List[SoundcloudTrack] = []
+
+        # Get the album info first. This tells us which track ids belong
+        # to an album. Unfortunately we cannot use download_archive or info.json for this
+        albums: List[SoundcloudAlbum] = self.download_albums(
+            artist_name=self.download_options.username.value
+        )
+
+        for album in albums:
+            tracks += album.album_tracks(
+                skip_premiere_tracks=self.download_options.skip_premiere_tracks.value
+            )
+
+        # only add tracks that are not part of an album
+        single_tracks = self.download_tracks(artist_name=self.download_options.username.value)
+        tracks += [
+            track for track in single_tracks if not any(album.contains(track) for album in albums)
+        ]
+
+        return tracks

ytdl_subscribe/downloaders/youtube_downloader.py

@@ -2,7 +2,11 @@ import json
 import os
 from abc import ABC
 from pathlib import Path
+from typing import Dict
+from typing import Generic
 from typing import List
+from typing import Optional
+from typing import TypeVar
 
 from yt_dlp.utils import RejectedVideoReached
 
@@ -12,14 +16,22 @@ from ytdl_subscribe.entries.youtube import YoutubeVideo
 from ytdl_subscribe.validators.date_range_validator import DownloadDateRangeSource
 from ytdl_subscribe.validators.validators import StringValidator
 
+###############################################################################
+# Abstract Youtube downloader + options
 
-class YoutubeDownloaderValidator(DownloaderValidator, ABC):
+
+class YoutubeDownloaderOptions(DownloaderValidator, ABC):
     """
     Abstract source validator for all soundcloud sources.
     """
 
 
-class YoutubeDownloader(Downloader):
+YoutubeDownloaderOptionsT = TypeVar("YoutubeDownloaderOptionsT", bound=YoutubeDownloaderOptions)
+
+
+class YoutubeDownloader(
+    Generic[YoutubeDownloaderOptionsT], Downloader[YoutubeDownloaderOptionsT, YoutubeVideo], ABC
+):
     """
     Class that handles downloading youtube entries via ytdl and converting them into
     YoutubeVideo objects
@@ -40,18 +52,22 @@ class YoutubeDownloader(Downloader):
         """Returns full channel url"""
         return f"https://youtube.com/channel/{channel_id}"
 
-    def _download_with_metadata(self, url: str) -> None:
+    def _download_with_metadata(
+        self, url: str, ytdl_options_overrides: Optional[Dict] = None
+    ) -> None:
         """
         Do not get entries from the extract info, let it write to the info.json file and load
         that instead. This is because if the video is already downloaded in a playlist, it will
         not fetch the metadata (maybe there is a way??)
         """
-        ytdl_metadata_override = {
+        ytdl_overrides = {
             "writeinfojson": True,
         }
+        if ytdl_options_overrides:
+            ytdl_overrides = dict(ytdl_overrides, **ytdl_options_overrides)
 
         try:
-            _ = self.extract_info(ytdl_options_overrides=ytdl_metadata_override, url=url)
+            _ = self.extract_info(ytdl_options_overrides=ytdl_overrides, url=url)
         except RejectedVideoReached:
             pass
 
@@ -79,11 +95,15 @@ class YoutubeDownloader(Downloader):
 
         return entries
 
-    def download_channel(self, channel_id: str) -> List[YoutubeVideo]:
+    def download_channel(
+        self, channel_id: str, ytdl_options_overrides: Optional[Dict] = None
+    ) -> List[YoutubeVideo]:
         """
         Downloads all videos from a channel
         """
-        self._download_with_metadata(url=self.channel_url(channel_id))
+        self._download_with_metadata(
+            url=self.channel_url(channel_id), ytdl_options_overrides=ytdl_options_overrides
+        )
 
         # Load the entries from info.json
         entries: List[YoutubeVideo] = []
@@ -98,7 +118,29 @@ class YoutubeDownloader(Downloader):
         return entries
 
 
-class YoutubePlaylistDownloaderValidator(YoutubeDownloaderValidator):
+###############################################################################
+# Youtube single video downloader + options
+
+
+class YoutubeVideoDownloaderOptions(YoutubeDownloaderOptions):
+    _required_keys = {"video_id"}
+
+    def __init__(self, name, value):
+        super().__init__(name, value)
+        self.video_id = self._validate_key("video_id", StringValidator)
+
+
+class YoutubeVideoDownloader(YoutubeDownloader[YoutubeVideoDownloaderOptions]):
+    def download(self) -> List[YoutubeVideo]:
+        video = self.download_video(video_id=self.download_options.video_id.value)
+        return [video]
+
+
+###############################################################################
+# Youtube playlist downloader + options
+
+
+class YoutubePlaylistDownloaderOptions(YoutubeDownloaderOptions):
     _required_keys = {"playlist_id"}
 
     def __init__(self, name, value):
@@ -106,19 +148,33 @@ class YoutubePlaylistDownloaderValidator(YoutubeDownloaderValidator):
         self.playlist_id = self._validate_key("playlist_id", StringValidator)
 
 
-class YoutubeChannelDownloaderValidator(YoutubeDownloaderValidator, DownloadDateRangeSource):
+class YoutubePlaylistDownloader(YoutubeDownloader[YoutubePlaylistDownloaderOptions]):
+    def download(self) -> List[YoutubeVideo]:
+        return self.download_playlist(playlist_id=self.download_options.playlist_id.value)
+
+
+###############################################################################
+# Youtube channel downloader + options
+
+
+class YoutubeChannelDownloaderOptions(YoutubeDownloaderOptions, DownloadDateRangeSource):
     _required_keys = {"channel_id"}
     _optional_keys = {"before", "after"}
 
     def __init__(self, name, value):
-        YoutubeDownloaderValidator.__init__(self, name, value)
+        YoutubeDownloaderOptions.__init__(self, name, value)
         DownloadDateRangeSource.__init__(self, name, value)
         self.channel_id = self._validate_key("channel_id", StringValidator)
 
 
-class YoutubeVideoDownloaderValidator(YoutubeDownloaderValidator):
+class YoutubeChannelDownloader(YoutubeDownloader[YoutubeChannelDownloaderOptions]):
-    _required_keys = {"video_id"}
+    def download(self) -> List[YoutubeVideo]:
+        ytdl_options_overrides = {}
+        source_date_range = self.download_options.get_date_range()
+        if source_date_range:
+            ytdl_options_overrides["daterange"] = source_date_range
 
-    def __init__(self, name, value):
+        return self.download_channel(
-        super().__init__(name, value)
+            channel_id=self.download_options.channel_id.value,
-        self.video_id = self._validate_key("video_id", StringValidator)
+            ytdl_options_overrides=ytdl_options_overrides,
+        )

ytdl_subscribe/plugins/plugin.py

@@ -14,8 +14,6 @@ class PluginValidator(StrictDictValidator):
     Class that defines the parameters to a plugin
     """
 
-    pass
-
 
 PluginValidatorT = TypeVar("PluginValidatorT", bound=PluginValidator)
 
@@ -62,10 +60,8 @@ class Plugin(Generic[PluginValidatorT], ABC):
         ----------
         entry: Entry to post process
         """
-        pass
 
     def post_process_subscription(self):
         """
         After all downloaded files have been post-processed, apply a subscription-wide post process
         """
-        pass

ytdl_subscribe/subscriptions/soundcloud.py

@@ -1,40 +0,0 @@
-from typing import List
-
-from ytdl_subscribe.downloaders.soundcloud_downloader import (
-    SoundcloudAlbumsAndSinglesSourceValidator,
-)
-from ytdl_subscribe.downloaders.soundcloud_downloader import SoundcloudDownloader
-from ytdl_subscribe.entries.soundcloud import SoundcloudAlbum
-from ytdl_subscribe.entries.soundcloud import SoundcloudTrack
-from ytdl_subscribe.subscriptions.subscription import Subscription
-
-
-class SoundcloudAlbumsAndSinglesSubscription(
-    Subscription[SoundcloudAlbumsAndSinglesSourceValidator, SoundcloudTrack]
-):
-    """
-    Soundcloud subscription to download albums and tracks as singles.
-    """
-
-    def _extract_info(self) -> List[SoundcloudTrack]:
-        tracks: List[SoundcloudTrack] = []
-        downloader = self.get_downloader(SoundcloudDownloader)
-
-        # Get the album info first. This tells us which track ids belong
-        # to an album. Unfortunately we cannot use download_archive or info.json for this
-        albums: List[SoundcloudAlbum] = downloader.download_albums(
-            artist_name=self.source_options.username.value
-        )
-
-        for album in albums:
-            tracks += album.album_tracks(
-                skip_premiere_tracks=self.source_options.skip_premiere_tracks.value
-            )
-
-        # only add tracks that are not part of an album
-        single_tracks = downloader.download_tracks(artist_name=self.source_options.username.value)
-        tracks += [
-            track for track in single_tracks if not any(album.contains(track) for album in albums)
-        ]
-
-        return tracks

ytdl_subscribe/subscriptions/youtube.py

@@ -1,46 +0,0 @@
-from typing import List
-
-from ytdl_subscribe.downloaders.youtube_downloader import YoutubeChannelDownloaderValidator
-from ytdl_subscribe.downloaders.youtube_downloader import YoutubeDownloader
-from ytdl_subscribe.downloaders.youtube_downloader import YoutubePlaylistDownloaderValidator
-from ytdl_subscribe.downloaders.youtube_downloader import YoutubeVideoDownloaderValidator
-from ytdl_subscribe.entries.youtube import YoutubeVideo
-from ytdl_subscribe.subscriptions.subscription import Subscription
-
-
-class YoutubePlaylistSubscription(Subscription[YoutubePlaylistDownloaderValidator, YoutubeVideo]):
-    """
-    Youtube subscription to download videos from a playlist
-    """
-
-    def _extract_info(self) -> List[YoutubeVideo]:
-        return self.get_downloader(YoutubeDownloader).download_playlist(
-            playlist_id=self.source_options.playlist_id.value
-        )
-
-
-class YoutubeChannelSubscription(Subscription[YoutubeChannelDownloaderValidator, YoutubeVideo]):
-    """
-    Youtube subscription to download videos from a channel
-    """
-
-    def _extract_info(self) -> List[YoutubeVideo]:
-        source_ytdl_options = {}
-        source_date_range = self.source_options.get_date_range()
-        if source_date_range:
-            source_ytdl_options["daterange"] = source_date_range
-
-        downloader = self.get_downloader(YoutubeDownloader, source_ytdl_options=source_ytdl_options)
-        return downloader.download_channel(channel_id=self.source_options.channel_id.value)
-
-
-class YoutubeVideoSubscription(Subscription[YoutubeVideoDownloaderValidator, YoutubeVideo]):
-    """
-    Youtube subscription to download a single video
-    """
-
-    def _extract_info(self) -> List[YoutubeVideo]:
-        entry = self.get_downloader(YoutubeDownloader).download_video(
-            video_id=self.source_options.video_id.value
-        )
-        return [entry]