[REFACTOR] Move downloaders to separate files (#91)

docs/config.rst

@@ -45,13 +45,13 @@ for available source variables to use.
 
 channel
 _______
-.. autoclass:: ytdl_sub.downloaders.youtube_downloader.YoutubeChannelDownloaderOptions()
+.. autoclass:: ytdl_sub.downloaders.youtube.channel.YoutubeChannelDownloaderOptions()
   :members:
   :member-order: bysource
   :inherited-members:
   :exclude-members: get_date_range
 
-.. autofunction:: ytdl_sub.downloaders.youtube_downloader.YoutubeChannelDownloader.ytdl_option_defaults()
+.. autofunction:: ytdl_sub.downloaders.youtube.channel.YoutubeChannelDownloader.ytdl_option_defaults()
 
 -------------------------------------------------------------------------------
 
@@ -59,23 +59,23 @@ _______
 
 playlist
 ________
-.. autoclass:: ytdl_sub.downloaders.youtube_downloader.YoutubePlaylistDownloaderOptions()
+.. autoclass:: ytdl_sub.downloaders.youtube.playlist.YoutubePlaylistDownloaderOptions()
   :members:
   :member-order: bysource
   :inherited-members:
 
-.. autofunction:: ytdl_sub.downloaders.youtube_downloader.YoutubePlaylistDownloader.ytdl_option_defaults()
+.. autofunction:: ytdl_sub.downloaders.youtube.playlist.YoutubePlaylistDownloader.ytdl_option_defaults()
 
 -------------------------------------------------------------------------------
 
 video
 _____
-.. autoclass:: ytdl_sub.downloaders.youtube_downloader.YoutubeVideoDownloaderOptions()
+.. autoclass:: ytdl_sub.downloaders.youtube.video.YoutubeVideoDownloaderOptions()
   :members:
   :member-order: bysource
   :inherited-members:
 
-.. autofunction:: ytdl_sub.downloaders.youtube_downloader.YoutubeVideoDownloader.ytdl_option_defaults()
+.. autofunction:: ytdl_sub.downloaders.youtube.video.YoutubeVideoDownloader.ytdl_option_defaults()
 
 -------------------------------------------------------------------------------
 
@@ -109,12 +109,12 @@ for available source variables to use.
 
 albums_and_singles
 __________________
-.. autoclass:: ytdl_sub.downloaders.soundcloud_downloader.SoundcloudAlbumsAndSinglesDownloadOptions()
+.. autoclass:: ytdl_sub.downloaders.soundcloud.albums_and_singles.SoundcloudAlbumsAndSinglesDownloadOptions()
   :members:
   :member-order: bysource
   :inherited-members:
 
-.. autofunction:: ytdl_sub.downloaders.soundcloud_downloader.SoundcloudAlbumsAndSinglesDownloader.ytdl_option_defaults()
+.. autofunction:: ytdl_sub.downloaders.soundcloud.albums_and_singles.SoundcloudAlbumsAndSinglesDownloader.ytdl_option_defaults()
 
 -------------------------------------------------------------------------------
 

src/ytdl_sub/config/preset_class_mappings.py

@@ -3,12 +3,12 @@ from typing import List
 from typing import Type
 
 from ytdl_sub.downloaders.downloader import Downloader
-from ytdl_sub.downloaders.soundcloud_downloader import SoundcloudAlbumsAndSinglesDownloader
+from ytdl_sub.downloaders.soundcloud.albums_and_singles import SoundcloudAlbumsAndSinglesDownloader
+from ytdl_sub.downloaders.youtube.channel import YoutubeChannelDownloader
 from ytdl_sub.downloaders.youtube.merge_playlist import YoutubeMergePlaylistDownloader
+from ytdl_sub.downloaders.youtube.playlist import YoutubePlaylistDownloader
 from ytdl_sub.downloaders.youtube.split_video import YoutubeSplitVideoDownloader
-from ytdl_sub.downloaders.youtube_downloader import YoutubeChannelDownloader
+from ytdl_sub.downloaders.youtube.video import YoutubeVideoDownloader
-from ytdl_sub.downloaders.youtube_downloader import YoutubePlaylistDownloader
-from ytdl_sub.downloaders.youtube_downloader import YoutubeVideoDownloader
 from ytdl_sub.plugins.music_tags import MusicTagsPlugin
 from ytdl_sub.plugins.nfo_tags import NfoTagsPlugin
 from ytdl_sub.plugins.output_directory_nfo_tags import OutputDirectoryNfoTagsPlugin

src/ytdl_sub/downloaders/soundcloud/abc.py

@@ -0,0 +1,65 @@
+from abc import ABC
+from typing import Generic
+from typing import TypeVar
+
+from ytdl_sub.downloaders.downloader import Downloader
+from ytdl_sub.downloaders.downloader import DownloaderValidator
+from ytdl_sub.entries.soundcloud import SoundcloudTrack
+from ytdl_sub.validators.validators import BoolValidator
+
+
+class SoundcloudDownloaderOptions(DownloaderValidator, ABC):
+    """
+    Abstract source validator for all soundcloud sources.
+    """
+
+    _optional_keys = {"skip_premiere_tracks"}
+
+    def __init__(self, name: str, value: dict):
+        super().__init__(name=name, value=value)
+        self._skip_premiere_tracks = self._validate_key(
+            "skip_premiere_tracks", BoolValidator, default=True
+        )
+
+    @property
+    def skip_premiere_tracks(self) -> bool:
+        """
+        Optional. True to skip tracks that require purchasing. False otherwise. Defaults to True.
+        """
+        return self._skip_premiere_tracks.value
+
+
+SoundcloudDownloaderOptionsT = TypeVar(
+    "SoundcloudDownloaderOptionsT", bound=SoundcloudDownloaderOptions
+)
+
+
+class SoundcloudDownloader(
+    Downloader[SoundcloudDownloaderOptionsT, SoundcloudTrack],
+    Generic[SoundcloudDownloaderOptionsT],
+    ABC,
+):
+    """
+    Class that handles downloading soundcloud entries via ytdl and converting them into
+    SoundcloudTrack / SoundcloudAlbumTrack objects
+    """
+
+    downloader_entry_type = SoundcloudTrack
+
+    @classmethod
+    def artist_albums_url(cls, artist_url: str) -> str:
+        """
+        Returns
+        -------
+        Full artist album url
+        """
+        return f"{artist_url}/albums"
+
+    @classmethod
+    def artist_tracks_url(cls, artist_url: str) -> str:
+        """
+        Returns
+        -------
+        Full artist tracks url
+        """
+        return f"{artist_url}/tracks"

src/ytdl_sub/downloaders/soundcloud/albums_and_singles.py

@@ -1,79 +1,11 @@
-from abc import ABC
 from typing import Dict
-from typing import Generic
 from typing import List
-from typing import TypeVar
 
-from ytdl_sub.downloaders.downloader import Downloader
+from ytdl_sub.downloaders.soundcloud.abc import SoundcloudDownloader
-from ytdl_sub.downloaders.downloader import DownloaderValidator
+from ytdl_sub.downloaders.soundcloud.abc import SoundcloudDownloaderOptions
 from ytdl_sub.entries.soundcloud import SoundcloudAlbum
 from ytdl_sub.entries.soundcloud import SoundcloudTrack
 from ytdl_sub.validators.url_validator import SoundcloudUsernameUrlValidator
-from ytdl_sub.validators.validators import BoolValidator
-
-###############################################################################
-# Abstract Soundcloud downloader + options
-
-
-class SoundcloudDownloaderOptions(DownloaderValidator, ABC):
-    """
-    Abstract source validator for all soundcloud sources.
-    """
-
-    _optional_keys = {"skip_premiere_tracks"}
-
-    def __init__(self, name: str, value: dict):
-        super().__init__(name=name, value=value)
-        self._skip_premiere_tracks = self._validate_key(
-            "skip_premiere_tracks", BoolValidator, default=True
-        )
-
-    @property
-    def skip_premiere_tracks(self) -> bool:
-        """
-        Optional. True to skip tracks that require purchasing. False otherwise. Defaults to True.
-        """
-        return self._skip_premiere_tracks.value
-
-
-SoundcloudDownloaderOptionsT = TypeVar(
-    "SoundcloudDownloaderOptionsT", bound=SoundcloudDownloaderOptions
-)
-
-
-class SoundcloudDownloader(
-    Downloader[SoundcloudDownloaderOptionsT, SoundcloudTrack],
-    Generic[SoundcloudDownloaderOptionsT],
-    ABC,
-):
-    """
-    Class that handles downloading soundcloud entries via ytdl and converting them into
-    SoundcloudTrack / SoundcloudAlbumTrack objects
-    """
-
-    downloader_entry_type = SoundcloudTrack
-
-    @classmethod
-    def artist_albums_url(cls, artist_url: str) -> str:
-        """
-        Returns
-        -------
-        Full artist album url
-        """
-        return f"{artist_url}/albums"
-
-    @classmethod
-    def artist_tracks_url(cls, artist_url: str) -> str:
-        """
-        Returns
-        -------
-        Full artist tracks url
-        """
-        return f"{artist_url}/tracks"
-
-
-###############################################################################
-# Soundcloud albums and singles downloader + options
 
 
 class SoundcloudAlbumsAndSinglesDownloadOptions(SoundcloudDownloaderOptions):

src/ytdl_sub/downloaders/youtube/abc.py

@@ -0,0 +1,29 @@
+from abc import ABC
+from typing import Generic
+from typing import TypeVar
+
+from ytdl_sub.downloaders.downloader import Downloader
+from ytdl_sub.downloaders.downloader import DownloaderValidator
+from ytdl_sub.entries.youtube import YoutubeVideo
+
+
+class YoutubeDownloaderOptions(DownloaderValidator, ABC):
+    """
+    Abstract source validator for all soundcloud sources.
+    """
+
+
+YoutubeDownloaderOptionsT = TypeVar("YoutubeDownloaderOptionsT", bound=YoutubeDownloaderOptions)
+YoutubeVideoT = TypeVar("YoutubeVideoT", bound=YoutubeVideo)
+
+
+class YoutubeDownloader(
+    Downloader[YoutubeDownloaderOptionsT, YoutubeVideoT],
+    Generic[YoutubeDownloaderOptionsT, YoutubeVideoT],
+    ABC,
+):
+    """
+    Class that handles downloading youtube entries via ytdl and converting them into
+    YoutubeVideo like objects. Reserved for any future logic that is shared amongst all YT
+    downloaders.
+    """

src/ytdl_sub/downloaders/youtube/channel.py

@@ -1,205 +1,23 @@
-from abc import ABC
 from pathlib import Path
 from typing import Dict
-from typing import Generic
 from typing import List
 from typing import Optional
-from typing import TypeVar
 
 from ytdl_sub.config.preset_options import Overrides
-from ytdl_sub.downloaders.downloader import Downloader
 from ytdl_sub.downloaders.downloader import DownloaderOptionsT
-from ytdl_sub.downloaders.downloader import DownloaderValidator
+from ytdl_sub.downloaders.youtube.abc import YoutubeDownloader
+from ytdl_sub.downloaders.youtube.abc import YoutubeDownloaderOptions
 from ytdl_sub.entries.youtube import YoutubeChannel
-from ytdl_sub.entries.youtube import YoutubePlaylistVideo
 from ytdl_sub.entries.youtube import YoutubeVideo
 from ytdl_sub.utils.logger import Logger
 from ytdl_sub.utils.thumbnail import convert_url_thumbnail
 from ytdl_sub.validators.date_range_validator import DateRangeValidator
 from ytdl_sub.validators.string_formatter_validators import OverridesStringFormatterValidator
 from ytdl_sub.validators.url_validator import YoutubeChannelUrlValidator
-from ytdl_sub.validators.url_validator import YoutubePlaylistUrlValidator
-from ytdl_sub.validators.url_validator import YoutubeVideoUrlValidator
 from ytdl_sub.ytdl_additions.enhanced_download_archive import EnhancedDownloadArchive
 
 logger = Logger.get()
 
-###############################################################################
-# Abstract Youtube downloader + options
-
-
-class YoutubeDownloaderOptions(DownloaderValidator, ABC):
-    """
-    Abstract source validator for all soundcloud sources.
-    """
-
-
-YoutubeDownloaderOptionsT = TypeVar("YoutubeDownloaderOptionsT", bound=YoutubeDownloaderOptions)
-YoutubeVideoT = TypeVar("YoutubeVideoT", bound=YoutubeVideo)
-
-
-class YoutubeDownloader(
-    Downloader[YoutubeDownloaderOptionsT, YoutubeVideoT],
-    Generic[YoutubeDownloaderOptionsT, YoutubeVideoT],
-    ABC,
-):
-    """
-    Class that handles downloading youtube entries via ytdl and converting them into
-    YoutubeVideo like objects. Reserved for any future logic that is shared amongst all YT
-    downloaders.
-    """
-
-
-###############################################################################
-# Youtube single video downloader + options
-
-
-class YoutubeVideoDownloaderOptions(YoutubeDownloaderOptions):
-    """
-    Downloads a single youtube video. This download strategy is intended for CLI usage performing
-    a one-time download of a video, not a subscription.
-
-    Usage:
-
-    .. code-block:: yaml
-
-      presets:
-        example_preset:
-          youtube:
-            # required
-            download_strategy: "video"
-            video_url: "youtube.com/watch?v=VMAPTo7RVDo"
-
-    CLI usage:
-
-    .. code-block:: bash
-
-       ytdl-sub dl --preset "example_preset" --youtube.video_url "youtube.com/watch?v=VMAPTo7RVDo"
-
-    """
-
-    _required_keys = {"video_url"}
-
-    def __init__(self, name, value):
-        super().__init__(name, value)
-        self._video_url = self._validate_key("video_url", YoutubeVideoUrlValidator).video_url
-
-    @property
-    def video_url(self) -> str:
-        """
-        Required. The url of the video, i.e. ``youtube.com/watch?v=VMAPTo7RVDo``.
-        """
-        return self._video_url
-
-
-class YoutubeVideoDownloader(YoutubeDownloader[YoutubeVideoDownloaderOptions, YoutubeVideo]):
-    downloader_options_type = YoutubeVideoDownloaderOptions
-    downloader_entry_type = YoutubeVideo
-
-    @classmethod
-    def ytdl_option_defaults(cls) -> Dict:
-        """
-        Default `ytdl_options`_ for ``video``
-
-        .. code-block:: yaml
-
-           ytdl_options:
-             ignoreerrors: True  # ignore errors like hidden videos, age restriction, etc
-        """
-        return dict(
-            super().ytdl_option_defaults(),
-            **{"break_on_existing": True},
-        )
-
-    def download(self) -> List[YoutubeVideo]:
-        """Download a single Youtube video"""
-        entry_dict = self.extract_info(url=self.download_options.video_url)
-        return [YoutubeVideo(entry_dict=entry_dict, working_directory=self.working_directory)]
-
-
-###############################################################################
-# Youtube playlist downloader + options
-
-
-class YoutubePlaylistDownloaderOptions(YoutubeDownloaderOptions):
-    """
-    Downloads all videos from a youtube playlist.
-
-    Usage:
-
-    .. code-block:: yaml
-
-      presets:
-        my_example_preset:
-          youtube:
-            # required
-            download_strategy: "playlist"
-            playlist_url: "https://www.youtube.com/playlist?list=UCsvn_Po0SmunchJYtttWpOxMg"
-    """
-
-    _required_keys = {"playlist_url"}
-
-    def __init__(self, name, value):
-        super().__init__(name, value)
-        self._playlist_url = self._validate_key(
-            "playlist_url", YoutubePlaylistUrlValidator
-        ).playlist_url
-
-    @property
-    def playlist_url(self) -> str:
-        """
-        Required. The playlist's url, i.e.
-        ``https://www.youtube.com/playlist?list=UCsvn_Po0SmunchJYtttWpOxMg``.
-        """
-        return self._playlist_url
-
-
-class YoutubePlaylistDownloader(
-    YoutubeDownloader[YoutubePlaylistDownloaderOptions, YoutubePlaylistVideo]
-):
-    downloader_options_type = YoutubePlaylistDownloaderOptions
-    downloader_entry_type = YoutubePlaylistVideo
-
-    # pylint: disable=line-too-long
-    @classmethod
-    def ytdl_option_defaults(cls) -> Dict:
-        """
-        Default `ytdl_options`_ for ``playlist``
-
-        .. code-block:: yaml
-
-           ytdl_options:
-             ignoreerrors: True  # ignore errors like hidden videos, age restriction, etc
-             break_on_existing: True  # stop downloads (newest to oldest) if a video is already downloaded
-        """
-        return dict(
-            super().ytdl_option_defaults(),
-            **{"break_on_existing": True},
-        )
-
-    # pylint: enable=line-too-long
-
-    def download(self) -> List[YoutubePlaylistVideo]:
-        """
-        Downloads all videos in a Youtube playlist
-        """
-        playlist_videos: List[YoutubePlaylistVideo] = []
-
-        entry_dicts = self.extract_info_via_info_json(url=self.download_options.playlist_url)
-        for entry_dict in entry_dicts:
-            if entry_dict.get("extractor") == "youtube":
-                playlist_videos.append(
-                    YoutubePlaylistVideo(
-                        entry_dict=entry_dict, working_directory=self.working_directory
-                    )
-                )
-
-        return playlist_videos
-
-
-###############################################################################
-# Youtube channel downloader + options
-
 
 class YoutubeChannelDownloaderOptions(YoutubeDownloaderOptions, DateRangeValidator):
     """

src/ytdl_sub/downloaders/youtube/merge_playlist.py

@@ -2,8 +2,8 @@ from typing import Dict
 from typing import List
 from typing import Optional
 
-from ytdl_sub.downloaders.youtube_downloader import YoutubeDownloader
+from ytdl_sub.downloaders.youtube.abc import YoutubeDownloader
-from ytdl_sub.downloaders.youtube_downloader import YoutubePlaylistDownloaderOptions
+from ytdl_sub.downloaders.youtube.playlist import YoutubePlaylistDownloaderOptions
 from ytdl_sub.entries.youtube import YoutubeVideo
 from ytdl_sub.utils.chapters import Chapters
 from ytdl_sub.utils.chapters import Timestamp

src/ytdl_sub/downloaders/youtube/playlist.py

@@ -0,0 +1,83 @@
+from typing import Dict
+from typing import List
+
+from ytdl_sub.downloaders.youtube.abc import YoutubeDownloader
+from ytdl_sub.downloaders.youtube.abc import YoutubeDownloaderOptions
+from ytdl_sub.entries.youtube import YoutubePlaylistVideo
+from ytdl_sub.validators.url_validator import YoutubePlaylistUrlValidator
+
+
+class YoutubePlaylistDownloaderOptions(YoutubeDownloaderOptions):
+    """
+    Downloads all videos from a youtube playlist.
+
+    Usage:
+
+    .. code-block:: yaml
+
+      presets:
+        my_example_preset:
+          youtube:
+            # required
+            download_strategy: "playlist"
+            playlist_url: "https://www.youtube.com/playlist?list=UCsvn_Po0SmunchJYtttWpOxMg"
+    """
+
+    _required_keys = {"playlist_url"}
+
+    def __init__(self, name, value):
+        super().__init__(name, value)
+        self._playlist_url = self._validate_key(
+            "playlist_url", YoutubePlaylistUrlValidator
+        ).playlist_url
+
+    @property
+    def playlist_url(self) -> str:
+        """
+        Required. The playlist's url, i.e.
+        ``https://www.youtube.com/playlist?list=UCsvn_Po0SmunchJYtttWpOxMg``.
+        """
+        return self._playlist_url
+
+
+class YoutubePlaylistDownloader(
+    YoutubeDownloader[YoutubePlaylistDownloaderOptions, YoutubePlaylistVideo]
+):
+    downloader_options_type = YoutubePlaylistDownloaderOptions
+    downloader_entry_type = YoutubePlaylistVideo
+
+    # pylint: disable=line-too-long
+    @classmethod
+    def ytdl_option_defaults(cls) -> Dict:
+        """
+        Default `ytdl_options`_ for ``playlist``
+
+        .. code-block:: yaml
+
+           ytdl_options:
+             ignoreerrors: True  # ignore errors like hidden videos, age restriction, etc
+             break_on_existing: True  # stop downloads (newest to oldest) if a video is already downloaded
+        """
+        return dict(
+            super().ytdl_option_defaults(),
+            **{"break_on_existing": True},
+        )
+
+    # pylint: enable=line-too-long
+
+    def download(self) -> List[YoutubePlaylistVideo]:
+        """
+        Downloads all videos in a Youtube playlist
+        """
+        playlist_videos: List[YoutubePlaylistVideo] = []
+
+        entry_dicts = self.extract_info_via_info_json(url=self.download_options.playlist_url)
+        for entry_dict in entry_dicts:
+            if entry_dict.get("extractor") == "youtube":
+                playlist_videos.append(
+                    YoutubePlaylistVideo(
+                        entry_dict=entry_dict, working_directory=self.working_directory
+                    )
+                )
+
+        return playlist_videos

src/ytdl_sub/downloaders/youtube/split_video.py

@@ -4,8 +4,8 @@ from pathlib import Path
 from typing import Dict
 from typing import List
 
-from ytdl_sub.downloaders.youtube_downloader import YoutubeDownloader
+from ytdl_sub.downloaders.youtube.abc import YoutubeDownloader
-from ytdl_sub.downloaders.youtube_downloader import YoutubeVideoDownloaderOptions
+from ytdl_sub.downloaders.youtube.video import YoutubeVideoDownloaderOptions
 from ytdl_sub.entries.youtube import YoutubePlaylistVideo
 from ytdl_sub.entries.youtube import YoutubeVideo
 from ytdl_sub.utils.chapters import Chapters

src/ytdl_sub/downloaders/youtube/video.py

@@ -0,0 +1,70 @@
+from typing import Dict
+from typing import List
+
+from ytdl_sub.downloaders.youtube.abc import YoutubeDownloader
+from ytdl_sub.downloaders.youtube.abc import YoutubeDownloaderOptions
+from ytdl_sub.entries.youtube import YoutubeVideo
+from ytdl_sub.validators.url_validator import YoutubeVideoUrlValidator
+
+
+class YoutubeVideoDownloaderOptions(YoutubeDownloaderOptions):
+    """
+    Downloads a single youtube video. This download strategy is intended for CLI usage performing
+    a one-time download of a video, not a subscription.
+
+    Usage:
+
+    .. code-block:: yaml
+
+      presets:
+        example_preset:
+          youtube:
+            # required
+            download_strategy: "video"
+            video_url: "youtube.com/watch?v=VMAPTo7RVDo"
+
+    CLI usage:
+
+    .. code-block:: bash
+
+       ytdl-sub dl --preset "example_preset" --youtube.video_url "youtube.com/watch?v=VMAPTo7RVDo"
+
+    """
+
+    _required_keys = {"video_url"}
+
+    def __init__(self, name, value):
+        super().__init__(name, value)
+        self._video_url = self._validate_key("video_url", YoutubeVideoUrlValidator).video_url
+
+    @property
+    def video_url(self) -> str:
+        """
+        Required. The url of the video, i.e. ``youtube.com/watch?v=VMAPTo7RVDo``.
+        """
+        return self._video_url
+
+
+class YoutubeVideoDownloader(YoutubeDownloader[YoutubeVideoDownloaderOptions, YoutubeVideo]):
+    downloader_options_type = YoutubeVideoDownloaderOptions
+    downloader_entry_type = YoutubeVideo
+
+    @classmethod
+    def ytdl_option_defaults(cls) -> Dict:
+        """
+        Default `ytdl_options`_ for ``video``
+
+        .. code-block:: yaml
+
+           ytdl_options:
+             ignoreerrors: True  # ignore errors like hidden videos, age restriction, etc
+        """
+        return dict(
+            super().ytdl_option_defaults(),
+            **{"break_on_existing": True},
+        )
+
+    def download(self) -> List[YoutubeVideo]:
+        """Download a single Youtube video"""
+        entry_dict = self.extract_info(url=self.download_options.video_url)
+        return [YoutubeVideo(entry_dict=entry_dict, working_directory=self.working_directory)]