[BACKEND] generic.source download strategy (#252)

src/ytdl_sub/config/preset_class_mappings.py

@@ -4,6 +4,7 @@ from typing import Type
 
 from ytdl_sub.downloaders.downloader import Downloader
 from ytdl_sub.downloaders.generic.collection import CollectionDownloader
+from ytdl_sub.downloaders.generic.source import SourceDownloader
 from ytdl_sub.downloaders.soundcloud.albums_and_singles import SoundcloudAlbumsAndSinglesDownloader
 from ytdl_sub.downloaders.youtube.channel import YoutubeChannelDownloader
 from ytdl_sub.downloaders.youtube.merge_playlist import YoutubeMergePlaylistDownloader
@@ -39,6 +40,7 @@ class DownloadStrategyMapping:
         },
         "generic": {
             "collection": CollectionDownloader,
+            "source": SourceDownloader,
         },
     }
 

src/ytdl_sub/downloaders/downloader.py

@@ -415,10 +415,12 @@ class Downloader(DownloadArchiver, Generic[DownloaderOptionsT], ABC):
             for child in entry.parent_children():
                 self._set_collection_variables(collection_url, child)
             for child in entry.entry_children():
-                child.add_variables(variables_to_add=collection_url.variables)
+                child.add_variables(
+                    variables_to_add=collection_url.variables.dict_with_format_strings
+                )
 
         elif isinstance(entry, Entry):
-            entry.add_variables(variables_to_add=collection_url.variables)
+            entry.add_variables(variables_to_add=collection_url.variables.dict_with_format_strings)
 
     def _download_url_metadata(
         self, collection_url: CollectionUrlValidator

src/ytdl_sub/downloaders/generic/collection.py

@@ -18,20 +18,24 @@ class CollectionDownloadOptions(CollectionValidator, DownloaderValidator):
             # required
             download_strategy: "collection"
             urls:
-            - url: "soundcloud.com/tracks"
+              - url: "youtube.com/channel/UCsvn_Po0SmunchJYtttWpOxMg"
-              variables:
+                variables:
-                season: "1"
+                  season_index: "1"
-                album: "{title}"
+                  season_name: "Uploads"
-              playlist_thumbnails:
+                playlist_thumbnails:
-                - name:
+                  - name: "poster.jpg"
-                  uid: "square" / "largest" / "last entry" / actual name
+                    uid: "avatar_uncropped"
-              source_thumbnail:
+                  - name: "fanart.jpg"
-                - path:
+                    uid: "banner_uncropped"
-                  type: "
+                  - name: "season{season_index}-poster.jpg"
-            - url: "soundcloud.com/albums"
+                    uid: "latest_entry"
-              variables:
+              - url: "https://www.youtube.com/playlist?list=UCsvn_Po0SmunchJYtttWpOxMg"
-                season: "1"
+                variables:
-                album: "{playlist_title}"
+                  season_index: "2"
+                  season_name: "Playlist as Season"
+                playlist_thumbnails:
+                  - name: "season{season_index}-poster.jpg"
+                    uid: "latest_entry"
     """
 
     @property

src/ytdl_sub/downloaders/generic/collection_validator.py

@@ -48,8 +48,9 @@ class CollectionUrlValidator(StrictDictValidator):
 
         # TODO: url validate using yt-dlp IE
         self._url = self._validate_key(key="url", validator=StringValidator)
-        variables = self._validate_key_if_present(key="variables", validator=DictFormatterValidator)
+        self._variables = self._validate_key_if_present(
-        self._variables = variables.dict_with_format_strings if variables else {}
+            key="variables", validator=DictFormatterValidator, default={}
+        )
 
         self._source_thumbnails = self._validate_key_if_present(
             key="source_thumbnails", validator=CollectionThumbnailListValidator, default=[]
@@ -61,30 +62,55 @@ class CollectionUrlValidator(StrictDictValidator):
     @property
     def url(self) -> str:
         """
-        Returns
+        URL to download from, listed in priority from lowest (top) to highest (bottom). If a
-        -------
+        download exists in more than one URL, it will resolve to the bottom-most one and inherit
-        URL to download from
+        those variables.
         """
         return self._url.value
 
     @property
-    def variables(self) -> Dict[str, str]:
+    def variables(self) -> DictFormatterValidator:
         """
-        Variables to add to each entry
+        Source variables to add to each entry. The top-most collection must define all possible
+        variables. Collections below can redefine all of them or a subset of the top-most variables.
         """
         return self._variables
 
     @property
     def source_thumbnails(self) -> Optional[CollectionThumbnailListValidator]:
         """
-        TODO:docstring
+        Thumbnails to download from the source, if any exist. The hierarchy is defined as
+        source -> playlist -> entry.
+
+        Usage:
+
+        .. code-block:: yaml
+
+           source_thumbnails:
+             - name: "poster.jpg"
+               uid: "avatar_uncropped"
+
+        UID is the yt-dlp thumbnail ID or can specify ``latest_entry`` to use the last entry's
+        thumbnail that was downloaded.
         """
         return self._source_thumbnails
 
     @property
     def playlist_thumbnails(self) -> Optional[CollectionThumbnailListValidator]:
         """
-        TODO:docstring
+        Thumbnails to download from the source, if any exist. The hierarchy is defined as
+        source -> playlist -> entry.
+
+        Usage:
+
+        .. code-block:: yaml
+
+           playlist_thumbnails:
+             - name: "poster.jpg"
+               uid: "avatar_uncropped"
+
+        UID is the yt-dlp thumbnail ID or can specify ``latest_entry`` to use the last entry's
+        thumbnail that was downloaded.
         """
         return self._playlist_thumbnails
 
@@ -96,10 +122,10 @@ class CollectionUrlListValidator(ListValidator[CollectionUrlValidator]):
     def __init__(self, name, value):
         super().__init__(name, value)
 
-        added_variables: Dict[str, str] = self.list[0].variables
+        added_variables: Dict[str, str] = self.list[0].variables.dict_with_format_strings
 
         for idx, collection_url_validator in enumerate(self.list[1:]):
-            collection_variables = collection_url_validator.variables
+            collection_variables = collection_url_validator.variables.dict_with_format_strings
 
             # see if this collection contains new added vars (it should not)
             for var in collection_variables.keys():
@@ -112,7 +138,7 @@ class CollectionUrlListValidator(ListValidator[CollectionUrlValidator]):
 
             # see if this collection is missing any added vars (if so, inherit from the top)
             for var in added_variables.keys():
-                if var not in collection_variables:
+                if var not in collection_variables.keys():
                     collection_variables[var] = added_variables[var]
 
 
@@ -141,7 +167,7 @@ class CollectionValidator(StrictDictValidator, AddsVariablesMixin):
         -------
         List of variables added. The first collection url always contains all the variables.
         """
-        return list(self._urls.list[0].variables.keys())
+        return list(self._urls.list[0].variables.keys)
 
     def validate_with_variables(
         self, source_variables: List[str], override_variables: Dict[str, str]
@@ -162,7 +188,10 @@ class CollectionValidator(StrictDictValidator, AddsVariablesMixin):
 
         # Apply formatting to each new source variable, ensure it resolves
         for collection_url in self.collection_urls.list:
-            for source_var_name, source_var_formatter_str in collection_url.variables.items():
+            for (
+                source_var_name,
+                source_var_formatter_str,
+            ) in collection_url.variables.dict_with_format_strings.items():
                 _ = StringFormatterValidator(
                     name=f"{self._name}.{source_var_name}", value=source_var_formatter_str
                 ).apply_formatter(base_variables)

src/ytdl_sub/downloaders/generic/source.py

@@ -0,0 +1,39 @@
+from ytdl_sub.downloaders.downloader import Downloader
+from ytdl_sub.downloaders.downloader import DownloaderValidator
+from ytdl_sub.downloaders.generic.collection_validator import CollectionUrlValidator
+from ytdl_sub.downloaders.generic.collection_validator import CollectionValidator
+
+
+class SourceDownloadOptions(CollectionUrlValidator, DownloaderValidator):
+    """
+    Downloads from a single URL supported by yt-dlp.
+
+    Usage:
+
+    .. code-block:: yaml
+
+      presets:
+        my_example_preset:
+          generic:
+            # required
+            download_strategy: "source"
+            url: "youtube.com/channel/UCsvn_Po0SmunchJYtttWpOxMg"
+            # optional
+            playlist_thumbnails:
+              - name: "poster.jpg"
+                uid: "avatar_uncropped"
+              - name: "fanart.jpg"
+                uid: "banner_uncropped"
+    """
+
+    @property
+    def collection_validator(self) -> CollectionValidator:
+        """Returns itself!"""
+        return CollectionValidator(
+            name=self._name,
+            value={"urls": [self._value]},
+        )
+
+
+class SourceDownloader(Downloader[SourceDownloadOptions]):
+    downloader_options_type = SourceDownloadOptions

src/ytdl_sub/downloaders/youtube/video.py

@@ -3,6 +3,7 @@ from typing import Dict
 from ytdl_sub.downloaders.downloader import Downloader
 from ytdl_sub.downloaders.downloader import DownloaderValidator
 from ytdl_sub.downloaders.generic.collection_validator import CollectionValidator
+from ytdl_sub.downloaders.generic.source import SourceDownloadOptions
 from ytdl_sub.validators.url_validator import YoutubeVideoUrlValidator
 
 
@@ -38,10 +39,9 @@ class YoutubeVideoDownloaderOptions(DownloaderValidator):
     @property
     def collection_validator(self) -> CollectionValidator:
         """Downloads the video url"""
-        return CollectionValidator(
+        return SourceDownloadOptions(
-            name=self._name,
+            name=self._name, value={"url": self.video_url}
-            value={"urls": [{"url": self.video_url, "variables": {"playlist_size": "1"}}]},
+        ).collection_validator
-        )
 
     @property
     def video_url(self) -> str: