[BACKEND] Make sibling metadata optional to compute (#855)

Partial fix to https://github.com/jmbannon/ytdl-sub/issues/853
Makes computing sibling metadata optional, defaulted to False. This prevents excessive memory usage when scraping large channels

src/ytdl_sub/downloaders/url/downloader.py

@@ -385,7 +385,9 @@ class MultiUrlDownloader(SourcePlugin[MultiUrlValidator]):
             ):
                 yield entry_child
 
-    def _download_url_metadata(self, url: str) -> Tuple[List[EntryParent], List[Entry]]:
+    def _download_url_metadata(
+        self, url: str, include_sibling_metadata: bool
+    ) -> Tuple[List[EntryParent], List[Entry]]:
         """
         Downloads only info.json files and forms EntryParent trees
         """
@@ -401,6 +403,7 @@ class MultiUrlDownloader(SourcePlugin[MultiUrlValidator]):
             url=url,
             entry_dicts=entry_dicts,
             working_directory=self.working_directory,
+            include_sibling_metadata=include_sibling_metadata,
         )
         orphans = EntryParent.from_entry_dicts_with_no_parents(
             parents=parents,
@@ -438,7 +441,9 @@ class MultiUrlDownloader(SourcePlugin[MultiUrlValidator]):
             if not (url := self.overrides.apply_formatter(collection_url.url)):
                 continue
 
-            parents, orphan_entries = self._download_url_metadata(url=url)
+            parents, orphan_entries = self._download_url_metadata(
+                url=url, include_sibling_metadata=collection_url.include_sibling_metadata
+            )
 
             # TODO: Encapsulate this logic into its own class
             self._url_state = URLDownloadState(

src/ytdl_sub/downloaders/url/validators.py

@@ -45,7 +45,13 @@ class UrlThumbnailListValidator(ListValidator[UrlThumbnailValidator]):
 
 class UrlValidator(StrictDictValidator):
     _required_keys = {"url"}
-    _optional_keys = {"variables", "source_thumbnails", "playlist_thumbnails", "download_reverse"}
+    _optional_keys = {
+        "variables",
+        "source_thumbnails",
+        "playlist_thumbnails",
+        "download_reverse",
+        "include_sibling_metadata",
+    }
 
     @classmethod
     def partial_validate(cls, name: str, value: Any) -> None:
@@ -74,6 +80,9 @@ class UrlValidator(StrictDictValidator):
         self._download_reverse = self._validate_key(
             key="download_reverse", validator=BoolValidator, default=True
         )
+        self._include_sibling_metadata = self._validate_key(
+            key="include_sibling_metadata", validator=BoolValidator, default=False
+        )
 
     @property
     def url(self) -> OverridesStringFormatterValidator:
@@ -147,6 +156,16 @@ class UrlValidator(StrictDictValidator):
         """
         return self._download_reverse.value
 
+    @property
+    def include_sibling_metadata(self) -> bool:
+        """
+        Optional. Whether to include sibling metadata as an entry variable, which comprises basic
+        metadata from all other entries (including itself) that belong to the same playlist. For
+        channels or large playlists, this becomes memory-intensive since you are storing
+        ``n^2`` metadata. Defaults to False.
+        """
+        return self._include_sibling_metadata.value
+
 
 class UrlStringOrDictValidator(UrlValidator):
     """

src/ytdl_sub/entries/entry_parent.py

@@ -62,13 +62,16 @@ class EntryParent(BaseEntry):
             )
         return sibling_entry_metadata
 
-    def _set_child_variables(self, parents: Optional[List["EntryParent"]] = None) -> "EntryParent":
+    def _set_child_variables(
+        self, include_sibling_metadata: bool, parents: Optional[List["EntryParent"]] = None
+    ) -> "EntryParent":
         if parents is None:
             parents = [self]
 
-        kwargs_to_add: Dict[str, Any] = {
+        kwargs_to_add: Dict[str, Any] = {}
-            v.sibling_metadata.metadata_key: self._sibling_entry_metadata()
+        if include_sibling_metadata:
-        }
+            kwargs_to_add[v.sibling_metadata.metadata_key] = self._sibling_entry_metadata()
+
         if len(parents) >= 1:
             kwargs_to_add[v.playlist_metadata.metadata_key] = parents[-1]._kwargs
         if len(parents) >= 2:
@@ -83,7 +86,9 @@ class EntryParent(BaseEntry):
             entry_child._kwargs = dict(entry_child._kwargs, **kwargs_to_add)
 
         for parent_child in self.parent_children():
-            parent_child._set_child_variables(parents=parents + [parent_child])
+            parent_child._set_child_variables(
+                include_sibling_metadata=include_sibling_metadata, parents=parents + [parent_child]
+            )
 
         return self
 
@@ -180,7 +185,11 @@ class EntryParent(BaseEntry):
 
     @classmethod
     def from_entry_dicts(
-        cls, url: str, entry_dicts: List[Dict], working_directory: str
+        cls,
+        url: str,
+        entry_dicts: List[Dict],
+        working_directory: str,
+        include_sibling_metadata: bool,
     ) -> List["EntryParent"]:
         """
         Reads all entry dicts and builds a tree of EntryParents
@@ -203,7 +212,7 @@ class EntryParent(BaseEntry):
             parents = [root_parent]
 
         for parent in parents:
-            parent._set_child_variables()
+            parent._set_child_variables(include_sibling_metadata=include_sibling_metadata)
 
         return parents
 

src/ytdl_sub/prebuilt_presets/music/music.yaml

@@ -67,12 +67,17 @@ presets:
       - "_music_base"
 
     download:
-      - "{url}"
+      - url: "{url}"
+        include_sibling_metadata: False
 
 
   _albums_from_playlists:
     preset:
-      - "Single"
+      - "_music_base"
+
+    download:
+      - url: "{url}"
+        include_sibling_metadata: True
 
     overrides:
       track_album: "{playlist_title}"
@@ -107,6 +112,7 @@ presets:
         # The first URL will be all the artist's tracks.
         # Treat these as singles - an album with a single track
         - url: "{url}/tracks"
+          include_sibling_metadata: False
           variables:
             sc_track_album: "{title}"
             sc_track_number: "1"
@@ -117,6 +123,7 @@ presets:
         # to an album and tracks (in the URL above), it will resolve to this
         # URL and include the album metadata we set below.
         - url: "{url}/albums"
+          include_sibling_metadata: True
           variables:
             sc_track_album: "{playlist_title}"
             sc_track_number: "{playlist_index}"

tests/unit/config/test_config_file.py

@@ -73,8 +73,8 @@ class TestConfigFilePartiallyValidatesPresets:
             preset_dict={"download": {"bad_key": "nope"}},
             expected_error_message="Validation error in partial_preset.download.1: "
             "'partial_preset.download.1' contains the field 'bad_key' which is not allowed. "
-            "Allowed fields: download_reverse, playlist_thumbnails, source_thumbnails, url, "
+            "Allowed fields: download_reverse, include_sibling_metadata, playlist_thumbnails, "
-            "variables",
+            "source_thumbnails, url, variables",
         )
 
     @pytest.mark.parametrize(