From 02d51d6aec516e8c590090427cdb3cab7e4f06c7 Mon Sep 17 00:00:00 2001 From: Jesse Bannon Date: Tue, 26 Dec 2023 11:57:03 -0800 Subject: [PATCH] [BACKEND] Make sibling metadata optional to compute (#855) Partial fix to https://github.com/jmbannon/ytdl-sub/issues/853 Makes computing sibling metadata optional, defaulted to False. This prevents excessive memory usage when scraping large channels --- src/ytdl_sub/downloaders/url/downloader.py | 9 ++++++-- src/ytdl_sub/downloaders/url/validators.py | 21 ++++++++++++++++- src/ytdl_sub/entries/entry_parent.py | 23 +++++++++++++------ .../prebuilt_presets/music/music.yaml | 11 +++++++-- tests/unit/config/test_config_file.py | 4 ++-- 5 files changed, 54 insertions(+), 14 deletions(-) diff --git a/src/ytdl_sub/downloaders/url/downloader.py b/src/ytdl_sub/downloaders/url/downloader.py index a0262963..67feabd0 100644 --- a/src/ytdl_sub/downloaders/url/downloader.py +++ b/src/ytdl_sub/downloaders/url/downloader.py @@ -385,7 +385,9 @@ class MultiUrlDownloader(SourcePlugin[MultiUrlValidator]): ): yield entry_child - def _download_url_metadata(self, url: str) -> Tuple[List[EntryParent], List[Entry]]: + def _download_url_metadata( + self, url: str, include_sibling_metadata: bool + ) -> Tuple[List[EntryParent], List[Entry]]: """ Downloads only info.json files and forms EntryParent trees """ @@ -401,6 +403,7 @@ class MultiUrlDownloader(SourcePlugin[MultiUrlValidator]): url=url, entry_dicts=entry_dicts, working_directory=self.working_directory, + include_sibling_metadata=include_sibling_metadata, ) orphans = EntryParent.from_entry_dicts_with_no_parents( parents=parents, @@ -438,7 +441,9 @@ class MultiUrlDownloader(SourcePlugin[MultiUrlValidator]): if not (url := self.overrides.apply_formatter(collection_url.url)): continue - parents, orphan_entries = self._download_url_metadata(url=url) + parents, orphan_entries = self._download_url_metadata( + url=url, include_sibling_metadata=collection_url.include_sibling_metadata + ) # TODO: Encapsulate this logic into its own class self._url_state = URLDownloadState( diff --git a/src/ytdl_sub/downloaders/url/validators.py b/src/ytdl_sub/downloaders/url/validators.py index bafe846a..eb999b15 100644 --- a/src/ytdl_sub/downloaders/url/validators.py +++ b/src/ytdl_sub/downloaders/url/validators.py @@ -45,7 +45,13 @@ class UrlThumbnailListValidator(ListValidator[UrlThumbnailValidator]): class UrlValidator(StrictDictValidator): _required_keys = {"url"} - _optional_keys = {"variables", "source_thumbnails", "playlist_thumbnails", "download_reverse"} + _optional_keys = { + "variables", + "source_thumbnails", + "playlist_thumbnails", + "download_reverse", + "include_sibling_metadata", + } @classmethod def partial_validate(cls, name: str, value: Any) -> None: @@ -74,6 +80,9 @@ class UrlValidator(StrictDictValidator): self._download_reverse = self._validate_key( key="download_reverse", validator=BoolValidator, default=True ) + self._include_sibling_metadata = self._validate_key( + key="include_sibling_metadata", validator=BoolValidator, default=False + ) @property def url(self) -> OverridesStringFormatterValidator: @@ -147,6 +156,16 @@ class UrlValidator(StrictDictValidator): """ return self._download_reverse.value + @property + def include_sibling_metadata(self) -> bool: + """ + Optional. Whether to include sibling metadata as an entry variable, which comprises basic + metadata from all other entries (including itself) that belong to the same playlist. For + channels or large playlists, this becomes memory-intensive since you are storing + ``n^2`` metadata. Defaults to False. + """ + return self._include_sibling_metadata.value + class UrlStringOrDictValidator(UrlValidator): """ diff --git a/src/ytdl_sub/entries/entry_parent.py b/src/ytdl_sub/entries/entry_parent.py index 4f2669e4..21ff7d18 100644 --- a/src/ytdl_sub/entries/entry_parent.py +++ b/src/ytdl_sub/entries/entry_parent.py @@ -62,13 +62,16 @@ class EntryParent(BaseEntry): ) return sibling_entry_metadata - def _set_child_variables(self, parents: Optional[List["EntryParent"]] = None) -> "EntryParent": + def _set_child_variables( + self, include_sibling_metadata: bool, parents: Optional[List["EntryParent"]] = None + ) -> "EntryParent": if parents is None: parents = [self] - kwargs_to_add: Dict[str, Any] = { - v.sibling_metadata.metadata_key: self._sibling_entry_metadata() - } + kwargs_to_add: Dict[str, Any] = {} + if include_sibling_metadata: + kwargs_to_add[v.sibling_metadata.metadata_key] = self._sibling_entry_metadata() + if len(parents) >= 1: kwargs_to_add[v.playlist_metadata.metadata_key] = parents[-1]._kwargs if len(parents) >= 2: @@ -83,7 +86,9 @@ class EntryParent(BaseEntry): entry_child._kwargs = dict(entry_child._kwargs, **kwargs_to_add) for parent_child in self.parent_children(): - parent_child._set_child_variables(parents=parents + [parent_child]) + parent_child._set_child_variables( + include_sibling_metadata=include_sibling_metadata, parents=parents + [parent_child] + ) return self @@ -180,7 +185,11 @@ class EntryParent(BaseEntry): @classmethod def from_entry_dicts( - cls, url: str, entry_dicts: List[Dict], working_directory: str + cls, + url: str, + entry_dicts: List[Dict], + working_directory: str, + include_sibling_metadata: bool, ) -> List["EntryParent"]: """ Reads all entry dicts and builds a tree of EntryParents @@ -203,7 +212,7 @@ class EntryParent(BaseEntry): parents = [root_parent] for parent in parents: - parent._set_child_variables() + parent._set_child_variables(include_sibling_metadata=include_sibling_metadata) return parents diff --git a/src/ytdl_sub/prebuilt_presets/music/music.yaml b/src/ytdl_sub/prebuilt_presets/music/music.yaml index ee1a290f..7c2f23a3 100644 --- a/src/ytdl_sub/prebuilt_presets/music/music.yaml +++ b/src/ytdl_sub/prebuilt_presets/music/music.yaml @@ -67,12 +67,17 @@ presets: - "_music_base" download: - - "{url}" + - url: "{url}" + include_sibling_metadata: False _albums_from_playlists: preset: - - "Single" + - "_music_base" + + download: + - url: "{url}" + include_sibling_metadata: True overrides: track_album: "{playlist_title}" @@ -107,6 +112,7 @@ presets: # The first URL will be all the artist's tracks. # Treat these as singles - an album with a single track - url: "{url}/tracks" + include_sibling_metadata: False variables: sc_track_album: "{title}" sc_track_number: "1" @@ -117,6 +123,7 @@ presets: # to an album and tracks (in the URL above), it will resolve to this # URL and include the album metadata we set below. - url: "{url}/albums" + include_sibling_metadata: True variables: sc_track_album: "{playlist_title}" sc_track_number: "{playlist_index}" diff --git a/tests/unit/config/test_config_file.py b/tests/unit/config/test_config_file.py index 1baab94f..74ec03d2 100644 --- a/tests/unit/config/test_config_file.py +++ b/tests/unit/config/test_config_file.py @@ -73,8 +73,8 @@ class TestConfigFilePartiallyValidatesPresets: preset_dict={"download": {"bad_key": "nope"}}, expected_error_message="Validation error in partial_preset.download.1: " "'partial_preset.download.1' contains the field 'bad_key' which is not allowed. " - "Allowed fields: download_reverse, playlist_thumbnails, source_thumbnails, url, " - "variables", + "Allowed fields: download_reverse, include_sibling_metadata, playlist_thumbnails, " + "source_thumbnails, url, variables", ) @pytest.mark.parametrize(