diff --git a/src/ytdl_sub/downloaders/ytdlp.py b/src/ytdl_sub/downloaders/ytdlp.py index 8fa7f10e..970acb1f 100644 --- a/src/ytdl_sub/downloaders/ytdlp.py +++ b/src/ytdl_sub/downloaders/ytdlp.py @@ -216,15 +216,11 @@ class YTDLP: **kwargs arguments passed directory to YoutubeDL extract_info """ - parent_dict: Dict = {} try: with cls._listen_and_log_downloaded_info_json( working_directory=working_directory, log_prefix=log_prefix_on_info_json_dl ): - # TODO: Fetch parent_dict if it breaks - parent_dict = cls.extract_info( - ytdl_options_overrides=ytdl_options_overrides, **kwargs - ) + cls.extract_info(ytdl_options_overrides=ytdl_options_overrides, **kwargs) except RejectedVideoReached: cls.logger.debug( "RejectedVideoReached, stopping additional downloads " @@ -238,23 +234,34 @@ class YTDLP: except MaxDownloadsReached: cls.logger.info("MaxDownloadsReached, stopping additional downloads.") - # For YouTube playlists in particular, channel metadata is not fetched. Attempt to get - # channel metadata via grabbing uploader_url info json a max of 3 times - if isinstance(parent_dict, dict): - current_iter = 0 - url = kwargs.get("url") - uploader_url = parent_dict.get("uploader_url") - while current_iter < 3 and uploader_url and url != uploader_url: - cls.logger.debug("Attempting to get parent metadata from URL %s", uploader_url) + parent_dicts: List[Dict] = [] + entry_dicts = cls._get_entry_dicts_from_info_json_files(working_directory=working_directory) + entry_ids = {entry_dict.get("id") for entry_dict in entry_dicts} + + # Try to get additional uploader (source) metadata that yt-dlp does not fetch + # in a single request + for entry_dict in entry_dicts: + if not (uploader_id := entry_dict.get("uploader_id")): + continue + + if uploader_id in entry_ids or not (uploader_url := entry_dict.get("uploader_url")): + continue + + cls.logger.debug("Attempting to get parent metadata from URL %s", uploader_url) + try: parent_dict = cls.extract_info( ytdl_options_overrides=ytdl_options_overrides | {"playlist_items": "0:0"}, url=uploader_url, ) + except Exception: # pylint: disable=broad-except + # Do not try this uploader_id again + entry_ids.add(uploader_id) + break - if not isinstance(parent_dict, dict): - break - current_iter += 1 - url = uploader_url - uploader_url = parent_dict.get("uploader_url") + if isinstance(parent_dict, dict): + parent_id = parent_dict.get("id") + parent_dicts.append(parent_dict) + entry_ids |= {uploader_id, parent_id} + cls.logger.debug("Adding parent metadata with ids [%s, %s]", uploader_id, parent_id) - return cls._get_entry_dicts_from_info_json_files(working_directory=working_directory) + return entry_dicts + parent_dicts diff --git a/tests/resources/expected_downloads_summaries/youtube/test_channel_full.json b/tests/resources/expected_downloads_summaries/youtube/test_channel_full.json index 74e55e90..38361867 100644 --- a/tests/resources/expected_downloads_summaries/youtube/test_channel_full.json +++ b/tests/resources/expected_downloads_summaries/youtube/test_channel_full.json @@ -30,7 +30,7 @@ "Project ⧸ Zombie/Season 2011/s2011.e063001 - Project Zombie |Fin|.nfo": "54ea4a48116aa98480a79495036c25e9", "Project ⧸ Zombie/Season 2011/s2011.e112101 - Skyrim 'Ultra HD w⧸Mods' [PC]-thumb.jpg": "1718599d5189c65f7d8cf6acfa5ea851", "Project ⧸ Zombie/Season 2011/s2011.e112101 - Skyrim 'Ultra HD w⧸Mods' [PC].info.json": "INFO_JSON", - "Project ⧸ Zombie/Season 2011/s2011.e112101 - Skyrim 'Ultra HD w⧸Mods' [PC].mp4": "2a18cc4baa198edecdc5154082f73eff", + "Project ⧸ Zombie/Season 2011/s2011.e112101 - Skyrim 'Ultra HD w⧸Mods' [PC].mp4": "554d46311112228c22ca9532007803cd", "Project ⧸ Zombie/Season 2011/s2011.e112101 - Skyrim 'Ultra HD w⧸Mods' [PC].nfo": "04f6aad56d85b1f65b81b6b8000f6479", "Project ⧸ Zombie/Season 2012/s2012.e012301 - Project Zombie |Map Trailer|-thumb.jpg": "54ebe9df801b278fdd17b21afa8373a6", "Project ⧸ Zombie/Season 2012/s2012.e012301 - Project Zombie |Map Trailer|.info.json": "INFO_JSON",