danvics/ytdl-sub
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

[BUGFIX] Fix source metadata getting fetched when yt-dlp breaks (#889)

Browse source 
Prior PR only got channel metadata for a playlist if it never 'broke' (i.e. MaxDownloads, BreakOnExisting). Will now fetch it regardless.
This commit is contained in:
Jesse Bannon 2024-01-09 19:25:09 -08:00 committed by GitHub
parent be0f6f2d2d
commit 3b36ffd050
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
2 changed files with 27 additions and 20 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

45
src/ytdl_sub/downloaders/ytdlp.py
View file

@ -216,15 +216,11 @@ class YTDLP:
**kwargs
arguments passed directory to YoutubeDL extract_info
"""
parent_dict: Dict = {}
try:
with cls._listen_and_log_downloaded_info_json(
working_directory=working_directory, log_prefix=log_prefix_on_info_json_dl
):
# TODO: Fetch parent_dict if it breaks
parent_dict = cls.extract_info(
ytdl_options_overrides=ytdl_options_overrides, **kwargs
)
cls.extract_info(ytdl_options_overrides=ytdl_options_overrides, **kwargs)
except RejectedVideoReached:
cls.logger.debug(
"RejectedVideoReached, stopping additional downloads "
@ -238,23 +234,34 @@ class YTDLP:
except MaxDownloadsReached:
cls.logger.info("MaxDownloadsReached, stopping additional downloads.")
# For YouTube playlists in particular, channel metadata is not fetched. Attempt to get
# channel metadata via grabbing uploader_url info json a max of 3 times
if isinstance(parent_dict, dict):
current_iter = 0
url = kwargs.get("url")
uploader_url = parent_dict.get("uploader_url")
while current_iter < 3 and uploader_url and url != uploader_url:
cls.logger.debug("Attempting to get parent metadata from URL %s", uploader_url)
parent_dicts: List[Dict] = []
entry_dicts = cls._get_entry_dicts_from_info_json_files(working_directory=working_directory)
entry_ids = {entry_dict.get("id") for entry_dict in entry_dicts}
# Try to get additional uploader (source) metadata that yt-dlp does not fetch
# in a single request
for entry_dict in entry_dicts:
if not (uploader_id := entry_dict.get("uploader_id")):
continue
if uploader_id in entry_ids or not (uploader_url := entry_dict.get("uploader_url")):
continue
cls.logger.debug("Attempting to get parent metadata from URL %s", uploader_url)
try:
parent_dict = cls.extract_info(
ytdl_options_overrides=ytdl_options_overrides | {"playlist_items": "0:0"},
url=uploader_url,
)
except Exception: # pylint: disable=broad-except
# Do not try this uploader_id again
entry_ids.add(uploader_id)
break
if not isinstance(parent_dict, dict):
break
current_iter += 1
url = uploader_url
uploader_url = parent_dict.get("uploader_url")
if isinstance(parent_dict, dict):
parent_id = parent_dict.get("id")
parent_dicts.append(parent_dict)
entry_ids |= {uploader_id, parent_id}
cls.logger.debug("Adding parent metadata with ids [%s, %s]", uploader_id, parent_id)
return cls._get_entry_dicts_from_info_json_files(working_directory=working_directory)
return entry_dicts + parent_dicts

2
tests/resources/expected_downloads_summaries/youtube/test_channel_full.json
View file

@ -30,7 +30,7 @@
"Project ⧸ Zombie/Season 2011/s2011.e063001 - Project Zombie ｜Fin｜.nfo": "54ea4a48116aa98480a79495036c25e9",
"Project ⧸ Zombie/Season 2011/s2011.e112101 - Skyrim 'Ultra HD w⧸Mods' [PC]-thumb.jpg": "1718599d5189c65f7d8cf6acfa5ea851",
"Project ⧸ Zombie/Season 2011/s2011.e112101 - Skyrim 'Ultra HD w⧸Mods' [PC].info.json": "INFO_JSON",
"Project ⧸ Zombie/Season 2011/s2011.e112101 - Skyrim 'Ultra HD w⧸Mods' [PC].mp4": "2a18cc4baa198edecdc5154082f73eff",
"Project ⧸ Zombie/Season 2011/s2011.e112101 - Skyrim 'Ultra HD w⧸Mods' [PC].mp4": "554d46311112228c22ca9532007803cd",
"Project ⧸ Zombie/Season 2011/s2011.e112101 - Skyrim 'Ultra HD w⧸Mods' [PC].nfo": "04f6aad56d85b1f65b81b6b8000f6479",
"Project ⧸ Zombie/Season 2012/s2012.e012301 - Project Zombie ｜Map Trailer｜-thumb.jpg": "54ebe9df801b278fdd17b21afa8373a6",
"Project ⧸ Zombie/Season 2012/s2012.e012301 - Project Zombie ｜Map Trailer｜.info.json": "INFO_JSON",