[BUGFIX] Retry downloading entry thumbnail if missing (#293)

This commit is contained in:
Jesse Bannon 2022-10-16 22:25:59 -07:00 committed by GitHub
parent 6125fd0efb
commit 86c7e7b84e
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
2 changed files with 49 additions and 4 deletions

View file

@ -147,6 +147,15 @@ class Downloader(DownloadArchiver, Generic[DownloaderOptionsT], ABC):
self.parents: List[EntryParent] = []
self.downloaded_entries: Dict[str, Entry] = {}
@property
def ytdl_options(self) -> Dict:
"""
Returns
-------
YTLD options dict
"""
return self._ytdl_options_builder.clone().to_dict()
@contextmanager
def ytdl_downloader(self, ytdl_options_overrides: Optional[Dict] = None) -> ytdl.YoutubeDL:
"""
@ -166,7 +175,16 @@ class Downloader(DownloadArchiver, Generic[DownloaderOptionsT], ABC):
-------
True if dry-run is enabled. False otherwise.
"""
return self._ytdl_options_builder.to_dict().get("skip_download", False)
return self.ytdl_options.get("skip_download", False)
@property
def is_entry_thumbnails_enabled(self) -> bool:
"""
Returns
-------
True if entry thumbnails should be downloaded. False otherwise.
"""
return self.ytdl_options.get("writethumbnail", False)
def extract_info(self, ytdl_options_overrides: Optional[Dict] = None, **kwargs) -> Dict:
"""
@ -186,6 +204,7 @@ class Downloader(DownloadArchiver, Generic[DownloaderOptionsT], ABC):
def extract_info_with_retry(
self,
is_downloaded_fn: Optional[Callable[[], bool]] = None,
is_thumbnail_downloaded_fn: Optional[Callable[[], bool]] = None,
ytdl_options_overrides: Optional[Dict] = None,
**kwargs,
) -> Dict:
@ -200,6 +219,8 @@ class Downloader(DownloadArchiver, Generic[DownloaderOptionsT], ABC):
----------
is_downloaded_fn
Optional. Function to check if the entry is downloaded
is_thumbnail_downloaded_fn
Optional. Function to check if the entry thumbnail is downloaded
ytdl_options_overrides
Optional. Dict containing ytdl args to override other predefined ytdl args
**kwargs
@ -218,9 +239,21 @@ class Downloader(DownloadArchiver, Generic[DownloaderOptionsT], ABC):
entry_dict = self.extract_info(
ytdl_options_overrides=copied_ytdl_options_overrides, **kwargs
)
if is_downloaded_fn is None or is_downloaded_fn():
is_downloaded = is_downloaded_fn is None or is_downloaded_fn()
is_thumbnail_downloaded = (
is_thumbnail_downloaded_fn is None or is_thumbnail_downloaded_fn()
)
if is_downloaded and is_thumbnail_downloaded:
return entry_dict
# If the video file is downloaded but the thumbnail is not, then do not download
# the video again
if is_downloaded and not is_thumbnail_downloaded:
copied_ytdl_options_overrides["skip_download"] = True
copied_ytdl_options_overrides["writethumbnail"] = True
time.sleep(self._extract_entry_retry_wait_sec)
num_tries += 1
@ -354,7 +387,7 @@ class Downloader(DownloadArchiver, Generic[DownloaderOptionsT], ABC):
Separate download archive writing between collection urls. This is so break_on_existing
does not break when downloading from subset urls.
"""
archive_path = self._ytdl_options_builder.to_dict().get("download_archive", "")
archive_path = self.ytdl_options.get("download_archive", "")
backup_archive_path = f"{archive_path}.backup"
# If archive path exists, maintain download archive is enable
@ -380,6 +413,9 @@ class Downloader(DownloadArchiver, Generic[DownloaderOptionsT], ABC):
def _extract_entry_info_with_retry(self, entry: Entry) -> Entry:
download_entry_dict = self.extract_info_with_retry(
is_downloaded_fn=None if self.is_dry_run else entry.is_downloaded,
is_thumbnail_downloaded_fn=None
if (self.is_dry_run or not self.is_entry_thumbnails_enabled)
else entry.is_thumbnail_downloaded,
url=entry.webpage_url,
ytdl_options_overrides={"writeinfojson": False, "skip_download": self.is_dry_run},
)

View file

@ -81,12 +81,21 @@ class Entry(EntryVariables, BaseEntry):
with open(self.get_download_info_json_path(), "w", encoding="utf-8") as file:
file.write(kwargs_json)
@final
def is_thumbnail_downloaded(self) -> bool:
"""
Returns
-------
True if the thumbnail file exist locally. False otherwise.
"""
return self.get_ytdlp_download_thumbnail_path() is not None
@final
def is_downloaded(self) -> bool:
"""
Returns
-------
True if the file and thumbnail exist locally. False otherwise.
True if the file exist locally. False otherwise.
"""
file_exists = os.path.isfile(self.get_download_file_path())