[BUGFIX] Retry downloading entry thumbnail if missing (#293)
This commit is contained in:
parent
6125fd0efb
commit
86c7e7b84e
2 changed files with 49 additions and 4 deletions
|
|
@ -147,6 +147,15 @@ class Downloader(DownloadArchiver, Generic[DownloaderOptionsT], ABC):
|
||||||
self.parents: List[EntryParent] = []
|
self.parents: List[EntryParent] = []
|
||||||
self.downloaded_entries: Dict[str, Entry] = {}
|
self.downloaded_entries: Dict[str, Entry] = {}
|
||||||
|
|
||||||
|
@property
|
||||||
|
def ytdl_options(self) -> Dict:
|
||||||
|
"""
|
||||||
|
Returns
|
||||||
|
-------
|
||||||
|
YTLD options dict
|
||||||
|
"""
|
||||||
|
return self._ytdl_options_builder.clone().to_dict()
|
||||||
|
|
||||||
@contextmanager
|
@contextmanager
|
||||||
def ytdl_downloader(self, ytdl_options_overrides: Optional[Dict] = None) -> ytdl.YoutubeDL:
|
def ytdl_downloader(self, ytdl_options_overrides: Optional[Dict] = None) -> ytdl.YoutubeDL:
|
||||||
"""
|
"""
|
||||||
|
|
@ -166,7 +175,16 @@ class Downloader(DownloadArchiver, Generic[DownloaderOptionsT], ABC):
|
||||||
-------
|
-------
|
||||||
True if dry-run is enabled. False otherwise.
|
True if dry-run is enabled. False otherwise.
|
||||||
"""
|
"""
|
||||||
return self._ytdl_options_builder.to_dict().get("skip_download", False)
|
return self.ytdl_options.get("skip_download", False)
|
||||||
|
|
||||||
|
@property
|
||||||
|
def is_entry_thumbnails_enabled(self) -> bool:
|
||||||
|
"""
|
||||||
|
Returns
|
||||||
|
-------
|
||||||
|
True if entry thumbnails should be downloaded. False otherwise.
|
||||||
|
"""
|
||||||
|
return self.ytdl_options.get("writethumbnail", False)
|
||||||
|
|
||||||
def extract_info(self, ytdl_options_overrides: Optional[Dict] = None, **kwargs) -> Dict:
|
def extract_info(self, ytdl_options_overrides: Optional[Dict] = None, **kwargs) -> Dict:
|
||||||
"""
|
"""
|
||||||
|
|
@ -186,6 +204,7 @@ class Downloader(DownloadArchiver, Generic[DownloaderOptionsT], ABC):
|
||||||
def extract_info_with_retry(
|
def extract_info_with_retry(
|
||||||
self,
|
self,
|
||||||
is_downloaded_fn: Optional[Callable[[], bool]] = None,
|
is_downloaded_fn: Optional[Callable[[], bool]] = None,
|
||||||
|
is_thumbnail_downloaded_fn: Optional[Callable[[], bool]] = None,
|
||||||
ytdl_options_overrides: Optional[Dict] = None,
|
ytdl_options_overrides: Optional[Dict] = None,
|
||||||
**kwargs,
|
**kwargs,
|
||||||
) -> Dict:
|
) -> Dict:
|
||||||
|
|
@ -200,6 +219,8 @@ class Downloader(DownloadArchiver, Generic[DownloaderOptionsT], ABC):
|
||||||
----------
|
----------
|
||||||
is_downloaded_fn
|
is_downloaded_fn
|
||||||
Optional. Function to check if the entry is downloaded
|
Optional. Function to check if the entry is downloaded
|
||||||
|
is_thumbnail_downloaded_fn
|
||||||
|
Optional. Function to check if the entry thumbnail is downloaded
|
||||||
ytdl_options_overrides
|
ytdl_options_overrides
|
||||||
Optional. Dict containing ytdl args to override other predefined ytdl args
|
Optional. Dict containing ytdl args to override other predefined ytdl args
|
||||||
**kwargs
|
**kwargs
|
||||||
|
|
@ -218,9 +239,21 @@ class Downloader(DownloadArchiver, Generic[DownloaderOptionsT], ABC):
|
||||||
entry_dict = self.extract_info(
|
entry_dict = self.extract_info(
|
||||||
ytdl_options_overrides=copied_ytdl_options_overrides, **kwargs
|
ytdl_options_overrides=copied_ytdl_options_overrides, **kwargs
|
||||||
)
|
)
|
||||||
if is_downloaded_fn is None or is_downloaded_fn():
|
|
||||||
|
is_downloaded = is_downloaded_fn is None or is_downloaded_fn()
|
||||||
|
is_thumbnail_downloaded = (
|
||||||
|
is_thumbnail_downloaded_fn is None or is_thumbnail_downloaded_fn()
|
||||||
|
)
|
||||||
|
|
||||||
|
if is_downloaded and is_thumbnail_downloaded:
|
||||||
return entry_dict
|
return entry_dict
|
||||||
|
|
||||||
|
# If the video file is downloaded but the thumbnail is not, then do not download
|
||||||
|
# the video again
|
||||||
|
if is_downloaded and not is_thumbnail_downloaded:
|
||||||
|
copied_ytdl_options_overrides["skip_download"] = True
|
||||||
|
copied_ytdl_options_overrides["writethumbnail"] = True
|
||||||
|
|
||||||
time.sleep(self._extract_entry_retry_wait_sec)
|
time.sleep(self._extract_entry_retry_wait_sec)
|
||||||
num_tries += 1
|
num_tries += 1
|
||||||
|
|
||||||
|
|
@ -354,7 +387,7 @@ class Downloader(DownloadArchiver, Generic[DownloaderOptionsT], ABC):
|
||||||
Separate download archive writing between collection urls. This is so break_on_existing
|
Separate download archive writing between collection urls. This is so break_on_existing
|
||||||
does not break when downloading from subset urls.
|
does not break when downloading from subset urls.
|
||||||
"""
|
"""
|
||||||
archive_path = self._ytdl_options_builder.to_dict().get("download_archive", "")
|
archive_path = self.ytdl_options.get("download_archive", "")
|
||||||
backup_archive_path = f"{archive_path}.backup"
|
backup_archive_path = f"{archive_path}.backup"
|
||||||
|
|
||||||
# If archive path exists, maintain download archive is enable
|
# If archive path exists, maintain download archive is enable
|
||||||
|
|
@ -380,6 +413,9 @@ class Downloader(DownloadArchiver, Generic[DownloaderOptionsT], ABC):
|
||||||
def _extract_entry_info_with_retry(self, entry: Entry) -> Entry:
|
def _extract_entry_info_with_retry(self, entry: Entry) -> Entry:
|
||||||
download_entry_dict = self.extract_info_with_retry(
|
download_entry_dict = self.extract_info_with_retry(
|
||||||
is_downloaded_fn=None if self.is_dry_run else entry.is_downloaded,
|
is_downloaded_fn=None if self.is_dry_run else entry.is_downloaded,
|
||||||
|
is_thumbnail_downloaded_fn=None
|
||||||
|
if (self.is_dry_run or not self.is_entry_thumbnails_enabled)
|
||||||
|
else entry.is_thumbnail_downloaded,
|
||||||
url=entry.webpage_url,
|
url=entry.webpage_url,
|
||||||
ytdl_options_overrides={"writeinfojson": False, "skip_download": self.is_dry_run},
|
ytdl_options_overrides={"writeinfojson": False, "skip_download": self.is_dry_run},
|
||||||
)
|
)
|
||||||
|
|
|
||||||
|
|
@ -81,12 +81,21 @@ class Entry(EntryVariables, BaseEntry):
|
||||||
with open(self.get_download_info_json_path(), "w", encoding="utf-8") as file:
|
with open(self.get_download_info_json_path(), "w", encoding="utf-8") as file:
|
||||||
file.write(kwargs_json)
|
file.write(kwargs_json)
|
||||||
|
|
||||||
|
@final
|
||||||
|
def is_thumbnail_downloaded(self) -> bool:
|
||||||
|
"""
|
||||||
|
Returns
|
||||||
|
-------
|
||||||
|
True if the thumbnail file exist locally. False otherwise.
|
||||||
|
"""
|
||||||
|
return self.get_ytdlp_download_thumbnail_path() is not None
|
||||||
|
|
||||||
@final
|
@final
|
||||||
def is_downloaded(self) -> bool:
|
def is_downloaded(self) -> bool:
|
||||||
"""
|
"""
|
||||||
Returns
|
Returns
|
||||||
-------
|
-------
|
||||||
True if the file and thumbnail exist locally. False otherwise.
|
True if the file exist locally. False otherwise.
|
||||||
"""
|
"""
|
||||||
file_exists = os.path.isfile(self.get_download_file_path())
|
file_exists = os.path.isfile(self.get_download_file_path())
|
||||||
|
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue