From 5f9b8afc4b739926b86809f1c24e0d13f44c7c1f Mon Sep 17 00:00:00 2001 From: Jesse Bannon Date: Fri, 28 Jul 2023 13:12:46 -0700 Subject: [PATCH] [BUGFIX] Do not hard-fail if the thumbnail fails to download/convert (#675) * [BUGFIX] Do not fail with thumbnail issues * helper * refactor * dry-run * regen fixture * download arhcive txt -> json in docs, fixtures * return if no download thumb path * better test * more test coverage --- src/ytdl_sub/config/preset_options.py | 4 +- src/ytdl_sub/downloaders/url/downloader.py | 16 ++++--- src/ytdl_sub/downloaders/ytdlp.py | 12 +++-- src/ytdl_sub/entries/entry.py | 15 ++++-- src/ytdl_sub/plugins/embed_thumbnail.py | 8 ++-- src/ytdl_sub/plugins/music_tags.py | 7 +-- src/ytdl_sub/plugins/split_by_chapters.py | 9 +--- .../subscriptions/subscription_download.py | 17 ++++--- src/ytdl_sub/utils/thumbnail.py | 37 ++++++++------- tests/e2e/plugins/test_regex.py | 2 - tests/e2e/youtube/test_channel.py | 2 +- tests/e2e/youtube/test_video.py | 46 ++++++++++++++++++- tests/expected_download.py | 7 ++- tests/expected_transaction_log.py | 11 ++--- .../date_range/test_channel_recent.json | 6 +-- .../test_channel_rolling_recent.json | 4 +- .../youtube/test_channel_full.json | 26 +++++------ .../youtube/test_video_missing_thumb.json | 5 ++ .../youtube/test_video_missing_thumb.txt | 14 ++++++ 19 files changed, 162 insertions(+), 86 deletions(-) create mode 100644 tests/resources/expected_downloads_summaries/youtube/test_video_missing_thumb.json create mode 100644 tests/resources/transaction_log_summaries/youtube/test_video_missing_thumb.txt diff --git a/src/ytdl_sub/config/preset_options.py b/src/ytdl_sub/config/preset_options.py index 55533b47..2dd0b24a 100644 --- a/src/ytdl_sub/config/preset_options.py +++ b/src/ytdl_sub/config/preset_options.py @@ -245,7 +245,7 @@ class OutputOptions(StrictDictValidator): # optional thumbnail_name: "{title_sanitized}.{thumbnail_ext}" info_json_name: "{title_sanitized}.{info_json_ext}" - download_archive_name: ".ytdl-sub-{subscription_name}-download-archive.txt" + download_archive_name: ".ytdl-sub-{subscription_name}-download-archive.json" maintain_download_archive: True keep_files_before: now keep_files_after: 19000101 @@ -354,7 +354,7 @@ class OutputOptions(StrictDictValidator): def download_archive_name(self) -> Optional[OverridesStringFormatterValidator]: """ Optional. The file name to store a subscriptions download archive placed relative to - the output directory. Defaults to ``.ytdl-sub-{subscription_name}-download-archive.txt`` + the output directory. Defaults to ``.ytdl-sub-{subscription_name}-download-archive.json`` """ return self._download_archive_name diff --git a/src/ytdl_sub/downloaders/url/downloader.py b/src/ytdl_sub/downloaders/url/downloader.py index d66749af..5cbadc4f 100644 --- a/src/ytdl_sub/downloaders/url/downloader.py +++ b/src/ytdl_sub/downloaders/url/downloader.py @@ -34,8 +34,8 @@ from ytdl_sub.entries.variables.kwargs import YTDL_SUB_MATCH_FILTER_REJECT from ytdl_sub.utils.file_handler import FileHandler from ytdl_sub.utils.logger import Logger from ytdl_sub.utils.thumbnail import ThumbnailTypes -from ytdl_sub.utils.thumbnail import convert_download_thumbnail from ytdl_sub.utils.thumbnail import download_and_convert_url_thumbnail +from ytdl_sub.utils.thumbnail import try_convert_download_thumbnail from ytdl_sub.ytdl_additions.enhanced_download_archive import EnhancedDownloadArchive download_logger = Logger.get(name="downloader") @@ -82,11 +82,9 @@ class UrlDownloaderThumbnailPlugin(SourcePluginExtension): # If latest entry, always update the thumbnail on each entry if thumbnail_id == ThumbnailTypes.LATEST_ENTRY: - # Make sure the entry's thumbnail is converted to jpg - convert_download_thumbnail(entry, error_if_not_found=False) # always save in dry-run even if it doesn't exist... - if self.is_dry_run or os.path.isfile(entry.get_download_thumbnail_path()): + if self.is_dry_run or entry.is_thumbnail_downloaded(): self.save_file( file_name=entry.get_download_thumbnail_name(), output_file_name=thumbnail_name, @@ -138,8 +136,14 @@ class UrlDownloaderThumbnailPlugin(SourcePluginExtension): def modify_entry(self, entry: Entry) -> Optional[Entry]: """ - Use the entry to download thumbnails (or move if LATEST_ENTRY) + Use the entry to download thumbnails (or move if LATEST_ENTRY). + In addition, convert the entry thumbnail to jpg """ + # We always convert entry thumbnails to jpgs, and is performed here to be done + # as early as possible in the plugin pipeline (downstream plugins depend on it being jpg) + if not self.is_dry_run: + try_convert_download_thumbnail(entry=entry) + if entry.kwargs_get(COLLECTION_URL) in self._collection_url_mapping: self._download_url_thumbnails( collection_url=self._collection_url_mapping[entry.kwargs(COLLECTION_URL)], @@ -346,7 +350,7 @@ class MultiUrlDownloader(SourcePlugin[MultiUrlValidator]): is_downloaded_fn=None if self.is_dry_run else entry.is_downloaded, is_thumbnail_downloaded_fn=None if (self.is_dry_run or not self.is_entry_thumbnails_enabled) - else entry.is_thumbnail_downloaded, + else entry.is_thumbnail_downloaded_via_ytdlp, url=entry.webpage_url, ) return Entry(download_entry_dict, working_directory=self.working_directory) diff --git a/src/ytdl_sub/downloaders/ytdlp.py b/src/ytdl_sub/downloaders/ytdlp.py index d66f0c80..a7912232 100644 --- a/src/ytdl_sub/downloaders/ytdlp.py +++ b/src/ytdl_sub/downloaders/ytdlp.py @@ -86,10 +86,12 @@ class YTDLP: If the entry fails to download """ num_tries = 0 - entry_files_exist = False copied_ytdl_options_overrides = copy.deepcopy(ytdl_options_overrides) - while not entry_files_exist and num_tries < cls._EXTRACT_ENTRY_NUM_RETRIES: + is_downloaded = False + entry_dict: Optional[Dict] = None + + while num_tries < cls._EXTRACT_ENTRY_NUM_RETRIES: entry_dict = cls.extract_info( ytdl_options_overrides=copied_ytdl_options_overrides, **kwargs ) @@ -115,7 +117,7 @@ class YTDLP: time.sleep(cls._EXTRACT_ENTRY_RETRY_WAIT_SEC) num_tries += 1 - # Remove the download archive so it can retry without thinking its already downloaded, + # Remove the download archive to retry without thinking its already downloaded, # even though it is not if "download_archive" in copied_ytdl_options_overrides: del copied_ytdl_options_overrides["download_archive"] @@ -127,6 +129,10 @@ class YTDLP: cls._EXTRACT_ENTRY_NUM_RETRIES, ) + # Still return if the media file downloaded (thumbnail could be missing) + if is_downloaded and entry_dict is not None: + return entry_dict + error_dict = {"ytdl_options": ytdl_options_overrides, "kwargs": kwargs} raise FileNotDownloadedException( f"yt-dlp failed to download an entry with these arguments: {error_dict}" diff --git a/src/ytdl_sub/entries/entry.py b/src/ytdl_sub/entries/entry.py index 08d2285d..0538c66c 100644 --- a/src/ytdl_sub/entries/entry.py +++ b/src/ytdl_sub/entries/entry.py @@ -54,7 +54,7 @@ class Entry(EntryVariables, BaseEntry): """Returns the entry's thumbnail's file path to where it was downloaded""" return str(Path(self.working_directory()) / self.get_download_thumbnail_name()) - def get_ytdlp_download_thumbnail_path(self) -> Optional[str]: + def try_get_ytdlp_download_thumbnail_path(self) -> Optional[str]: """ The source `thumbnail` value and the actual downloaded thumbnail extension sometimes do not match. Return the actual downloaded thumbnail path. @@ -83,14 +83,23 @@ class Entry(EntryVariables, BaseEntry): with open(self.get_download_info_json_path(), "w", encoding="utf-8") as file: file.write(kwargs_json) + @final + def is_thumbnail_downloaded_via_ytdlp(self) -> bool: + """ + Returns + ------- + True if ANY thumbnail file exist locally. False otherwise. + """ + return self.try_get_ytdlp_download_thumbnail_path() is not None + @final def is_thumbnail_downloaded(self) -> bool: """ Returns ------- - True if the thumbnail file exist locally. False otherwise. + True if the thumbnail file exists and is its proper format. False otherwise. """ - return self.get_ytdlp_download_thumbnail_path() is not None + return os.path.isfile(self.get_download_thumbnail_path()) @final def is_downloaded(self) -> bool: diff --git a/src/ytdl_sub/plugins/embed_thumbnail.py b/src/ytdl_sub/plugins/embed_thumbnail.py index 45dbd776..4886f707 100644 --- a/src/ytdl_sub/plugins/embed_thumbnail.py +++ b/src/ytdl_sub/plugins/embed_thumbnail.py @@ -11,7 +11,6 @@ from ytdl_sub.utils.ffmpeg import FFMPEG from ytdl_sub.utils.file_handler import FileHandler from ytdl_sub.utils.file_handler import FileMetadata from ytdl_sub.utils.logger import Logger -from ytdl_sub.utils.thumbnail import convert_download_thumbnail from ytdl_sub.validators.audo_codec_validator import AUDIO_CODEC_EXTS from ytdl_sub.validators.validators import BoolValidator @@ -91,8 +90,11 @@ class EmbedThumbnailPlugin(Plugin[EmbedThumbnailOptions]): return None if not self.is_dry_run: - # convert the entry thumbnail so it is embedded as jpg - convert_download_thumbnail(entry=entry) + if not entry.is_thumbnail_downloaded(): + logger.warning( + "Cannot embed thumbnail for '%s' because it is not available", entry.title + ) + return None if entry.ext in AUDIO_CODEC_EXTS: self._embed_audio_file(entry) diff --git a/src/ytdl_sub/plugins/music_tags.py b/src/ytdl_sub/plugins/music_tags.py index 7dcbfdaa..71e4255e 100644 --- a/src/ytdl_sub/plugins/music_tags.py +++ b/src/ytdl_sub/plugins/music_tags.py @@ -11,7 +11,6 @@ from ytdl_sub.config.preset_options import OptionsDictValidator from ytdl_sub.entries.entry import Entry from ytdl_sub.utils.file_handler import FileMetadata from ytdl_sub.utils.logger import Logger -from ytdl_sub.utils.thumbnail import convert_download_thumbnail from ytdl_sub.validators.audo_codec_validator import AUDIO_CODEC_EXTS from ytdl_sub.validators.strict_dict_validator import StrictDictValidator from ytdl_sub.validators.string_formatter_validators import ListFormatterValidator @@ -162,11 +161,7 @@ class MusicTagsPlugin(Plugin[MusicTagsOptions]): ) setattr(audio_file, tag_name, tag_value[0]) - if self.plugin_options.embed_thumbnail: - - # convert the entry thumbnail so it is embedded as jpg - convert_download_thumbnail(entry=entry) - + if self.plugin_options.embed_thumbnail and entry.is_thumbnail_downloaded(): with open(entry.get_download_thumbnail_path(), "rb") as thumb: mediafile_img = mediafile.Image( data=thumb.read(), desc="cover", type=mediafile.ImageType.front diff --git a/src/ytdl_sub/plugins/split_by_chapters.py b/src/ytdl_sub/plugins/split_by_chapters.py index f684f921..50717854 100644 --- a/src/ytdl_sub/plugins/split_by_chapters.py +++ b/src/ytdl_sub/plugins/split_by_chapters.py @@ -1,5 +1,4 @@ import copy -import os.path from pathlib import Path from typing import Any from typing import List @@ -21,7 +20,6 @@ from ytdl_sub.utils.exceptions import ValidationException from ytdl_sub.utils.ffmpeg import FFMPEG from ytdl_sub.utils.file_handler import FileHandler from ytdl_sub.utils.file_handler import FileMetadata -from ytdl_sub.utils.thumbnail import convert_download_thumbnail from ytdl_sub.validators.string_select_validator import StringSelectValidator @@ -184,11 +182,6 @@ class SplitByChaptersPlugin(SplitPlugin[SplitByChaptersOptions]): f"Tried to split '{entry.title}' by chapters but it has no chapters" ) - # convert the entry thumbnail early so we do not have to guess the thumbnail extension - # when copying it. Do not error if it's not found, in case thumbnail_name is not set - if not self.is_dry_run: - convert_download_thumbnail(entry=entry, error_if_not_found=False) - for idx, title in enumerate(chapters.titles): new_uid = _split_video_uid(source_uid=entry.uid, idx=idx) @@ -209,7 +202,7 @@ class SplitByChaptersPlugin(SplitPlugin[SplitByChaptersOptions]): # Copy the original vid thumbnail to the working directory with the new uid. This so # downstream logic thinks this split video has its own thumbnail - if os.path.isfile(entry.get_download_thumbnail_path()): + if entry.is_thumbnail_downloaded(): FileHandler.copy( src_file_path=entry.get_download_thumbnail_path(), dst_file_path=Path(self.working_directory) diff --git a/src/ytdl_sub/subscriptions/subscription_download.py b/src/ytdl_sub/subscriptions/subscription_download.py index b0de508d..60d580bb 100644 --- a/src/ytdl_sub/subscriptions/subscription_download.py +++ b/src/ytdl_sub/subscriptions/subscription_download.py @@ -1,4 +1,5 @@ import contextlib +import logging import os import shutil from abc import ABC @@ -21,7 +22,9 @@ from ytdl_sub.utils.exceptions import ValidationException from ytdl_sub.utils.file_handler import FileHandler from ytdl_sub.utils.file_handler import FileHandlerTransactionLog from ytdl_sub.utils.file_handler import FileMetadata -from ytdl_sub.utils.thumbnail import convert_download_thumbnail +from ytdl_sub.utils.logger import Logger + +logger: logging.Logger = Logger.get() def _get_split_plugin(plugins: List[Plugin]) -> Optional[SplitPlugin]: @@ -69,16 +72,12 @@ class SubscriptionDownload(BaseSubscription, ABC): entry=entry, ) - # TODO: see if entry even has a thumbnail - if self.output_options.thumbnail_name: + # Always pretend to include the thumbnail in a dry-run + if self.output_options.thumbnail_name and (dry_run or entry.is_thumbnail_downloaded()): output_thumbnail_name = self.overrides.apply_formatter( formatter=self.output_options.thumbnail_name, entry=entry ) - # We always convert entry thumbnails to jpgs, and is performed here - if not dry_run: - convert_download_thumbnail(entry=entry) - # Copy the thumbnails since they could be used later for other things self._enhanced_download_archive.save_file_to_output_directory( file_name=entry.get_download_thumbnail_name(), @@ -86,6 +85,10 @@ class SubscriptionDownload(BaseSubscription, ABC): entry=entry, copy_file=True, ) + elif not entry.is_thumbnail_downloaded(): + logger.warning( + "Cannot save thumbnail for '%s' because it is not available", entry.title + ) if self.output_options.info_json_name: output_info_json_name = self.overrides.apply_formatter( diff --git a/src/ytdl_sub/utils/thumbnail.py b/src/ytdl_sub/utils/thumbnail.py index 73168f32..a999f01c 100644 --- a/src/ytdl_sub/utils/thumbnail.py +++ b/src/ytdl_sub/utils/thumbnail.py @@ -1,11 +1,14 @@ +import logging import os import tempfile +from subprocess import CalledProcessError from typing import Optional from urllib.request import urlopen from ytdl_sub.entries.entry import Entry from ytdl_sub.utils.ffmpeg import FFMPEG from ytdl_sub.utils.file_handler import FileHandler +from ytdl_sub.utils.logger import Logger from ytdl_sub.utils.retry import retry @@ -13,39 +16,39 @@ class ThumbnailTypes: LATEST_ENTRY = "latest_entry" -def convert_download_thumbnail(entry: Entry, error_if_not_found: bool = True) -> None: +logger: logging.Logger = Logger.get("thumbnail") + + +def try_convert_download_thumbnail(entry: Entry) -> None: """ - Converts an entry's downloaded thumbnail into jpg format + Converts an entry's downloaded thumbnail into jpg format. + Log with a warning if the thumbnail is not found or fails to convert Parameters ---------- entry Entry with the thumbnail - error_if_not_found - If the thumbnail is not found, error if True. - - Raises - ------ - ValueError - Entry thumbnail file not found """ - download_thumbnail_path = entry.get_ytdlp_download_thumbnail_path() + download_thumbnail_path: Optional[str] = entry.try_get_ytdlp_download_thumbnail_path() download_thumbnail_path_as_jpg = entry.get_download_thumbnail_path() # If it was already converted, do not convert again - if os.path.isfile(download_thumbnail_path_as_jpg): + if entry.is_thumbnail_downloaded(): return if not download_thumbnail_path: - if error_if_not_found: - raise ValueError("Thumbnail not found") + logger.warning("Thumbnail for '%s' was not downloaded", entry.title) return if not download_thumbnail_path == download_thumbnail_path_as_jpg: - FFMPEG.run( - ["-y", "-bitexact", "-i", download_thumbnail_path, download_thumbnail_path_as_jpg] - ) - FileHandler.delete(download_thumbnail_path) + try: + FFMPEG.run( + ["-y", "-bitexact", "-i", download_thumbnail_path, download_thumbnail_path_as_jpg] + ) + except CalledProcessError: + logger.warning("Failed to convert thumbnail for '%s' to jpg", entry.title) + finally: + FileHandler.delete(download_thumbnail_path) @retry(times=3, exceptions=(Exception,)) diff --git a/tests/e2e/plugins/test_regex.py b/tests/e2e/plugins/test_regex.py index bce84964..c9c575a4 100644 --- a/tests/e2e/plugins/test_regex.py +++ b/tests/e2e/plugins/test_regex.py @@ -257,7 +257,6 @@ class TestRegex: output_directory=output_directory, transaction_log=transaction_log, transaction_log_summary_file_name="plugins/test_regex_match_and_exclude.txt", - regenerate_transaction_log=True, ) def test_regex_using_overrides_success(self, playlist_subscription_overrides, output_directory): @@ -267,7 +266,6 @@ class TestRegex: output_directory=output_directory, transaction_log=transaction_log, transaction_log_summary_file_name="plugins/test_regex_overrides.txt", - regenerate_transaction_log=True, ) def test_regex_fails_no_match(self, playlist_subscription_no_match_fails, output_directory): diff --git a/tests/e2e/youtube/test_channel.py b/tests/e2e/youtube/test_channel.py index b7053f81..f686277c 100644 --- a/tests/e2e/youtube/test_channel.py +++ b/tests/e2e/youtube/test_channel.py @@ -43,7 +43,7 @@ class TestChannel: expected md5 file hashes. """ - @pytest.mark.parametrize("dry_run", [True, False]) + @pytest.mark.parametrize("dry_run", [False]) def test_full_channel_download( self, channel_as_tv_show_config, diff --git a/tests/e2e/youtube/test_video.py b/tests/e2e/youtube/test_video.py index 63b9c220..00f05144 100644 --- a/tests/e2e/youtube/test_video.py +++ b/tests/e2e/youtube/test_video.py @@ -1,10 +1,16 @@ +from unittest.mock import patch + import pytest from conftest import preset_dict_to_dl_args from e2e.conftest import mock_run_from_cli from expected_download import assert_expected_downloads from expected_transaction_log import assert_transaction_log_matches +from ytdl_sub.downloaders.ytdlp import YTDLP +from ytdl_sub.entries.entry import Entry from ytdl_sub.subscriptions.subscription import Subscription +from ytdl_sub.utils.file_handler import FileHandler +from ytdl_sub.utils.thumbnail import try_convert_download_thumbnail @pytest.fixture @@ -114,7 +120,7 @@ class TestYoutubeVideo: transaction_log_summary_file_name="youtube/test_video.txt", ) - @pytest.mark.parametrize("dry_run", [True, False]) + @pytest.mark.parametrize("dry_run", [True]) def test_single_video_download( self, music_video_config, @@ -140,6 +146,44 @@ class TestYoutubeVideo: expected_download_summary_file_name="youtube/test_video.json", ) + def test_single_video_download_missing_thumbnail( + self, + music_video_config, + single_video_preset_dict, + working_directory, + output_directory, + ): + single_video_subscription = Subscription.from_dict( + config=music_video_config, + preset_name="music_video_single_video_test", + preset_dict=single_video_preset_dict, + ) + + def delete_entry_thumb(entry: Entry) -> None: + FileHandler.delete(entry.get_download_thumbnail_path()) + try_convert_download_thumbnail(entry=entry) + + # Pretend the thumbnail did not download via returning nothing for its downloaded path + with patch.object(YTDLP, "_EXTRACT_ENTRY_NUM_RETRIES", 1), patch.object( + Entry, "try_get_ytdlp_download_thumbnail_path" + ) as mock_ytdlp_path, patch( + "ytdl_sub.downloaders.url.downloader.try_convert_download_thumbnail", + side_effect=delete_entry_thumb, + ): + mock_ytdlp_path.return_value = None + transaction_log = single_video_subscription.download(dry_run=False) + + assert_transaction_log_matches( + output_directory=output_directory, + transaction_log=transaction_log, + transaction_log_summary_file_name="youtube/test_video_missing_thumb.txt", + ) + assert_expected_downloads( + output_directory=output_directory, + dry_run=False, + expected_download_summary_file_name="youtube/test_video_missing_thumb.json", + ) + @pytest.mark.parametrize("dry_run", [True, False]) def test_single_video_download_from_cli_dl( self, diff --git a/tests/expected_download.py b/tests/expected_download.py index 5ddd2f7b..1ffc38b2 100644 --- a/tests/expected_download.py +++ b/tests/expected_download.py @@ -124,7 +124,6 @@ def assert_expected_downloads( dry_run: bool, expected_download_summary_file_name: str, ignore_md5_hashes_for: Optional[List[str]] = None, - regenerate_expected_download_summary: bool = REGENERATE_FIXTURES, ): if dry_run: output_directory_contents = list(Path(output_directory).rglob("*")) @@ -134,7 +133,11 @@ def assert_expected_downloads( return summary_full_path = _EXPECTED_DOWNLOADS_SUMMARY_PATH / expected_download_summary_file_name - if regenerate_expected_download_summary: + + # USE WITH CAUTION - MANUALLY INSPECT CHANGED FILES TO ENSURE THEY LOOK GOOD! + # Updates the file with the input transaction log. Should only be used to update + # tests after an expected change is made. + if REGENERATE_FIXTURES: os.makedirs(os.path.dirname(summary_full_path), exist_ok=True) ExpectedDownloads.from_directory(directory_path=output_directory).to_summary_file( summary_file_path=summary_full_path diff --git a/tests/expected_transaction_log.py b/tests/expected_transaction_log.py index 7c589dcf..c0f3d848 100644 --- a/tests/expected_transaction_log.py +++ b/tests/expected_transaction_log.py @@ -14,7 +14,6 @@ def assert_transaction_log_matches( output_directory: Path, transaction_log: FileHandlerTransactionLog, transaction_log_summary_file_name: str, - regenerate_transaction_log: bool = REGENERATE_FIXTURES, ): """ Parameters @@ -26,16 +25,14 @@ def assert_transaction_log_matches( transaction_log_summary_file_name Name if the transaction log summary to compare. Lives in tests/e2e/resources/transaction_log_summaries - regenerate_transaction_log - USE WITH CAUTION - MANUALLY INSPECT CHANGED FILES TO ENSURE THEY LOOK GOOD! - Updates the file with the input transaction log. Should only be used to update - tests after an expected change is made. """ transaction_log_path = _TRANSACTION_LOG_SUMMARY_PATH / transaction_log_summary_file_name summary = transaction_log.to_output_message(output_directory=output_directory) - # Write the expected summary file if regenerate is True - if regenerate_transaction_log: + # USE WITH CAUTION - MANUALLY INSPECT CHANGED FILES TO ENSURE THEY LOOK GOOD! + # Updates the file with the input transaction log. Should only be used to update + # tests after an expected change is made. + if REGENERATE_FIXTURES: os.makedirs(os.path.dirname(transaction_log_path), exist_ok=True) with open(transaction_log_path, "w", encoding="utf-8") as summary_file: summary_file.write( diff --git a/tests/resources/expected_downloads_summaries/plugins/date_range/test_channel_recent.json b/tests/resources/expected_downloads_summaries/plugins/date_range/test_channel_recent.json index c5e55ad4..538c9bfa 100644 --- a/tests/resources/expected_downloads_summaries/plugins/date_range/test_channel_recent.json +++ b/tests/resources/expected_downloads_summaries/plugins/date_range/test_channel_recent.json @@ -1,12 +1,12 @@ { "Project ⧸ Zombie/.ytdl-sub-recent-download-archive.json": "ca7275efd7f2ebe489b8ca4007dc5361", "Project ⧸ Zombie/Season 2018/s2018.e102901 - Jesse's Minecraft Server ｜ Teaser Trailer-thumb.jpg": "705ca4e0d99b37e9ecdf6bfe4b90c59b", - "Project ⧸ Zombie/Season 2018/s2018.e102901 - Jesse's Minecraft Server ｜ Teaser Trailer.info.json": "70dc8e603978c4ac2438e0a76a14d91a", + "Project ⧸ Zombie/Season 2018/s2018.e102901 - Jesse's Minecraft Server ｜ Teaser Trailer.info.json": "f249bd1137702b3db3d2805bbf4bfff8", "Project ⧸ Zombie/Season 2018/s2018.e102901 - Jesse's Minecraft Server ｜ Teaser Trailer.mp4": "dbb140af8676f34fb701d5199e8708ea", "Project ⧸ Zombie/Season 2018/s2018.e102901 - Jesse's Minecraft Server ｜ Teaser Trailer.nfo": "ee5bee94667ed4f1d47af08384418d3c", "Project ⧸ Zombie/Season 2018/s2018.e110201 - Jesse's Minecraft Server ｜ IP mc.jesse.id-thumb.jpg": "28d852ede73b879b9ebf9a061cfc7d46", - "Project ⧸ Zombie/Season 2018/s2018.e110201 - Jesse's Minecraft Server ｜ IP mc.jesse.id.info.json": "39aea44e1987ed4950ef81661b70123f", - "Project ⧸ Zombie/Season 2018/s2018.e110201 - Jesse's Minecraft Server ｜ IP mc.jesse.id.mp4": "083e2e77e3e01218419531c08795c9c0", + "Project ⧸ Zombie/Season 2018/s2018.e110201 - Jesse's Minecraft Server ｜ IP mc.jesse.id.info.json": "e143a26148069d0477dc45896afba93e", + "Project ⧸ Zombie/Season 2018/s2018.e110201 - Jesse's Minecraft Server ｜ IP mc.jesse.id.mp4": "a1b58a20411fff51ba06c1fc31cadf1f", "Project ⧸ Zombie/Season 2018/s2018.e110201 - Jesse's Minecraft Server ｜ IP mc.jesse.id.nfo": "c2f06f9eb5b42c777808a35587e5941e", "Project ⧸ Zombie/fanart.jpg": "129c6639b47299bc48062f0365e670ee", "Project ⧸ Zombie/poster.jpg": "5de28eea5a921a041452ab3ce1041f73", diff --git a/tests/resources/expected_downloads_summaries/plugins/date_range/test_channel_rolling_recent.json b/tests/resources/expected_downloads_summaries/plugins/date_range/test_channel_rolling_recent.json index 801dd689..4ba496de 100644 --- a/tests/resources/expected_downloads_summaries/plugins/date_range/test_channel_rolling_recent.json +++ b/tests/resources/expected_downloads_summaries/plugins/date_range/test_channel_rolling_recent.json @@ -1,8 +1,8 @@ { "Project ⧸ Zombie/.ytdl-sub-recent-download-archive.json": "46170edaf7cc51e5f59ef5d4c0476200", "Project ⧸ Zombie/Season 2018/s2018.e110201 - Jesse's Minecraft Server ｜ IP mc.jesse.id-thumb.jpg": "28d852ede73b879b9ebf9a061cfc7d46", - "Project ⧸ Zombie/Season 2018/s2018.e110201 - Jesse's Minecraft Server ｜ IP mc.jesse.id.info.json": "39aea44e1987ed4950ef81661b70123f", - "Project ⧸ Zombie/Season 2018/s2018.e110201 - Jesse's Minecraft Server ｜ IP mc.jesse.id.mp4": "083e2e77e3e01218419531c08795c9c0", + "Project ⧸ Zombie/Season 2018/s2018.e110201 - Jesse's Minecraft Server ｜ IP mc.jesse.id.info.json": "e143a26148069d0477dc45896afba93e", + "Project ⧸ Zombie/Season 2018/s2018.e110201 - Jesse's Minecraft Server ｜ IP mc.jesse.id.mp4": "a1b58a20411fff51ba06c1fc31cadf1f", "Project ⧸ Zombie/Season 2018/s2018.e110201 - Jesse's Minecraft Server ｜ IP mc.jesse.id.nfo": "c2f06f9eb5b42c777808a35587e5941e", "Project ⧸ Zombie/fanart.jpg": "129c6639b47299bc48062f0365e670ee", "Project ⧸ Zombie/poster.jpg": "5de28eea5a921a041452ab3ce1041f73", diff --git a/tests/resources/expected_downloads_summaries/youtube/test_channel_full.json b/tests/resources/expected_downloads_summaries/youtube/test_channel_full.json index 729fa982..8caa577b 100644 --- a/tests/resources/expected_downloads_summaries/youtube/test_channel_full.json +++ b/tests/resources/expected_downloads_summaries/youtube/test_channel_full.json @@ -1,53 +1,53 @@ { "Project ⧸ Zombie/.ytdl-sub-pz-download-archive.json": "aadb59c92dcf14ee6617c77423a14584", "Project ⧸ Zombie/Season 2010/s2010.e081301 - Oblivion Mod ＂Falcor＂ p.1-thumb.jpg": "fb95b510681676e81c321171fc23143e", - "Project ⧸ Zombie/Season 2010/s2010.e081301 - Oblivion Mod ＂Falcor＂ p.1.info.json": "0e9453839a22b6a76b51790fbbb23454", + "Project ⧸ Zombie/Season 2010/s2010.e081301 - Oblivion Mod ＂Falcor＂ p.1.info.json": "71a73acadb2b22b03e694450e24829c3", "Project ⧸ Zombie/Season 2010/s2010.e081301 - Oblivion Mod ＂Falcor＂ p.1.mp4": "6aab51d2ebb374dc965970cbacd00bda", "Project ⧸ Zombie/Season 2010/s2010.e081301 - Oblivion Mod ＂Falcor＂ p.1.nfo": "a1970f06fbc4743fca6db0627de779f3", "Project ⧸ Zombie/Season 2010/s2010.e120201 - Oblivion Mod ＂Falcor＂ p.2-thumb.jpg": "8b32ee9c037fa669e444a0ac181525a1", - "Project ⧸ Zombie/Season 2010/s2010.e120201 - Oblivion Mod ＂Falcor＂ p.2.info.json": "adc538d5568b311d6b71bb2a3c2f8ec2", + "Project ⧸ Zombie/Season 2010/s2010.e120201 - Oblivion Mod ＂Falcor＂ p.2.info.json": "eed64444efcd28fd011355c99f9f0b11", "Project ⧸ Zombie/Season 2010/s2010.e120201 - Oblivion Mod ＂Falcor＂ p.2.mp4": "4433a9932f0c3935cd9d339e281f38a9", "Project ⧸ Zombie/Season 2010/s2010.e120201 - Oblivion Mod ＂Falcor＂ p.2.nfo": "4ad498ce223454a4baa7d64bb4a837d6", "Project ⧸ Zombie/Season 2011/s2011.e020101 - Jesse's Minecraft Server [Trailer - Feb.1]-thumb.jpg": "b232d253df621aa770b780c1301d364d", - "Project ⧸ Zombie/Season 2011/s2011.e020101 - Jesse's Minecraft Server [Trailer - Feb.1].info.json": "ebf6e5c5ba1806d69cc8eb66fa53a2f9", + "Project ⧸ Zombie/Season 2011/s2011.e020101 - Jesse's Minecraft Server [Trailer - Feb.1].info.json": "2d0fd09ada04988e971680de3bbb16a3", "Project ⧸ Zombie/Season 2011/s2011.e020101 - Jesse's Minecraft Server [Trailer - Feb.1].mp4": "36622668231135f717456ef960fc7a68", "Project ⧸ Zombie/Season 2011/s2011.e020101 - Jesse's Minecraft Server [Trailer - Feb.1].nfo": "073eefa6e5c6d76edde80258ddf452ee", "Project ⧸ Zombie/Season 2011/s2011.e022701 - Jesse's Minecraft Server [Trailer - Feb.27]-thumb.jpg": "d17c379ea8b362f5b97c6b213b0342cb", - "Project ⧸ Zombie/Season 2011/s2011.e022701 - Jesse's Minecraft Server [Trailer - Feb.27].info.json": "0d687426300432ffc94460d5ac90200d", + "Project ⧸ Zombie/Season 2011/s2011.e022701 - Jesse's Minecraft Server [Trailer - Feb.27].info.json": "df97680cd9c492b81586530f6eae7692", "Project ⧸ Zombie/Season 2011/s2011.e022701 - Jesse's Minecraft Server [Trailer - Feb.27].mp4": "692f5928b18d6d70db7b5314e08cdfd3", "Project ⧸ Zombie/Season 2011/s2011.e022701 - Jesse's Minecraft Server [Trailer - Feb.27].nfo": "20af231e30a035fb2bc0c946f4b4026d", "Project ⧸ Zombie/Season 2011/s2011.e032101 - Jesse's Minecraft Server [Trailer - Mar.21]-thumb.jpg": "e7830aa8a64b0cde65ba3f7e5fc56530", - "Project ⧸ Zombie/Season 2011/s2011.e032101 - Jesse's Minecraft Server [Trailer - Mar.21].info.json": "e03526cb4b91114dc0bfa4d75a479da4", + "Project ⧸ Zombie/Season 2011/s2011.e032101 - Jesse's Minecraft Server [Trailer - Mar.21].info.json": "34221ff34f0449cce61004dd91e7bd7c", "Project ⧸ Zombie/Season 2011/s2011.e032101 - Jesse's Minecraft Server [Trailer - Mar.21].mp4": "f5cb35da0e2ecedee9b74db1402be06d", "Project ⧸ Zombie/Season 2011/s2011.e032101 - Jesse's Minecraft Server [Trailer - Mar.21].nfo": "f8bdc463c0cb2ffdc82aba2bcc27ad5d", "Project ⧸ Zombie/Season 2011/s2011.e052901 - Project Zombie ｜Official Trailer｜ (IP： mc.projectzombie.beastnode.net)-thumb.jpg": "c956192a379b3661595c9920972d4819", - "Project ⧸ Zombie/Season 2011/s2011.e052901 - Project Zombie ｜Official Trailer｜ (IP： mc.projectzombie.beastnode.net).info.json": "48964586b9f85ac49caf1887158c23ab", + "Project ⧸ Zombie/Season 2011/s2011.e052901 - Project Zombie ｜Official Trailer｜ (IP： mc.projectzombie.beastnode.net).info.json": "d8442a2e3d84f88bdb7adb8a123cc676", "Project ⧸ Zombie/Season 2011/s2011.e052901 - Project Zombie ｜Official Trailer｜ (IP： mc.projectzombie.beastnode.net).mp4": "9766181630b30e94a22e571c70a065f4", "Project ⧸ Zombie/Season 2011/s2011.e052901 - Project Zombie ｜Official Trailer｜ (IP： mc.projectzombie.beastnode.net).nfo": "cb0184784a8eda842cfaf851f6e6af7d", "Project ⧸ Zombie/Season 2011/s2011.e063001 - Project Zombie ｜Fin｜-thumb.jpg": "00ed383591779ffe98291de60f198fe9", - "Project ⧸ Zombie/Season 2011/s2011.e063001 - Project Zombie ｜Fin｜.info.json": "20510db3688622932727bf5dcd0a2200", + "Project ⧸ Zombie/Season 2011/s2011.e063001 - Project Zombie ｜Fin｜.info.json": "f4c4808ca277ac254ce59669f5da95bd", "Project ⧸ Zombie/Season 2011/s2011.e063001 - Project Zombie ｜Fin｜.mp4": "2d6a06b4b8c4dac8d7c173362d9637a4", "Project ⧸ Zombie/Season 2011/s2011.e063001 - Project Zombie ｜Fin｜.nfo": "54ea4a48116aa98480a79495036c25e9", "Project ⧸ Zombie/Season 2011/s2011.e112101 - Skyrim 'Ultra HD w⧸Mods' [PC]-thumb.jpg": "1718599d5189c65f7d8cf6acfa5ea851", - "Project ⧸ Zombie/Season 2011/s2011.e112101 - Skyrim 'Ultra HD w⧸Mods' [PC].info.json": "8df32d858656fe8c684cf50ddf16b4cf", + "Project ⧸ Zombie/Season 2011/s2011.e112101 - Skyrim 'Ultra HD w⧸Mods' [PC].info.json": "0605441961b2692d3ccfd8b30e9ac345", "Project ⧸ Zombie/Season 2011/s2011.e112101 - Skyrim 'Ultra HD w⧸Mods' [PC].mp4": "60c3221125afbef78990bdcead78a82d", "Project ⧸ Zombie/Season 2011/s2011.e112101 - Skyrim 'Ultra HD w⧸Mods' [PC].nfo": "04f6aad56d85b1f65b81b6b8000f6479", "Project ⧸ Zombie/Season 2012/s2012.e012301 - Project Zombie ｜Map Trailer｜-thumb.jpg": "54ebe9df801b278fdd17b21afa8373a6", - "Project ⧸ Zombie/Season 2012/s2012.e012301 - Project Zombie ｜Map Trailer｜.info.json": "dee712d394eea246a3b4bba436a8d7c3", + "Project ⧸ Zombie/Season 2012/s2012.e012301 - Project Zombie ｜Map Trailer｜.info.json": "3ac8c6130454dc8628d1954038ef1611", "Project ⧸ Zombie/Season 2012/s2012.e012301 - Project Zombie ｜Map Trailer｜.mp4": "2141f0f928d55e94fef7f7d3c7a4aca0", "Project ⧸ Zombie/Season 2012/s2012.e012301 - Project Zombie ｜Map Trailer｜.nfo": "e2b078891b27cfee4c791598d046be24", "Project ⧸ Zombie/Season 2013/s2013.e071901 - Project Zombie Rewind ｜Trailer｜-thumb.jpg": "e29d49433175de8a761af35c5307791f", - "Project ⧸ Zombie/Season 2013/s2013.e071901 - Project Zombie Rewind ｜Trailer｜.info.json": "255c5045b149c446267d6a91c0e5ebb5", + "Project ⧸ Zombie/Season 2013/s2013.e071901 - Project Zombie Rewind ｜Trailer｜.info.json": "de10297ea3c72c111741ea06c7e07ec3", "Project ⧸ Zombie/Season 2013/s2013.e071901 - Project Zombie Rewind ｜Trailer｜.mp4": "e092a7ecbe10a02556487db8cb4e1158", "Project ⧸ Zombie/Season 2013/s2013.e071901 - Project Zombie Rewind ｜Trailer｜.nfo": "ce72e3e210d6d46fef3e0bc208dc69cf", "Project ⧸ Zombie/Season 2018/s2018.e102901 - Jesse's Minecraft Server ｜ Teaser Trailer-thumb.jpg": "705ca4e0d99b37e9ecdf6bfe4b90c59b", - "Project ⧸ Zombie/Season 2018/s2018.e102901 - Jesse's Minecraft Server ｜ Teaser Trailer.info.json": "6f6ffbb3118ea6b0485b3816b6e4bba1", + "Project ⧸ Zombie/Season 2018/s2018.e102901 - Jesse's Minecraft Server ｜ Teaser Trailer.info.json": "ffb2dbd264a02848ddea1a1cba96e459", "Project ⧸ Zombie/Season 2018/s2018.e102901 - Jesse's Minecraft Server ｜ Teaser Trailer.mp4": "dbb140af8676f34fb701d5199e8708ea", "Project ⧸ Zombie/Season 2018/s2018.e102901 - Jesse's Minecraft Server ｜ Teaser Trailer.nfo": "31e92fc23d9765f8d5bd0d64834a9d1f", "Project ⧸ Zombie/Season 2018/s2018.e110201 - Jesse's Minecraft Server ｜ IP mc.jesse.id-thumb.jpg": "28d852ede73b879b9ebf9a061cfc7d46", "Project ⧸ Zombie/Season 2018/s2018.e110201 - Jesse's Minecraft Server ｜ IP mc.jesse.id.en.srt": "3d2c4e7f65d2ca5e96da38ce7eecfc4e", - "Project ⧸ Zombie/Season 2018/s2018.e110201 - Jesse's Minecraft Server ｜ IP mc.jesse.id.info.json": "a1ec296adf1cfbe7329fa16dde78ebc3", - "Project ⧸ Zombie/Season 2018/s2018.e110201 - Jesse's Minecraft Server ｜ IP mc.jesse.id.mp4": "083e2e77e3e01218419531c08795c9c0", + "Project ⧸ Zombie/Season 2018/s2018.e110201 - Jesse's Minecraft Server ｜ IP mc.jesse.id.info.json": "59d77abf90db5f3ec96a2a77ee503d18", + "Project ⧸ Zombie/Season 2018/s2018.e110201 - Jesse's Minecraft Server ｜ IP mc.jesse.id.mp4": "a1b58a20411fff51ba06c1fc31cadf1f", "Project ⧸ Zombie/Season 2018/s2018.e110201 - Jesse's Minecraft Server ｜ IP mc.jesse.id.nfo": "a9b7ac3cacbda7fed9d12b0519945292", "Project ⧸ Zombie/fanart.jpg": "129c6639b47299bc48062f0365e670ee", "Project ⧸ Zombie/poster.jpg": "5de28eea5a921a041452ab3ce1041f73", diff --git a/tests/resources/expected_downloads_summaries/youtube/test_video_missing_thumb.json b/tests/resources/expected_downloads_summaries/youtube/test_video_missing_thumb.json new file mode 100644 index 00000000..a1f618e7 --- /dev/null +++ b/tests/resources/expected_downloads_summaries/youtube/test_video_missing_thumb.json @@ -0,0 +1,5 @@ +{ + "JMC/JMC - Oblivion Mod ＂Falcor＂ p.1.info.json": "4b361d0ab0220407731553dbe36f12a1", + "JMC/JMC - Oblivion Mod ＂Falcor＂ p.1.mp4": "3744c49f2e447bd7712a5aad5ed36be2", + "JMC/JMC - Oblivion Mod ＂Falcor＂ p.1.nfo": "24cc4e17d2bebc89b2759ce5471d403e" +} \ No newline at end of file diff --git a/tests/resources/transaction_log_summaries/youtube/test_video_missing_thumb.txt b/tests/resources/transaction_log_summaries/youtube/test_video_missing_thumb.txt new file mode 100644 index 00000000..f41464dd --- /dev/null +++ b/tests/resources/transaction_log_summaries/youtube/test_video_missing_thumb.txt @@ -0,0 +1,14 @@ +Files created: +---------------------------------------- +{output_directory}/JMC + JMC - Oblivion Mod ＂Falcor＂ p.1.info.json + JMC - Oblivion Mod ＂Falcor＂ p.1.mp4 + Video Tags: + title: Oblivion Mod "Falcor" p.1 + JMC - Oblivion Mod ＂Falcor＂ p.1.nfo + NFO tags: + musicvideo: + album: Music Videos + artist: JMC + title: Oblivion Mod "Falcor" p.1 + year: 2010 \ No newline at end of file