From 0f38f3ee77bb811e3cc15eecc65925de0f59f048 Mon Sep 17 00:00:00 2001 From: jbannon Date: Sat, 2 Jul 2022 06:38:30 +0000 Subject: [PATCH] fix youtube channel thumbnail issue, need to fix fixtures on test --- src/ytdl_sub/downloaders/downloader.py | 19 ++++---- .../downloaders/youtube_downloader.py | 15 +++--- src/ytdl_sub/plugins/plugin.py | 32 ++----------- src/ytdl_sub/subscriptions/subscription.py | 10 ++-- .../enhanced_download_archive.py | 46 +++++++++++++++++-- 5 files changed, 66 insertions(+), 56 deletions(-) diff --git a/src/ytdl_sub/downloaders/downloader.py b/src/ytdl_sub/downloaders/downloader.py index c376af9b..d44a9d62 100644 --- a/src/ytdl_sub/downloaders/downloader.py +++ b/src/ytdl_sub/downloaders/downloader.py @@ -20,6 +20,8 @@ from ytdl_sub.entries.base_entry import BaseEntry from ytdl_sub.entries.entry import Entry from ytdl_sub.utils.logger import Logger from ytdl_sub.validators.strict_dict_validator import StrictDictValidator +from ytdl_sub.ytdl_additions.enhanced_download_archive import DownloadArchiver +from ytdl_sub.ytdl_additions.enhanced_download_archive import EnhancedDownloadArchive logger = Logger.get(name="downloader") @@ -35,7 +37,7 @@ DownloaderEntryT = TypeVar("DownloaderEntryT", bound=Entry) DownloaderParentEntryT = TypeVar("DownloaderParentEntryT", bound=BaseEntry) -class Downloader(Generic[DownloaderOptionsT, DownloaderEntryT], ABC): +class Downloader(DownloadArchiver, Generic[DownloaderOptionsT, DownloaderEntryT], ABC): """ Class that interacts with ytdl to perform the download of metadata and content, and should translate that to list of Entry objects. @@ -84,21 +86,21 @@ class Downloader(Generic[DownloaderOptionsT, DownloaderEntryT], ABC): def __init__( self, - working_directory: str, download_options: DownloaderOptionsT, + enhanced_download_archive: EnhancedDownloadArchive, ytdl_options: Optional[Dict] = None, ): """ Parameters ---------- - working_directory - Path to the working directory download_options Options validator for this downloader + enhanced_download_archive + Download archive ytdl_options YTDL options validator """ - self.working_directory = working_directory + DownloadArchiver.__init__(self=self, enhanced_download_archive=enhanced_download_archive) self.download_options = download_options self.ytdl_options = self._configure_ytdl_options( ytdl_options=ytdl_options, @@ -198,19 +200,14 @@ class Downloader(Generic[DownloaderOptionsT, DownloaderEntryT], ABC): def download(self) -> List[DownloaderEntryT]: """The function to perform the download of all media entries""" - def post_download(self, overrides: Overrides, output_directory: str): + def post_download(self, overrides: Overrides): """ After all media entries have been downloaded, post processed, and moved to the output directory, run this function. This lets the downloader add any extra files directly to the output directory, for things like YT channel image, banner. - This ideally should not perform any extra downloads, but rather, use the content already - downloaded in the working directory and use it in the output directory. - Parameters ---------- overrides: Subscription overrides - output_directory: - Output directory to potentially store extra files downloaded """ diff --git a/src/ytdl_sub/downloaders/youtube_downloader.py b/src/ytdl_sub/downloaders/youtube_downloader.py index 8520e203..2a2c8ffd 100644 --- a/src/ytdl_sub/downloaders/youtube_downloader.py +++ b/src/ytdl_sub/downloaders/youtube_downloader.py @@ -20,6 +20,7 @@ from ytdl_sub.validators.string_formatter_validators import OverridesStringForma from ytdl_sub.validators.url_validator import YoutubeChannelUrlValidator from ytdl_sub.validators.url_validator import YoutubePlaylistUrlValidator from ytdl_sub.validators.url_validator import YoutubeVideoUrlValidator +from ytdl_sub.ytdl_additions.enhanced_download_archive import EnhancedDownloadArchive logger = Logger.get() @@ -290,13 +291,13 @@ class YoutubeChannelDownloader(YoutubeDownloader[YoutubeChannelDownloaderOptions def __init__( self, - working_directory: str, download_options: DownloaderOptionsT, + enhanced_download_archive: EnhancedDownloadArchive, ytdl_options: Optional[Dict] = None, ): super().__init__( - working_directory=working_directory, download_options=download_options, + enhanced_download_archive=enhanced_download_archive, ytdl_options=ytdl_options, ) self.channel: Optional[YoutubeChannel] = None @@ -352,7 +353,7 @@ class YoutubeChannelDownloader(YoutubeDownloader[YoutubeChannelDownloaderOptions thumbnail_url=thumbnail_url, output_thumbnail_path=output_thumbnail_path ) - def post_download(self, overrides: Overrides, output_directory: str): + def post_download(self, overrides: Overrides): """ Downloads and moves channel avatar and banner images to the output directory. @@ -360,17 +361,17 @@ class YoutubeChannelDownloader(YoutubeDownloader[YoutubeChannelDownloaderOptions ---------- overrides Overrides that can contain variables in the avatar or banner file path - output_directory - Output directory path """ avatar_thumbnail_name = overrides.apply_formatter(self.download_options.channel_avatar_path) self._download_thumbnail( thumbnail_url=self.channel.avatar_thumbnail_url(), - output_thumbnail_path=str(Path(output_directory) / avatar_thumbnail_name), + output_thumbnail_path=str(Path(self.working_directory) / avatar_thumbnail_name), ) + self.save_file(file_name=avatar_thumbnail_name) banner_thumbnail_name = overrides.apply_formatter(self.download_options.channel_banner_path) self._download_thumbnail( thumbnail_url=self.channel.banner_thumbnail_url(), - output_thumbnail_path=str(Path(output_directory) / banner_thumbnail_name), + output_thumbnail_path=str(Path(self.working_directory) / banner_thumbnail_name), ) + self.save_file(file_name=banner_thumbnail_name) diff --git a/src/ytdl_sub/plugins/plugin.py b/src/ytdl_sub/plugins/plugin.py index 65e916da..fa17191e 100644 --- a/src/ytdl_sub/plugins/plugin.py +++ b/src/ytdl_sub/plugins/plugin.py @@ -10,6 +10,7 @@ from ytdl_sub.entries.entry import Entry from ytdl_sub.utils.file_handler import FileMetadata from ytdl_sub.utils.logger import Logger from ytdl_sub.validators.strict_dict_validator import StrictDictValidator +from ytdl_sub.ytdl_additions.enhanced_download_archive import DownloadArchiver from ytdl_sub.ytdl_additions.enhanced_download_archive import EnhancedDownloadArchive @@ -22,7 +23,7 @@ class PluginOptions(StrictDictValidator): PluginOptionsT = TypeVar("PluginOptionsT", bound=PluginOptions) -class Plugin(Generic[PluginOptionsT], ABC): +class Plugin(DownloadArchiver, Generic[PluginOptionsT], ABC): """ Class to define the new plugin functionality """ @@ -36,39 +37,12 @@ class Plugin(Generic[PluginOptionsT], ABC): overrides: Overrides, enhanced_download_archive: EnhancedDownloadArchive, ): + DownloadArchiver.__init__(self=self, enhanced_download_archive=enhanced_download_archive) self.plugin_options = plugin_options self.overrides = overrides - self.__enhanced_download_archive = enhanced_download_archive # TODO pass yaml snake case name in the class somewhere, and use it for the logger self._logger = Logger.get(self.__class__.__name__) - @property - def working_directory(self) -> str: - return self.__enhanced_download_archive.working_directory - - @property - def output_directory(self) -> str: - return self.__enhanced_download_archive.output_directory - - @property - def is_dry_run(self) -> bool: - return self.__enhanced_download_archive.is_dry_run - - def save_file(self, file_name: str, entry: Optional[Entry] = None) -> None: - """ - Saves a file in the working directory to the output directory. - - Parameters - ---------- - file_name - Name of the file relative to the working directory - entry - Optional. Entry that the file belongs to - """ - self.__enhanced_download_archive.save_file( - file_name=file_name, output_file_name=file_name, entry=entry - ) - def post_process_entry(self, entry: Entry) -> Optional[FileMetadata]: """ For each file downloaded, apply post processing to it. diff --git a/src/ytdl_sub/subscriptions/subscription.py b/src/ytdl_sub/subscriptions/subscription.py index ce6625a2..7f36e902 100644 --- a/src/ytdl_sub/subscriptions/subscription.py +++ b/src/ytdl_sub/subscriptions/subscription.py @@ -159,7 +159,7 @@ class Subscription: output_file_name = self.overrides.apply_formatter( formatter=self.output_options.file_name, entry=entry ) - self._enhanced_download_archive.save_file( + self._enhanced_download_archive.save_file_to_output_directory( file_name=entry.get_download_file_name(), output_file_name=output_file_name, entry=entry ) @@ -171,7 +171,7 @@ class Subscription: # We always convert entry thumbnails to jpgs, and is performed here convert_download_thumbnail(entry=entry) - self._enhanced_download_archive.save_file( + self._enhanced_download_archive.save_file_to_output_directory( file_name=entry.get_download_thumbnail_name(), output_file_name=output_thumbnail_name, entry=entry, @@ -252,8 +252,8 @@ class Subscription: plugins = self._initialize_plugins() with self._prepare_working_directory(), self._maintain_archive_file(): downloader = self.downloader_class( - working_directory=self.working_directory, download_options=self.downloader_options, + enhanced_download_archive=self._enhanced_download_archive, ytdl_options=ytdl_options, ) @@ -267,9 +267,7 @@ class Subscription: for entry in entries: self._copy_entry_files_to_output_directory(entry=entry) - downloader.post_download( - overrides=self.overrides, output_directory=self.output_directory - ) + downloader.post_download(overrides=self.overrides) return self._enhanced_download_archive.get_file_handler_transaction_log() diff --git a/src/ytdl_sub/ytdl_additions/enhanced_download_archive.py b/src/ytdl_sub/ytdl_additions/enhanced_download_archive.py index c637fa40..5f46bff6 100644 --- a/src/ytdl_sub/ytdl_additions/enhanced_download_archive.py +++ b/src/ytdl_sub/ytdl_additions/enhanced_download_archive.py @@ -541,11 +541,13 @@ class EnhancedDownloadArchive: # TODO: Make this cleaner. It writes the file to the working dir, the copies it to the # output dir. Should be just a single write self._download_mapping.to_file(output_json_file=self._mapping_working_file_path) - self.save_file(file_name=self._mapping_file_name, output_file_name=self._mapping_file_name) + self.save_file_to_output_directory(file_name=self._mapping_file_name) return self - def save_file(self, file_name: str, output_file_name: str, entry: Optional[Entry] = None): + def save_file_to_output_directory( + self, file_name: str, output_file_name: Optional[str] = None, entry: Optional[Entry] = None + ): """ Saves a file from the working directory to the output directory @@ -554,10 +556,14 @@ class EnhancedDownloadArchive: file_name Name of the file to move (does not include working directory path) output_file_name - Final name of the file in the output directory (does not include output directory path) + Optional. Final name of the file in the output directory (does not include output + directory path). If None, use the same working_directory file_name entry Optional. Entry that this file belongs to """ + if output_file_name is None: + output_file_name = file_name + if entry: self.mapping.add_entry(entry=entry, entry_file_path=output_file_name) @@ -567,3 +573,37 @@ class EnhancedDownloadArchive: def get_file_handler_transaction_log(self) -> FileHandlerTransactionLog: return self._file_handler.file_handler_transaction_log + + +class DownloadArchiver: + """ + Used for any class that saves files. Does not allow direct access to output_directory, + forcing the user of the class to use ``save_file`` so it gets archived and avoids any writes + during dry-run. + """ + + def __init__(self, enhanced_download_archive: EnhancedDownloadArchive): + self.__enhanced_download_archive = enhanced_download_archive + + @property + def working_directory(self) -> str: + return self.__enhanced_download_archive.working_directory + + @property + def is_dry_run(self) -> bool: + return self.__enhanced_download_archive.is_dry_run + + def save_file(self, file_name: str, entry: Optional[Entry] = None) -> None: + """ + Saves a file in the working directory to the output directory. + + Parameters + ---------- + file_name + Name of the file relative to the working directory + entry + Optional. Entry that the file belongs to + """ + self.__enhanced_download_archive.save_file_to_output_directory( + file_name=file_name, entry=entry + )