danvics/ytdl-sub
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

fix youtube channel thumbnail issue, need to fix fixtures on test

Browse source
This commit is contained in:
jbannon 2022-07-02 06:38:30 +00:00
parent d89cb1c177
commit 0f38f3ee77
5 changed files with 66 additions and 56 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

19
src/ytdl_sub/downloaders/downloader.py
View file

@ -20,6 +20,8 @@ from ytdl_sub.entries.base_entry import BaseEntry
from ytdl_sub.entries.entry import Entry
from ytdl_sub.utils.logger import Logger
from ytdl_sub.validators.strict_dict_validator import StrictDictValidator
from ytdl_sub.ytdl_additions.enhanced_download_archive import DownloadArchiver
from ytdl_sub.ytdl_additions.enhanced_download_archive import EnhancedDownloadArchive
logger = Logger.get(name="downloader")
@ -35,7 +37,7 @@ DownloaderEntryT = TypeVar("DownloaderEntryT", bound=Entry)
DownloaderParentEntryT = TypeVar("DownloaderParentEntryT", bound=BaseEntry)
class Downloader(Generic[DownloaderOptionsT, DownloaderEntryT], ABC):
class Downloader(DownloadArchiver, Generic[DownloaderOptionsT, DownloaderEntryT], ABC):
"""
Class that interacts with ytdl to perform the download of metadata and content,
and should translate that to list of Entry objects.
@ -84,21 +86,21 @@ class Downloader(Generic[DownloaderOptionsT, DownloaderEntryT], ABC):
def __init__(
self,
working_directory: str,
download_options: DownloaderOptionsT,
enhanced_download_archive: EnhancedDownloadArchive,
ytdl_options: Optional[Dict] = None,
):
"""
Parameters
----------
working_directory
Path to the working directory
download_options
Options validator for this downloader
enhanced_download_archive
Download archive
ytdl_options
YTDL options validator
"""
self.working_directory = working_directory
DownloadArchiver.__init__(self=self, enhanced_download_archive=enhanced_download_archive)
self.download_options = download_options
self.ytdl_options = self._configure_ytdl_options(
ytdl_options=ytdl_options,
@ -198,19 +200,14 @@ class Downloader(Generic[DownloaderOptionsT, DownloaderEntryT], ABC):
def download(self) -> List[DownloaderEntryT]:
"""The function to perform the download of all media entries"""
def post_download(self, overrides: Overrides, output_directory: str):
def post_download(self, overrides: Overrides):
"""
After all media entries have been downloaded, post processed, and moved to the output
directory, run this function. This lets the downloader add any extra files directly to the
output directory, for things like YT channel image, banner.
This ideally should not perform any extra downloads, but rather, use the content already
downloaded in the working directory and use it in the output directory.
Parameters
----------
overrides:
Subscription overrides
output_directory:
Output directory to potentially store extra files downloaded
"""

15
src/ytdl_sub/downloaders/youtube_downloader.py
View file

@ -20,6 +20,7 @@ from ytdl_sub.validators.string_formatter_validators import OverridesStringForma
from ytdl_sub.validators.url_validator import YoutubeChannelUrlValidator
from ytdl_sub.validators.url_validator import YoutubePlaylistUrlValidator
from ytdl_sub.validators.url_validator import YoutubeVideoUrlValidator
from ytdl_sub.ytdl_additions.enhanced_download_archive import EnhancedDownloadArchive
logger = Logger.get()
@ -290,13 +291,13 @@ class YoutubeChannelDownloader(YoutubeDownloader[YoutubeChannelDownloaderOptions
def __init__(
self,
working_directory: str,
download_options: DownloaderOptionsT,
enhanced_download_archive: EnhancedDownloadArchive,
ytdl_options: Optional[Dict] = None,
):
super().__init__(
working_directory=working_directory,
download_options=download_options,
enhanced_download_archive=enhanced_download_archive,
ytdl_options=ytdl_options,
)
self.channel: Optional[YoutubeChannel] = None
@ -352,7 +353,7 @@ class YoutubeChannelDownloader(YoutubeDownloader[YoutubeChannelDownloaderOptions
thumbnail_url=thumbnail_url, output_thumbnail_path=output_thumbnail_path
)
def post_download(self, overrides: Overrides, output_directory: str):
def post_download(self, overrides: Overrides):
"""
Downloads and moves channel avatar and banner images to the output directory.
@ -360,17 +361,17 @@ class YoutubeChannelDownloader(YoutubeDownloader[YoutubeChannelDownloaderOptions
----------
overrides
Overrides that can contain variables in the avatar or banner file path
output_directory
Output directory path
"""
avatar_thumbnail_name = overrides.apply_formatter(self.download_options.channel_avatar_path)
self._download_thumbnail(
thumbnail_url=self.channel.avatar_thumbnail_url(),
output_thumbnail_path=str(Path(output_directory) / avatar_thumbnail_name),
output_thumbnail_path=str(Path(self.working_directory) / avatar_thumbnail_name),
)
self.save_file(file_name=avatar_thumbnail_name)
banner_thumbnail_name = overrides.apply_formatter(self.download_options.channel_banner_path)
self._download_thumbnail(
thumbnail_url=self.channel.banner_thumbnail_url(),
output_thumbnail_path=str(Path(output_directory) / banner_thumbnail_name),
output_thumbnail_path=str(Path(self.working_directory) / banner_thumbnail_name),
)
self.save_file(file_name=banner_thumbnail_name)

32
src/ytdl_sub/plugins/plugin.py
View file

@ -10,6 +10,7 @@ from ytdl_sub.entries.entry import Entry
from ytdl_sub.utils.file_handler import FileMetadata
from ytdl_sub.utils.logger import Logger
from ytdl_sub.validators.strict_dict_validator import StrictDictValidator
from ytdl_sub.ytdl_additions.enhanced_download_archive import DownloadArchiver
from ytdl_sub.ytdl_additions.enhanced_download_archive import EnhancedDownloadArchive
@ -22,7 +23,7 @@ class PluginOptions(StrictDictValidator):
PluginOptionsT = TypeVar("PluginOptionsT", bound=PluginOptions)
class Plugin(Generic[PluginOptionsT], ABC):
class Plugin(DownloadArchiver, Generic[PluginOptionsT], ABC):
"""
Class to define the new plugin functionality
"""
@ -36,39 +37,12 @@ class Plugin(Generic[PluginOptionsT], ABC):
overrides: Overrides,
enhanced_download_archive: EnhancedDownloadArchive,
):
DownloadArchiver.__init__(self=self, enhanced_download_archive=enhanced_download_archive)
self.plugin_options = plugin_options
self.overrides = overrides
self.__enhanced_download_archive = enhanced_download_archive
# TODO pass yaml snake case name in the class somewhere, and use it for the logger
self._logger = Logger.get(self.__class__.__name__)
@property
def working_directory(self) -> str:
return self.__enhanced_download_archive.working_directory
@property
def output_directory(self) -> str:
return self.__enhanced_download_archive.output_directory
@property
def is_dry_run(self) -> bool:
return self.__enhanced_download_archive.is_dry_run
def save_file(self, file_name: str, entry: Optional[Entry] = None) -> None:
"""
Saves a file in the working directory to the output directory.
Parameters
----------
file_name
Name of the file relative to the working directory
entry
Optional. Entry that the file belongs to
"""
self.__enhanced_download_archive.save_file(
file_name=file_name, output_file_name=file_name, entry=entry
)
def post_process_entry(self, entry: Entry) -> Optional[FileMetadata]:
"""
For each file downloaded, apply post processing to it.

10
src/ytdl_sub/subscriptions/subscription.py
View file

@ -159,7 +159,7 @@ class Subscription:
output_file_name = self.overrides.apply_formatter(
formatter=self.output_options.file_name, entry=entry
)
self._enhanced_download_archive.save_file(
self._enhanced_download_archive.save_file_to_output_directory(
file_name=entry.get_download_file_name(), output_file_name=output_file_name, entry=entry
)
@ -171,7 +171,7 @@ class Subscription:
# We always convert entry thumbnails to jpgs, and is performed here
convert_download_thumbnail(entry=entry)
self._enhanced_download_archive.save_file(
self._enhanced_download_archive.save_file_to_output_directory(
file_name=entry.get_download_thumbnail_name(),
output_file_name=output_thumbnail_name,
entry=entry,
@ -252,8 +252,8 @@ class Subscription:
plugins = self._initialize_plugins()
with self._prepare_working_directory(), self._maintain_archive_file():
downloader = self.downloader_class(
working_directory=self.working_directory,
download_options=self.downloader_options,
enhanced_download_archive=self._enhanced_download_archive,
ytdl_options=ytdl_options,
)
@ -267,9 +267,7 @@ class Subscription:
for entry in entries:
self._copy_entry_files_to_output_directory(entry=entry)
downloader.post_download(
overrides=self.overrides, output_directory=self.output_directory
)
downloader.post_download(overrides=self.overrides)
return self._enhanced_download_archive.get_file_handler_transaction_log()

46
src/ytdl_sub/ytdl_additions/enhanced_download_archive.py
View file

@ -541,11 +541,13 @@ class EnhancedDownloadArchive:
# TODO: Make this cleaner. It writes the file to the working dir, the copies it to the
# output dir. Should be just a single write
self._download_mapping.to_file(output_json_file=self._mapping_working_file_path)
self.save_file(file_name=self._mapping_file_name, output_file_name=self._mapping_file_name)
self.save_file_to_output_directory(file_name=self._mapping_file_name)
return self
def save_file(self, file_name: str, output_file_name: str, entry: Optional[Entry] = None):
def save_file_to_output_directory(
self, file_name: str, output_file_name: Optional[str] = None, entry: Optional[Entry] = None
):
"""
Saves a file from the working directory to the output directory
@ -554,10 +556,14 @@ class EnhancedDownloadArchive:
file_name
Name of the file to move (does not include working directory path)
output_file_name
Final name of the file in the output directory (does not include output directory path)
Optional. Final name of the file in the output directory (does not include output
directory path). If None, use the same working_directory file_name
entry
Optional. Entry that this file belongs to
"""
if output_file_name is None:
output_file_name = file_name
if entry:
self.mapping.add_entry(entry=entry, entry_file_path=output_file_name)
@ -567,3 +573,37 @@ class EnhancedDownloadArchive:
def get_file_handler_transaction_log(self) -> FileHandlerTransactionLog:
return self._file_handler.file_handler_transaction_log
class DownloadArchiver:
"""
Used for any class that saves files. Does not allow direct access to output_directory,
forcing the user of the class to use ``save_file`` so it gets archived and avoids any writes
during dry-run.
"""
def __init__(self, enhanced_download_archive: EnhancedDownloadArchive):
self.__enhanced_download_archive = enhanced_download_archive
@property
def working_directory(self) -> str:
return self.__enhanced_download_archive.working_directory
@property
def is_dry_run(self) -> bool:
return self.__enhanced_download_archive.is_dry_run
def save_file(self, file_name: str, entry: Optional[Entry] = None) -> None:
"""
Saves a file in the working directory to the output directory.
Parameters
----------
file_name
Name of the file relative to the working directory
entry
Optional. Entry that the file belongs to
"""
self.__enhanced_download_archive.save_file_to_output_directory(
file_name=file_name, entry=entry
)