fix youtube channel thumbnail issue, need to fix fixtures on test
This commit is contained in:
parent
d89cb1c177
commit
0f38f3ee77
5 changed files with 66 additions and 56 deletions
|
|
@ -20,6 +20,8 @@ from ytdl_sub.entries.base_entry import BaseEntry
|
||||||
from ytdl_sub.entries.entry import Entry
|
from ytdl_sub.entries.entry import Entry
|
||||||
from ytdl_sub.utils.logger import Logger
|
from ytdl_sub.utils.logger import Logger
|
||||||
from ytdl_sub.validators.strict_dict_validator import StrictDictValidator
|
from ytdl_sub.validators.strict_dict_validator import StrictDictValidator
|
||||||
|
from ytdl_sub.ytdl_additions.enhanced_download_archive import DownloadArchiver
|
||||||
|
from ytdl_sub.ytdl_additions.enhanced_download_archive import EnhancedDownloadArchive
|
||||||
|
|
||||||
logger = Logger.get(name="downloader")
|
logger = Logger.get(name="downloader")
|
||||||
|
|
||||||
|
|
@ -35,7 +37,7 @@ DownloaderEntryT = TypeVar("DownloaderEntryT", bound=Entry)
|
||||||
DownloaderParentEntryT = TypeVar("DownloaderParentEntryT", bound=BaseEntry)
|
DownloaderParentEntryT = TypeVar("DownloaderParentEntryT", bound=BaseEntry)
|
||||||
|
|
||||||
|
|
||||||
class Downloader(Generic[DownloaderOptionsT, DownloaderEntryT], ABC):
|
class Downloader(DownloadArchiver, Generic[DownloaderOptionsT, DownloaderEntryT], ABC):
|
||||||
"""
|
"""
|
||||||
Class that interacts with ytdl to perform the download of metadata and content,
|
Class that interacts with ytdl to perform the download of metadata and content,
|
||||||
and should translate that to list of Entry objects.
|
and should translate that to list of Entry objects.
|
||||||
|
|
@ -84,21 +86,21 @@ class Downloader(Generic[DownloaderOptionsT, DownloaderEntryT], ABC):
|
||||||
|
|
||||||
def __init__(
|
def __init__(
|
||||||
self,
|
self,
|
||||||
working_directory: str,
|
|
||||||
download_options: DownloaderOptionsT,
|
download_options: DownloaderOptionsT,
|
||||||
|
enhanced_download_archive: EnhancedDownloadArchive,
|
||||||
ytdl_options: Optional[Dict] = None,
|
ytdl_options: Optional[Dict] = None,
|
||||||
):
|
):
|
||||||
"""
|
"""
|
||||||
Parameters
|
Parameters
|
||||||
----------
|
----------
|
||||||
working_directory
|
|
||||||
Path to the working directory
|
|
||||||
download_options
|
download_options
|
||||||
Options validator for this downloader
|
Options validator for this downloader
|
||||||
|
enhanced_download_archive
|
||||||
|
Download archive
|
||||||
ytdl_options
|
ytdl_options
|
||||||
YTDL options validator
|
YTDL options validator
|
||||||
"""
|
"""
|
||||||
self.working_directory = working_directory
|
DownloadArchiver.__init__(self=self, enhanced_download_archive=enhanced_download_archive)
|
||||||
self.download_options = download_options
|
self.download_options = download_options
|
||||||
self.ytdl_options = self._configure_ytdl_options(
|
self.ytdl_options = self._configure_ytdl_options(
|
||||||
ytdl_options=ytdl_options,
|
ytdl_options=ytdl_options,
|
||||||
|
|
@ -198,19 +200,14 @@ class Downloader(Generic[DownloaderOptionsT, DownloaderEntryT], ABC):
|
||||||
def download(self) -> List[DownloaderEntryT]:
|
def download(self) -> List[DownloaderEntryT]:
|
||||||
"""The function to perform the download of all media entries"""
|
"""The function to perform the download of all media entries"""
|
||||||
|
|
||||||
def post_download(self, overrides: Overrides, output_directory: str):
|
def post_download(self, overrides: Overrides):
|
||||||
"""
|
"""
|
||||||
After all media entries have been downloaded, post processed, and moved to the output
|
After all media entries have been downloaded, post processed, and moved to the output
|
||||||
directory, run this function. This lets the downloader add any extra files directly to the
|
directory, run this function. This lets the downloader add any extra files directly to the
|
||||||
output directory, for things like YT channel image, banner.
|
output directory, for things like YT channel image, banner.
|
||||||
|
|
||||||
This ideally should not perform any extra downloads, but rather, use the content already
|
|
||||||
downloaded in the working directory and use it in the output directory.
|
|
||||||
|
|
||||||
Parameters
|
Parameters
|
||||||
----------
|
----------
|
||||||
overrides:
|
overrides:
|
||||||
Subscription overrides
|
Subscription overrides
|
||||||
output_directory:
|
|
||||||
Output directory to potentially store extra files downloaded
|
|
||||||
"""
|
"""
|
||||||
|
|
|
||||||
|
|
@ -20,6 +20,7 @@ from ytdl_sub.validators.string_formatter_validators import OverridesStringForma
|
||||||
from ytdl_sub.validators.url_validator import YoutubeChannelUrlValidator
|
from ytdl_sub.validators.url_validator import YoutubeChannelUrlValidator
|
||||||
from ytdl_sub.validators.url_validator import YoutubePlaylistUrlValidator
|
from ytdl_sub.validators.url_validator import YoutubePlaylistUrlValidator
|
||||||
from ytdl_sub.validators.url_validator import YoutubeVideoUrlValidator
|
from ytdl_sub.validators.url_validator import YoutubeVideoUrlValidator
|
||||||
|
from ytdl_sub.ytdl_additions.enhanced_download_archive import EnhancedDownloadArchive
|
||||||
|
|
||||||
logger = Logger.get()
|
logger = Logger.get()
|
||||||
|
|
||||||
|
|
@ -290,13 +291,13 @@ class YoutubeChannelDownloader(YoutubeDownloader[YoutubeChannelDownloaderOptions
|
||||||
|
|
||||||
def __init__(
|
def __init__(
|
||||||
self,
|
self,
|
||||||
working_directory: str,
|
|
||||||
download_options: DownloaderOptionsT,
|
download_options: DownloaderOptionsT,
|
||||||
|
enhanced_download_archive: EnhancedDownloadArchive,
|
||||||
ytdl_options: Optional[Dict] = None,
|
ytdl_options: Optional[Dict] = None,
|
||||||
):
|
):
|
||||||
super().__init__(
|
super().__init__(
|
||||||
working_directory=working_directory,
|
|
||||||
download_options=download_options,
|
download_options=download_options,
|
||||||
|
enhanced_download_archive=enhanced_download_archive,
|
||||||
ytdl_options=ytdl_options,
|
ytdl_options=ytdl_options,
|
||||||
)
|
)
|
||||||
self.channel: Optional[YoutubeChannel] = None
|
self.channel: Optional[YoutubeChannel] = None
|
||||||
|
|
@ -352,7 +353,7 @@ class YoutubeChannelDownloader(YoutubeDownloader[YoutubeChannelDownloaderOptions
|
||||||
thumbnail_url=thumbnail_url, output_thumbnail_path=output_thumbnail_path
|
thumbnail_url=thumbnail_url, output_thumbnail_path=output_thumbnail_path
|
||||||
)
|
)
|
||||||
|
|
||||||
def post_download(self, overrides: Overrides, output_directory: str):
|
def post_download(self, overrides: Overrides):
|
||||||
"""
|
"""
|
||||||
Downloads and moves channel avatar and banner images to the output directory.
|
Downloads and moves channel avatar and banner images to the output directory.
|
||||||
|
|
||||||
|
|
@ -360,17 +361,17 @@ class YoutubeChannelDownloader(YoutubeDownloader[YoutubeChannelDownloaderOptions
|
||||||
----------
|
----------
|
||||||
overrides
|
overrides
|
||||||
Overrides that can contain variables in the avatar or banner file path
|
Overrides that can contain variables in the avatar or banner file path
|
||||||
output_directory
|
|
||||||
Output directory path
|
|
||||||
"""
|
"""
|
||||||
avatar_thumbnail_name = overrides.apply_formatter(self.download_options.channel_avatar_path)
|
avatar_thumbnail_name = overrides.apply_formatter(self.download_options.channel_avatar_path)
|
||||||
self._download_thumbnail(
|
self._download_thumbnail(
|
||||||
thumbnail_url=self.channel.avatar_thumbnail_url(),
|
thumbnail_url=self.channel.avatar_thumbnail_url(),
|
||||||
output_thumbnail_path=str(Path(output_directory) / avatar_thumbnail_name),
|
output_thumbnail_path=str(Path(self.working_directory) / avatar_thumbnail_name),
|
||||||
)
|
)
|
||||||
|
self.save_file(file_name=avatar_thumbnail_name)
|
||||||
|
|
||||||
banner_thumbnail_name = overrides.apply_formatter(self.download_options.channel_banner_path)
|
banner_thumbnail_name = overrides.apply_formatter(self.download_options.channel_banner_path)
|
||||||
self._download_thumbnail(
|
self._download_thumbnail(
|
||||||
thumbnail_url=self.channel.banner_thumbnail_url(),
|
thumbnail_url=self.channel.banner_thumbnail_url(),
|
||||||
output_thumbnail_path=str(Path(output_directory) / banner_thumbnail_name),
|
output_thumbnail_path=str(Path(self.working_directory) / banner_thumbnail_name),
|
||||||
)
|
)
|
||||||
|
self.save_file(file_name=banner_thumbnail_name)
|
||||||
|
|
|
||||||
|
|
@ -10,6 +10,7 @@ from ytdl_sub.entries.entry import Entry
|
||||||
from ytdl_sub.utils.file_handler import FileMetadata
|
from ytdl_sub.utils.file_handler import FileMetadata
|
||||||
from ytdl_sub.utils.logger import Logger
|
from ytdl_sub.utils.logger import Logger
|
||||||
from ytdl_sub.validators.strict_dict_validator import StrictDictValidator
|
from ytdl_sub.validators.strict_dict_validator import StrictDictValidator
|
||||||
|
from ytdl_sub.ytdl_additions.enhanced_download_archive import DownloadArchiver
|
||||||
from ytdl_sub.ytdl_additions.enhanced_download_archive import EnhancedDownloadArchive
|
from ytdl_sub.ytdl_additions.enhanced_download_archive import EnhancedDownloadArchive
|
||||||
|
|
||||||
|
|
||||||
|
|
@ -22,7 +23,7 @@ class PluginOptions(StrictDictValidator):
|
||||||
PluginOptionsT = TypeVar("PluginOptionsT", bound=PluginOptions)
|
PluginOptionsT = TypeVar("PluginOptionsT", bound=PluginOptions)
|
||||||
|
|
||||||
|
|
||||||
class Plugin(Generic[PluginOptionsT], ABC):
|
class Plugin(DownloadArchiver, Generic[PluginOptionsT], ABC):
|
||||||
"""
|
"""
|
||||||
Class to define the new plugin functionality
|
Class to define the new plugin functionality
|
||||||
"""
|
"""
|
||||||
|
|
@ -36,39 +37,12 @@ class Plugin(Generic[PluginOptionsT], ABC):
|
||||||
overrides: Overrides,
|
overrides: Overrides,
|
||||||
enhanced_download_archive: EnhancedDownloadArchive,
|
enhanced_download_archive: EnhancedDownloadArchive,
|
||||||
):
|
):
|
||||||
|
DownloadArchiver.__init__(self=self, enhanced_download_archive=enhanced_download_archive)
|
||||||
self.plugin_options = plugin_options
|
self.plugin_options = plugin_options
|
||||||
self.overrides = overrides
|
self.overrides = overrides
|
||||||
self.__enhanced_download_archive = enhanced_download_archive
|
|
||||||
# TODO pass yaml snake case name in the class somewhere, and use it for the logger
|
# TODO pass yaml snake case name in the class somewhere, and use it for the logger
|
||||||
self._logger = Logger.get(self.__class__.__name__)
|
self._logger = Logger.get(self.__class__.__name__)
|
||||||
|
|
||||||
@property
|
|
||||||
def working_directory(self) -> str:
|
|
||||||
return self.__enhanced_download_archive.working_directory
|
|
||||||
|
|
||||||
@property
|
|
||||||
def output_directory(self) -> str:
|
|
||||||
return self.__enhanced_download_archive.output_directory
|
|
||||||
|
|
||||||
@property
|
|
||||||
def is_dry_run(self) -> bool:
|
|
||||||
return self.__enhanced_download_archive.is_dry_run
|
|
||||||
|
|
||||||
def save_file(self, file_name: str, entry: Optional[Entry] = None) -> None:
|
|
||||||
"""
|
|
||||||
Saves a file in the working directory to the output directory.
|
|
||||||
|
|
||||||
Parameters
|
|
||||||
----------
|
|
||||||
file_name
|
|
||||||
Name of the file relative to the working directory
|
|
||||||
entry
|
|
||||||
Optional. Entry that the file belongs to
|
|
||||||
"""
|
|
||||||
self.__enhanced_download_archive.save_file(
|
|
||||||
file_name=file_name, output_file_name=file_name, entry=entry
|
|
||||||
)
|
|
||||||
|
|
||||||
def post_process_entry(self, entry: Entry) -> Optional[FileMetadata]:
|
def post_process_entry(self, entry: Entry) -> Optional[FileMetadata]:
|
||||||
"""
|
"""
|
||||||
For each file downloaded, apply post processing to it.
|
For each file downloaded, apply post processing to it.
|
||||||
|
|
|
||||||
|
|
@ -159,7 +159,7 @@ class Subscription:
|
||||||
output_file_name = self.overrides.apply_formatter(
|
output_file_name = self.overrides.apply_formatter(
|
||||||
formatter=self.output_options.file_name, entry=entry
|
formatter=self.output_options.file_name, entry=entry
|
||||||
)
|
)
|
||||||
self._enhanced_download_archive.save_file(
|
self._enhanced_download_archive.save_file_to_output_directory(
|
||||||
file_name=entry.get_download_file_name(), output_file_name=output_file_name, entry=entry
|
file_name=entry.get_download_file_name(), output_file_name=output_file_name, entry=entry
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
@ -171,7 +171,7 @@ class Subscription:
|
||||||
# We always convert entry thumbnails to jpgs, and is performed here
|
# We always convert entry thumbnails to jpgs, and is performed here
|
||||||
convert_download_thumbnail(entry=entry)
|
convert_download_thumbnail(entry=entry)
|
||||||
|
|
||||||
self._enhanced_download_archive.save_file(
|
self._enhanced_download_archive.save_file_to_output_directory(
|
||||||
file_name=entry.get_download_thumbnail_name(),
|
file_name=entry.get_download_thumbnail_name(),
|
||||||
output_file_name=output_thumbnail_name,
|
output_file_name=output_thumbnail_name,
|
||||||
entry=entry,
|
entry=entry,
|
||||||
|
|
@ -252,8 +252,8 @@ class Subscription:
|
||||||
plugins = self._initialize_plugins()
|
plugins = self._initialize_plugins()
|
||||||
with self._prepare_working_directory(), self._maintain_archive_file():
|
with self._prepare_working_directory(), self._maintain_archive_file():
|
||||||
downloader = self.downloader_class(
|
downloader = self.downloader_class(
|
||||||
working_directory=self.working_directory,
|
|
||||||
download_options=self.downloader_options,
|
download_options=self.downloader_options,
|
||||||
|
enhanced_download_archive=self._enhanced_download_archive,
|
||||||
ytdl_options=ytdl_options,
|
ytdl_options=ytdl_options,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
@ -267,9 +267,7 @@ class Subscription:
|
||||||
for entry in entries:
|
for entry in entries:
|
||||||
self._copy_entry_files_to_output_directory(entry=entry)
|
self._copy_entry_files_to_output_directory(entry=entry)
|
||||||
|
|
||||||
downloader.post_download(
|
downloader.post_download(overrides=self.overrides)
|
||||||
overrides=self.overrides, output_directory=self.output_directory
|
|
||||||
)
|
|
||||||
|
|
||||||
return self._enhanced_download_archive.get_file_handler_transaction_log()
|
return self._enhanced_download_archive.get_file_handler_transaction_log()
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -541,11 +541,13 @@ class EnhancedDownloadArchive:
|
||||||
# TODO: Make this cleaner. It writes the file to the working dir, the copies it to the
|
# TODO: Make this cleaner. It writes the file to the working dir, the copies it to the
|
||||||
# output dir. Should be just a single write
|
# output dir. Should be just a single write
|
||||||
self._download_mapping.to_file(output_json_file=self._mapping_working_file_path)
|
self._download_mapping.to_file(output_json_file=self._mapping_working_file_path)
|
||||||
self.save_file(file_name=self._mapping_file_name, output_file_name=self._mapping_file_name)
|
self.save_file_to_output_directory(file_name=self._mapping_file_name)
|
||||||
|
|
||||||
return self
|
return self
|
||||||
|
|
||||||
def save_file(self, file_name: str, output_file_name: str, entry: Optional[Entry] = None):
|
def save_file_to_output_directory(
|
||||||
|
self, file_name: str, output_file_name: Optional[str] = None, entry: Optional[Entry] = None
|
||||||
|
):
|
||||||
"""
|
"""
|
||||||
Saves a file from the working directory to the output directory
|
Saves a file from the working directory to the output directory
|
||||||
|
|
||||||
|
|
@ -554,10 +556,14 @@ class EnhancedDownloadArchive:
|
||||||
file_name
|
file_name
|
||||||
Name of the file to move (does not include working directory path)
|
Name of the file to move (does not include working directory path)
|
||||||
output_file_name
|
output_file_name
|
||||||
Final name of the file in the output directory (does not include output directory path)
|
Optional. Final name of the file in the output directory (does not include output
|
||||||
|
directory path). If None, use the same working_directory file_name
|
||||||
entry
|
entry
|
||||||
Optional. Entry that this file belongs to
|
Optional. Entry that this file belongs to
|
||||||
"""
|
"""
|
||||||
|
if output_file_name is None:
|
||||||
|
output_file_name = file_name
|
||||||
|
|
||||||
if entry:
|
if entry:
|
||||||
self.mapping.add_entry(entry=entry, entry_file_path=output_file_name)
|
self.mapping.add_entry(entry=entry, entry_file_path=output_file_name)
|
||||||
|
|
||||||
|
|
@ -567,3 +573,37 @@ class EnhancedDownloadArchive:
|
||||||
|
|
||||||
def get_file_handler_transaction_log(self) -> FileHandlerTransactionLog:
|
def get_file_handler_transaction_log(self) -> FileHandlerTransactionLog:
|
||||||
return self._file_handler.file_handler_transaction_log
|
return self._file_handler.file_handler_transaction_log
|
||||||
|
|
||||||
|
|
||||||
|
class DownloadArchiver:
|
||||||
|
"""
|
||||||
|
Used for any class that saves files. Does not allow direct access to output_directory,
|
||||||
|
forcing the user of the class to use ``save_file`` so it gets archived and avoids any writes
|
||||||
|
during dry-run.
|
||||||
|
"""
|
||||||
|
|
||||||
|
def __init__(self, enhanced_download_archive: EnhancedDownloadArchive):
|
||||||
|
self.__enhanced_download_archive = enhanced_download_archive
|
||||||
|
|
||||||
|
@property
|
||||||
|
def working_directory(self) -> str:
|
||||||
|
return self.__enhanced_download_archive.working_directory
|
||||||
|
|
||||||
|
@property
|
||||||
|
def is_dry_run(self) -> bool:
|
||||||
|
return self.__enhanced_download_archive.is_dry_run
|
||||||
|
|
||||||
|
def save_file(self, file_name: str, entry: Optional[Entry] = None) -> None:
|
||||||
|
"""
|
||||||
|
Saves a file in the working directory to the output directory.
|
||||||
|
|
||||||
|
Parameters
|
||||||
|
----------
|
||||||
|
file_name
|
||||||
|
Name of the file relative to the working directory
|
||||||
|
entry
|
||||||
|
Optional. Entry that the file belongs to
|
||||||
|
"""
|
||||||
|
self.__enhanced_download_archive.save_file_to_output_directory(
|
||||||
|
file_name=file_name, entry=entry
|
||||||
|
)
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue