diff --git a/examples/soundcloud_discography_config.yaml b/examples/soundcloud_discography_config.yaml index 13304eb4..5b10b3c8 100644 --- a/examples/soundcloud_discography_config.yaml +++ b/examples/soundcloud_discography_config.yaml @@ -43,9 +43,9 @@ presets: # store previously downloaded song IDs to tell YTDL not to re-download # them on a successive invocation. output_options: - output_directory: "/{music_directory}/{artist_sanitized}" - file_name: "{album_directory_name}/{track_number_padded} - {title_sanitized}.{ext}" - thumbnail_name: "{album_directory_name}/folder.jpg" + output_directory: "{music_directory}" + file_name: "{artist_sanitized}/{album_directory_name}/{track_number_padded} - {title_sanitized}.{ext}" + thumbnail_name: "{artist_sanitized}/{album_directory_name}/folder.jpg" maintain_download_archive: True # For each song downloaded, populate the audio file with music tags. diff --git a/src/ytdl_sub/cli/main_args_parser.py b/src/ytdl_sub/cli/main_args_parser.py index 38c7ad41..189c303a 100644 --- a/src/ytdl_sub/cli/main_args_parser.py +++ b/src/ytdl_sub/cli/main_args_parser.py @@ -1,9 +1,9 @@ import argparse +from ytdl_sub.utils.logger import LoggerLevels + ################################################################################################### # GLOBAL PARSER -from ytdl_sub.utils.logger import LoggerLevels - parser = argparse.ArgumentParser( description="ytdl-sub: Automate download and adding metadata with YoutubeDL" ) @@ -15,6 +15,11 @@ parser.add_argument( help="path to the config yaml, uses config.yaml if not provided", default="config.yaml", ) +parser.add_argument( + "--dry-run", + action="store_true", + help="does not perform any video downloads or writes to output directories", +) parser.add_argument( "--log-level", metavar="|".join(LoggerLevels.names()), diff --git a/src/ytdl_sub/downloaders/downloader.py b/src/ytdl_sub/downloaders/downloader.py index 6e9f3e66..d44a9d62 100644 --- a/src/ytdl_sub/downloaders/downloader.py +++ b/src/ytdl_sub/downloaders/downloader.py @@ -20,6 +20,8 @@ from ytdl_sub.entries.base_entry import BaseEntry from ytdl_sub.entries.entry import Entry from ytdl_sub.utils.logger import Logger from ytdl_sub.validators.strict_dict_validator import StrictDictValidator +from ytdl_sub.ytdl_additions.enhanced_download_archive import DownloadArchiver +from ytdl_sub.ytdl_additions.enhanced_download_archive import EnhancedDownloadArchive logger = Logger.get(name="downloader") @@ -35,7 +37,7 @@ DownloaderEntryT = TypeVar("DownloaderEntryT", bound=Entry) DownloaderParentEntryT = TypeVar("DownloaderParentEntryT", bound=BaseEntry) -class Downloader(Generic[DownloaderOptionsT, DownloaderEntryT], ABC): +class Downloader(DownloadArchiver, Generic[DownloaderOptionsT, DownloaderEntryT], ABC): """ Class that interacts with ytdl to perform the download of metadata and content, and should translate that to list of Entry objects. @@ -66,7 +68,6 @@ class Downloader(Generic[DownloaderOptionsT, DownloaderEntryT], ABC): cls, working_directory: str, ytdl_options: Optional[Dict], - download_archive_file_name: Optional[str], ) -> Dict: """Configure the ytdl options for the downloader""" if ytdl_options is None: @@ -81,39 +82,29 @@ class Downloader(Generic[DownloaderOptionsT, DownloaderEntryT], ABC): # Overwrite the output location with the specified working directory ytdl_options["outtmpl"] = str(Path(working_directory) / "%(id)s.%(ext)s") - # If a download archive file name is provided, set it to that - if download_archive_file_name: - ytdl_options["download_archive"] = str( - Path(working_directory) / download_archive_file_name - ) - return ytdl_options def __init__( self, - working_directory: str, download_options: DownloaderOptionsT, + enhanced_download_archive: EnhancedDownloadArchive, ytdl_options: Optional[Dict] = None, - download_archive_file_name: Optional[str] = None, ): """ Parameters ---------- - working_directory - Path to the working directory download_options Options validator for this downloader + enhanced_download_archive + Download archive ytdl_options YTDL options validator - download_archive_file_name - Optional. Name of the download archive file that should reside in the working directory """ - self.working_directory = working_directory + DownloadArchiver.__init__(self=self, enhanced_download_archive=enhanced_download_archive) self.download_options = download_options self.ytdl_options = self._configure_ytdl_options( ytdl_options=ytdl_options, working_directory=self.working_directory, - download_archive_file_name=download_archive_file_name, ) @contextmanager @@ -129,6 +120,15 @@ class Downloader(Generic[DownloaderOptionsT, DownloaderEntryT], ABC): with ytdl.YoutubeDL(ytdl_options) as ytdl_downloader: yield ytdl_downloader + @property + def is_dry_run(self) -> bool: + """ + Returns + ------- + True if dry-run is enabled. False otherwise. + """ + return self.ytdl_options.get("skip_download", False) + def extract_info(self, ytdl_options_overrides: Optional[Dict] = None, **kwargs) -> Dict: """ Wrapper around yt_dlp.YoutubeDL.YoutubeDL.extract_info @@ -200,19 +200,14 @@ class Downloader(Generic[DownloaderOptionsT, DownloaderEntryT], ABC): def download(self) -> List[DownloaderEntryT]: """The function to perform the download of all media entries""" - def post_download(self, overrides: Overrides, output_directory: str): + def post_download(self, overrides: Overrides): """ After all media entries have been downloaded, post processed, and moved to the output directory, run this function. This lets the downloader add any extra files directly to the output directory, for things like YT channel image, banner. - This ideally should not perform any extra downloads, but rather, use the content already - downloaded in the working directory and use it in the output directory. - Parameters ---------- overrides: Subscription overrides - output_directory: - Output directory to potentially store extra files downloaded """ diff --git a/src/ytdl_sub/downloaders/youtube/merge_playlist.py b/src/ytdl_sub/downloaders/youtube/merge_playlist.py index a7a7c620..dcbfea89 100644 --- a/src/ytdl_sub/downloaders/youtube/merge_playlist.py +++ b/src/ytdl_sub/downloaders/youtube/merge_playlist.py @@ -100,8 +100,7 @@ class YoutubeMergePlaylistDownloader( }, ) - @classmethod - def _add_chapters(cls, merged_video: YoutubeVideo) -> None: + def _add_chapters(self, merged_video: YoutubeVideo) -> None: titles: List[str] = [] timestamps: List[Timestamp] = [] @@ -112,11 +111,13 @@ class YoutubeMergePlaylistDownloader( current_timestamp_sec += video_entry["duration"] - add_ffmpeg_metadata( - file_path=merged_video.get_download_file_path(), - chapters=Chapters(timestamps=timestamps, titles=titles), - file_duration_sec=merged_video.kwargs("duration"), - ) + # TODO: return chapter metadata here + if not self.is_dry_run: + add_ffmpeg_metadata( + file_path=merged_video.get_download_file_path(), + chapters=Chapters(timestamps=timestamps, titles=titles), + file_duration_sec=merged_video.kwargs("duration"), + ) def _to_merged_video(self, entry_dict: Dict) -> YoutubeVideo: """ @@ -130,7 +131,11 @@ class YoutubeMergePlaylistDownloader( entry_dict["duration"] = sum( playlist_entry["duration"] for playlist_entry in entry_dict["entries"] ) - entry_dict["ext"] = entry_dict["requested_downloads"][0]["ext"] + entry_dict["ext"] = ( + entry_dict["requested_downloads"][0]["ext"] + if "requested_downloads" in entry_dict + else "mkv" + ) return YoutubeVideo(entry_dict=entry_dict, working_directory=self.working_directory) def download(self) -> List[YoutubeVideo]: diff --git a/src/ytdl_sub/downloaders/youtube/split_video.py b/src/ytdl_sub/downloaders/youtube/split_video.py index bd030b88..168e8cf3 100644 --- a/src/ytdl_sub/downloaders/youtube/split_video.py +++ b/src/ytdl_sub/downloaders/youtube/split_video.py @@ -1,7 +1,6 @@ import copy import re from pathlib import Path -from shutil import copyfile from typing import Dict from typing import List @@ -12,8 +11,6 @@ from ytdl_sub.entries.youtube import YoutubeVideo from ytdl_sub.utils.chapters import Chapters from ytdl_sub.utils.chapters import Timestamp from ytdl_sub.utils.ffmpeg import FFMPEG -from ytdl_sub.utils.thumbnail import convert_download_thumbnail -from ytdl_sub.validators.validators import StringValidator # Captures the following formats: # 0:00 title @@ -21,6 +18,10 @@ from ytdl_sub.validators.validators import StringValidator # 1:00:00 title # 01:00:00 title # where capture group 1 and 2 are the timestamp and title, respectively +from ytdl_sub.utils.file_handler import FileHandler +from ytdl_sub.utils.thumbnail import convert_download_thumbnail +from ytdl_sub.validators.validators import StringValidator + _SPLIT_TIMESTAMP_REGEX = re.compile(r"^((?:\d\d:)?(?:\d:)?(?:\d)?\d:\d\d) (.+)$") @@ -154,24 +155,27 @@ class YoutubeSplitVideoDownloader( for idx, title in enumerate(chapters.titles): new_uid = _split_video_uid(source_uid=entry.uid, idx=idx) - # Get the input/output file paths - input_file = entry.get_download_file_path() - output_file = str(Path(self.working_directory) / f"{new_uid}.{entry.ext}") - output_thumbnail_file = str( - Path(self.working_directory) / f"{new_uid}.{entry.thumbnail_ext}" - ) + if not self.is_dry_run: + # Get the input/output file paths + input_file = entry.get_download_file_path() + output_file = str(Path(self.working_directory) / f"{new_uid}.{entry.ext}") - # Run ffmpeg to create the split the video - FFMPEG.run( - _split_video_ffmpeg_cmd( - input_file=input_file, - output_file=output_file, - timestamps=chapters.timestamps, - idx=idx, + # Run ffmpeg to create the split the video + FFMPEG.run( + _split_video_ffmpeg_cmd( + input_file=input_file, + output_file=output_file, + timestamps=chapters.timestamps, + idx=idx, + ) ) + + # Copy the original vid thumbnail to the working directory with the new uid. This so + # downstream logic thinks this split video has its own thumbnail + FileHandler.copy( + src_file_path=entry.get_download_thumbnail_path(), + dst_file_path=Path(self.working_directory) / f"{new_uid}.{entry.thumbnail_ext}", ) - # Copy the thumbnail - copyfile(src=entry.get_download_thumbnail_path(), dst=output_thumbnail_file) # Format the split video as a YoutubePlaylistVideo split_videos.append( diff --git a/src/ytdl_sub/downloaders/youtube_downloader.py b/src/ytdl_sub/downloaders/youtube_downloader.py index 9690c833..2a2c8ffd 100644 --- a/src/ytdl_sub/downloaders/youtube_downloader.py +++ b/src/ytdl_sub/downloaders/youtube_downloader.py @@ -20,6 +20,7 @@ from ytdl_sub.validators.string_formatter_validators import OverridesStringForma from ytdl_sub.validators.url_validator import YoutubeChannelUrlValidator from ytdl_sub.validators.url_validator import YoutubePlaylistUrlValidator from ytdl_sub.validators.url_validator import YoutubeVideoUrlValidator +from ytdl_sub.ytdl_additions.enhanced_download_archive import EnhancedDownloadArchive logger = Logger.get() @@ -290,16 +291,14 @@ class YoutubeChannelDownloader(YoutubeDownloader[YoutubeChannelDownloaderOptions def __init__( self, - working_directory: str, download_options: DownloaderOptionsT, + enhanced_download_archive: EnhancedDownloadArchive, ytdl_options: Optional[Dict] = None, - download_archive_file_name: Optional[str] = None, ): super().__init__( - working_directory=working_directory, download_options=download_options, + enhanced_download_archive=enhanced_download_archive, ytdl_options=ytdl_options, - download_archive_file_name=download_archive_file_name, ) self.channel: Optional[YoutubeChannel] = None @@ -354,7 +353,7 @@ class YoutubeChannelDownloader(YoutubeDownloader[YoutubeChannelDownloaderOptions thumbnail_url=thumbnail_url, output_thumbnail_path=output_thumbnail_path ) - def post_download(self, overrides: Overrides, output_directory: str): + def post_download(self, overrides: Overrides): """ Downloads and moves channel avatar and banner images to the output directory. @@ -362,17 +361,17 @@ class YoutubeChannelDownloader(YoutubeDownloader[YoutubeChannelDownloaderOptions ---------- overrides Overrides that can contain variables in the avatar or banner file path - output_directory - Output directory path """ avatar_thumbnail_name = overrides.apply_formatter(self.download_options.channel_avatar_path) self._download_thumbnail( thumbnail_url=self.channel.avatar_thumbnail_url(), - output_thumbnail_path=str(Path(output_directory) / avatar_thumbnail_name), + output_thumbnail_path=str(Path(self.working_directory) / avatar_thumbnail_name), ) + self.save_file(file_name=avatar_thumbnail_name) banner_thumbnail_name = overrides.apply_formatter(self.download_options.channel_banner_path) self._download_thumbnail( thumbnail_url=self.channel.banner_thumbnail_url(), - output_thumbnail_path=str(Path(output_directory) / banner_thumbnail_name), + output_thumbnail_path=str(Path(self.working_directory) / banner_thumbnail_name), ) + self.save_file(file_name=banner_thumbnail_name) diff --git a/src/ytdl_sub/entries/entry.py b/src/ytdl_sub/entries/entry.py index 7735e0ac..7d47d1bf 100644 --- a/src/ytdl_sub/entries/entry.py +++ b/src/ytdl_sub/entries/entry.py @@ -11,13 +11,29 @@ class Entry(EntryVariables, BaseEntry): Entry object to represent a single media object returned from yt-dlp. """ + def get_download_file_name(self) -> str: + """ + Returns + ------- + The entry's file name + """ + return f"{self.uid}.{self.ext}" + def get_download_file_path(self) -> str: """Returns the entry's file path to where it was downloaded""" - return str(Path(self.working_directory()) / f"{self.uid}.{self.ext}") + return str(Path(self.working_directory()) / self.get_download_file_name()) + + def get_download_thumbnail_name(self) -> str: + """ + Returns + ------- + The download thumbnail's file name + """ + return f"{self.uid}.{self.thumbnail_ext}" def get_download_thumbnail_path(self) -> str: """Returns the entry's thumbnail's file path to where it was downloaded""" - return str(Path(self.working_directory()) / f"{self.uid}.{self.thumbnail_ext}") + return str(Path(self.working_directory()) / self.get_download_thumbnail_name()) @final def to_dict(self) -> Dict[str, str]: diff --git a/src/ytdl_sub/main.py b/src/ytdl_sub/main.py index e1fa0e72..dc656cfa 100644 --- a/src/ytdl_sub/main.py +++ b/src/ytdl_sub/main.py @@ -17,8 +17,12 @@ def _download_subscriptions_from_yaml_files(config: ConfigFile, args: argparse.N """ Downloads all subscriptions from one or many subscription yaml files. - :param config: Configuration file - :param args: Arguments from argparse + Parameters + ---------- + config + Configuration file + args + Arguments from argparse """ preset_paths: List[str] = args.subscription_paths presets: List[Preset] = [] @@ -30,15 +34,23 @@ def _download_subscriptions_from_yaml_files(config: ConfigFile, args: argparse.N subscription = Subscription.from_preset(preset=preset, config=config) logger.info("Beginning subscription download for %s", subscription.name) - subscription.download() + subscription.download(dry_run=args.dry_run) -def _download_subscription_from_cli(config: ConfigFile, extra_args: List[str]) -> None: +def _download_subscription_from_cli( + config: ConfigFile, args: argparse.Namespace, extra_args: List[str] +) -> None: """ Downloads a one-off subscription using the CLI - :param config: Configuration file - :param extra_args: Extra arguments from argparse that contain dynamic subscription options + Parameters + ---------- + config + Configuration file + args + Arguments from argparse + extra_args + Extra arguments from argparse that contain dynamic subscription options """ dl_args_parser = DownloadArgsParser( extra_arguments=extra_args, config_options=config.config_options @@ -57,7 +69,7 @@ def _download_subscription_from_cli(config: ConfigFile, extra_args: List[str]) - config=config, ) - subscription.download() + subscription.download(dry_run=args.dry_run) def _main(): @@ -80,7 +92,7 @@ def _main(): # One-off download if args.subparser == "dl": - _download_subscription_from_cli(config=config, extra_args=extra_args) + _download_subscription_from_cli(config=config, args=args, extra_args=extra_args) logger.info("Download complete!") # Ran successfully, so we can delete the debug file diff --git a/src/ytdl_sub/plugins/music_tags.py b/src/ytdl_sub/plugins/music_tags.py index c2b13b91..14724c37 100644 --- a/src/ytdl_sub/plugins/music_tags.py +++ b/src/ytdl_sub/plugins/music_tags.py @@ -1,8 +1,11 @@ +from typing import Dict + import mediafile from ytdl_sub.entries.entry import Entry from ytdl_sub.plugins.plugin import Plugin from ytdl_sub.plugins.plugin import PluginOptions +from ytdl_sub.utils.file_handler import FileMetadata from ytdl_sub.validators.string_formatter_validators import DictFormatterValidator from ytdl_sub.validators.validators import StringValidator @@ -57,17 +60,28 @@ class MusicTagsPlugin(Plugin[MusicTagsOptions]): """ Tags the entry's audio file using values defined in the metadata options """ - audio_file = mediafile.MediaFile(entry.get_download_file_path()) - for tag, tag_formatter in self.plugin_options.tags.dict.items(): - if tag not in audio_file.fields(): + supported_fields = list(mediafile.MediaFile.sorted_fields()) + tags_to_write: Dict[str, str] = {} + for tag_name, tag_formatter in self.plugin_options.tags.dict.items(): + if tag_name not in supported_fields: + # TODO: Add support for custom fields self._logger.warning( "tag '%s' is not supported for %s files. Supported tags: %s", - tag, + tag_name, entry.ext, - ", ".join(audio_file.sorted_fields()), + ", ".join(sorted(supported_fields)), ) + continue tag_value = self.overrides.apply_formatter(formatter=tag_formatter, entry=entry) - setattr(audio_file, tag, tag_value) + tags_to_write[tag_name] = tag_value - audio_file.save() + # write the actual tags if its not a dry run + if not self.is_dry_run: + audio_file = mediafile.MediaFile(entry.get_download_file_path()) + for tag_name, tag_value in tags_to_write.items(): + setattr(audio_file, tag_name, tag_value) + audio_file.save() + + # report the tags written + return FileMetadata.from_dict(value_dict=tags_to_write, title="Music Tags:") diff --git a/src/ytdl_sub/plugins/nfo_tags.py b/src/ytdl_sub/plugins/nfo_tags.py index df4bfdc0..f1b9705a 100644 --- a/src/ytdl_sub/plugins/nfo_tags.py +++ b/src/ytdl_sub/plugins/nfo_tags.py @@ -6,6 +6,7 @@ import dicttoxml from ytdl_sub.entries.entry import Entry from ytdl_sub.plugins.plugin import Plugin from ytdl_sub.plugins.plugin import PluginOptions +from ytdl_sub.utils.file_handler import FileMetadata from ytdl_sub.validators.string_formatter_validators import DictFormatterValidator from ytdl_sub.validators.string_formatter_validators import StringFormatterValidator @@ -109,10 +110,12 @@ class NfoTagsPlugin(Plugin[NfoTagsOptions]): ) # Save the nfo's XML to file - nfo_file_path = Path(self.output_directory) / nfo_file_name + nfo_file_path = Path(self.working_directory) / nfo_file_name os.makedirs(os.path.dirname(nfo_file_path), exist_ok=True) with open(nfo_file_path, "wb") as nfo_file: nfo_file.write(xml) # Archive the nfo's file name - self.archive_entry_file_name(entry=entry, relative_file_path=nfo_file_name) + self.save_file(file_name=nfo_file_name, entry=entry) + + return FileMetadata.from_dict(value_dict={nfo_root: nfo}, title="NFO tags:") diff --git a/src/ytdl_sub/plugins/output_directory_nfo_tags.py b/src/ytdl_sub/plugins/output_directory_nfo_tags.py index 24123f8a..e0aa7294 100644 --- a/src/ytdl_sub/plugins/output_directory_nfo_tags.py +++ b/src/ytdl_sub/plugins/output_directory_nfo_tags.py @@ -5,6 +5,7 @@ import dicttoxml from ytdl_sub.plugins.plugin import Plugin from ytdl_sub.plugins.plugin import PluginOptions +from ytdl_sub.utils.file_handler import FileMetadata from ytdl_sub.validators.string_formatter_validators import OverridesDictFormatterValidator from ytdl_sub.validators.string_formatter_validators import OverridesStringFormatterValidator @@ -100,7 +101,10 @@ class OutputDirectoryNfoTagsPlugin(Plugin[OutputDirectoryNfoTagsOptions]): nfo_file_name = self.overrides.apply_formatter(formatter=self.plugin_options.nfo_name) # Save the nfo's XML to file - nfo_file_path = Path(self.output_directory) / nfo_file_name + nfo_file_path = Path(self.working_directory) / nfo_file_name os.makedirs(os.path.dirname(nfo_file_path), exist_ok=True) with open(nfo_file_path, "wb") as nfo_file: nfo_file.write(xml) + + self.save_file(file_name=nfo_file_name) + return FileMetadata.from_dict(value_dict={nfo_root: nfo}, title="NFO tags:") diff --git a/src/ytdl_sub/plugins/plugin.py b/src/ytdl_sub/plugins/plugin.py index 74362a9d..fa17191e 100644 --- a/src/ytdl_sub/plugins/plugin.py +++ b/src/ytdl_sub/plugins/plugin.py @@ -7,8 +7,10 @@ from typing import final from ytdl_sub.config.preset_options import Overrides from ytdl_sub.entries.entry import Entry +from ytdl_sub.utils.file_handler import FileMetadata from ytdl_sub.utils.logger import Logger from ytdl_sub.validators.strict_dict_validator import StrictDictValidator +from ytdl_sub.ytdl_additions.enhanced_download_archive import DownloadArchiver from ytdl_sub.ytdl_additions.enhanced_download_archive import EnhancedDownloadArchive @@ -21,7 +23,7 @@ class PluginOptions(StrictDictValidator): PluginOptionsT = TypeVar("PluginOptionsT", bound=PluginOptions) -class Plugin(Generic[PluginOptionsT], ABC): +class Plugin(DownloadArchiver, Generic[PluginOptionsT], ABC): """ Class to define the new plugin functionality """ @@ -32,36 +34,16 @@ class Plugin(Generic[PluginOptionsT], ABC): def __init__( self, plugin_options: PluginOptionsT, - output_directory: str, overrides: Overrides, - enhanced_download_archive: Optional[EnhancedDownloadArchive], + enhanced_download_archive: EnhancedDownloadArchive, ): + DownloadArchiver.__init__(self=self, enhanced_download_archive=enhanced_download_archive) self.plugin_options = plugin_options - self.output_directory = output_directory self.overrides = overrides - self.__enhanced_download_archive = enhanced_download_archive # TODO pass yaml snake case name in the class somewhere, and use it for the logger self._logger = Logger.get(self.__class__.__name__) - @final - def archive_entry_file_name(self, entry: Entry, relative_file_path: str) -> None: - """ - Adds an entry and a file name that belongs to it into the archive mapping. - If maintain_download_archive is False for the subscription, this method will do nothing. - - Parameters - ---------- - entry: - Optional. The entry the file belongs to - relative_file_path: - The name of the file path relative to the output directory - """ - if self.__enhanced_download_archive: - self.__enhanced_download_archive.mapping.add_entry( - entry=entry, entry_file_path=relative_file_path - ) - - def post_process_entry(self, entry: Entry): + def post_process_entry(self, entry: Entry) -> Optional[FileMetadata]: """ For each file downloaded, apply post processing to it. @@ -69,6 +51,10 @@ class Plugin(Generic[PluginOptionsT], ABC): ---------- entry: Entry to post process + + Returns + ------- + Optional file metadata for the entry media file. """ def post_process_subscription(self): diff --git a/src/ytdl_sub/subscriptions/subscription.py b/src/ytdl_sub/subscriptions/subscription.py index 9d07ffa3..5601674c 100644 --- a/src/ytdl_sub/subscriptions/subscription.py +++ b/src/ytdl_sub/subscriptions/subscription.py @@ -1,9 +1,10 @@ import contextlib +import copy import os import shutil from pathlib import Path -from shutil import copyfile from typing import List +from typing import Optional from typing import Tuple from typing import Type @@ -18,6 +19,8 @@ from ytdl_sub.downloaders.downloader import DownloaderValidator from ytdl_sub.entries.entry import Entry from ytdl_sub.plugins.plugin import Plugin from ytdl_sub.plugins.plugin import PluginOptions +from ytdl_sub.utils.file_handler import FileHandlerTransactionLog +from ytdl_sub.utils.file_handler import FileMetadata from ytdl_sub.utils.thumbnail import convert_download_thumbnail from ytdl_sub.ytdl_additions.enhanced_download_archive import EnhancedDownloadArchive @@ -144,30 +147,9 @@ class Subscription: and self.downloader_class.supports_download_archive ) - def _copy_file_to_output_directory( - self, entry: Entry, source_file_path: str, output_file_name: str + def _copy_entry_files_to_output_directory( + self, entry: Entry, entry_metadata: Optional[FileMetadata] = None ): - """ - Helper function to move a file from the working directory to the output directory. - Will add it to the download archive mapping if the archive is being maintained. - - Parameters - ---------- - entry: - Entry that the file belongs to - source_file_path: - Path to the source file - output_file_name: - Desired output file name with the output directory as its relative directory - """ - destination_file_path = Path(self.output_directory) / Path(output_file_name) - os.makedirs(os.path.dirname(destination_file_path), exist_ok=True) - copyfile(source_file_path, destination_file_path) - - if self.maintain_download_archive: - self._enhanced_download_archive.mapping.add_entry(entry, output_file_name) - - def _copy_entry_files_to_output_directory(self, entry: Entry): """ Helper function to move the media file and optionally thumbnail file to the output directory for a single entry. @@ -176,15 +158,18 @@ class Subscription: ---------- entry: The entry with files to move + entry_metadata + Optional. Metadata to record to the transaction log for this entry """ # Move the file after all direct file modifications are complete output_file_name = self.overrides.apply_formatter( formatter=self.output_options.file_name, entry=entry ) - self._copy_file_to_output_directory( - entry=entry, - source_file_path=entry.get_download_file_path(), + self._enhanced_download_archive.save_file_to_output_directory( + file_name=entry.get_download_file_name(), + file_metadata=entry_metadata, output_file_name=output_file_name, + entry=entry, ) if self.output_options.thumbnail_name: @@ -195,10 +180,10 @@ class Subscription: # We always convert entry thumbnails to jpgs, and is performed here convert_download_thumbnail(entry=entry) - self._copy_file_to_output_directory( - entry=entry, - source_file_path=entry.get_download_thumbnail_path(), + self._enhanced_download_archive.save_file_to_output_directory( + file_name=entry.get_download_thumbnail_name(), output_file_name=output_thumbnail_name, + entry=entry, ) @contextlib.contextmanager @@ -243,45 +228,59 @@ class Subscription: for plugin_type, plugin_options in self.plugins: plugin = plugin_type( plugin_options=plugin_options, - output_directory=self.output_directory, overrides=self.overrides, - enhanced_download_archive=self._enhanced_download_archive - if self.maintain_download_archive - else None, + enhanced_download_archive=self._enhanced_download_archive, ) plugins.append(plugin) return plugins - def download(self): + def download(self, dry_run: bool = False) -> FileHandlerTransactionLog: """ - Performs the subscription download. + Performs the subscription download + + Parameters + ---------- + dry_run + If true, do not download any video/audio files or move anything to the output + directory. """ + self._enhanced_download_archive.reinitialize(dry_run=dry_run) + + # TODO: Move this logic to separate function + # TODO: set id here as well + ytdl_options = copy.deepcopy(self.ytdl_options.dict) + if dry_run: + ytdl_options["skip_download"] = True + if self.downloader_class.supports_download_archive and self.maintain_download_archive: + ytdl_options["download_archive"] = str( + Path(self.working_directory) / self._enhanced_download_archive.archive_file_name + ) + plugins = self._initialize_plugins() with self._prepare_working_directory(), self._maintain_archive_file(): downloader = self.downloader_class( - working_directory=self.working_directory, download_options=self.downloader_options, - ytdl_options=self.ytdl_options.dict, - download_archive_file_name=self._enhanced_download_archive.archive_file_name - if self.maintain_download_archive - else None, + enhanced_download_archive=self._enhanced_download_archive, + ytdl_options=ytdl_options, ) - entries = downloader.download() - for plugin in plugins: - for entry in entries: - plugin.post_process_entry(entry) + for entry in downloader.download(): + # TODO: Add entry metadata from the downloader.download function + entry_metadata = FileMetadata() + for plugin in plugins: + entry_metadata.extend(plugin.post_process_entry(entry)) + self._copy_entry_files_to_output_directory( + entry=entry, entry_metadata=entry_metadata + ) + + downloader.post_download(overrides=self.overrides) + for plugin in plugins: plugin.post_process_subscription() - for entry in entries: - self._copy_entry_files_to_output_directory(entry=entry) - - downloader.post_download( - overrides=self.overrides, output_directory=self.output_directory - ) + return self._enhanced_download_archive.get_file_handler_transaction_log() @classmethod def from_preset(cls, preset: Preset, config: ConfigFile) -> "Subscription": diff --git a/src/ytdl_sub/utils/file_handler.py b/src/ytdl_sub/utils/file_handler.py new file mode 100644 index 00000000..2ef61724 --- /dev/null +++ b/src/ytdl_sub/utils/file_handler.py @@ -0,0 +1,195 @@ +import os +from pathlib import Path +from shutil import copyfile +from typing import Any +from typing import Dict +from typing import List +from typing import Optional +from typing import Set +from typing import Union + + +class FileMetadata: + """ + Stores pretty-printed information about a file. Each line in the metadata represents a newline + """ + + def __init__(self, metadata: Optional[List[str]] = None): + self.metadata: List[str] = metadata if metadata else [] + + def append(self, line: str) -> "FileMetadata": + """ + Parameters + ---------- + line + Line of metadata to append + """ + self.metadata.append(line) + return self + + def extend(self, other: Optional["FileMetadata"]) -> "FileMetadata": + """ + Parameters + ---------- + other + Other metadata to extend to this one in its entirety + """ + if other is not None: + self.metadata.extend(other.metadata) + return self + + @classmethod + def from_dict(cls, value_dict: Dict[str, Any], title: Optional[str] = None) -> "FileMetadata": + """ + Parameters + ---------- + value_dict + Dict of things to print indented + title + Optional. Title line to put above the dict + """ + lines: List[str] = [] + if title is not None: + lines.append(title) + + def _recursive_add_dict_lines(rdict: Dict, indent: int): + for key, value in sorted(rdict.items()): + if isinstance(value, Dict): + _recursive_add_dict_lines(rdict=value, indent=indent + 2) + + _indent = " " * indent + lines.append(f"{_indent}{key}: {value}") + + _recursive_add_dict_lines(rdict=value_dict, indent=2) + return cls(metadata=lines) + + +class FileHandlerTransactionLog: + """ + Tracks file 'transactions' performed by a FileHandler + """ + + def __init__(self): + self.files_created: Dict[str, FileMetadata] = {} + self.files_removed: Set[str] = set() + + def log_created_file( + self, file_name: str, file_metadata: Optional[FileMetadata] = None + ) -> "FileHandlerTransactionLog": + """ + Adds a created file to the transaction log + + Parameters + ---------- + file_name + Name of the file in the output directory + file_metadata + Optional. If the file has metadata, add it to the transaction log + """ + if not file_metadata: + file_metadata = FileMetadata() + + self.files_created[file_name] = file_metadata + return self + + def log_removed_file(self, file_name: str) -> "FileHandlerTransactionLog": + """ + Records a file removed from the output directory + Parameters + ---------- + file_name + Name of the file in the output directory getting removed + """ + self.files_removed.add(file_name) + return self + + +class FileHandler: + """ + Performs and tracks all file moving/copying/deleting + """ + + def __init__(self, working_directory: str, output_directory: str, dry_run: bool): + self.dry_run = dry_run + self.working_directory = working_directory + self.output_directory = output_directory + self._file_handler_transaction_log = FileHandlerTransactionLog() + + @property + def file_handler_transaction_log(self) -> FileHandlerTransactionLog: + """ + Returns + ------- + Transaction logs of this file handler + """ + return self._file_handler_transaction_log + + @classmethod + def copy(cls, src_file_path: Union[str, Path], dst_file_path: Union[str, Path]): + """ + Parameters + ---------- + src_file_path + Source file + dst_file_path + Destination file + """ + copyfile(src=src_file_path, dst=dst_file_path) + + @classmethod + def delete(cls, file_path: Union[str, Path]): + """ + Parameters + ---------- + file_path + File to delete + """ + if os.path.isfile(file_path): + os.remove(file_path) + + def copy_file_to_output_directory( + self, file_name: str, output_file_name: str, file_metadata: Optional[FileMetadata] = None + ): + """ + Copies a file from the working directory to the output directory. + All file copies from working to output directory should use this function for tracking and + handling dry-run logic. + + Parameters + ---------- + file_name + File in the working directory + output_file_name + Desired output file name in the output_directory + file_metadata + Optional. Metadata to record to the transaction log for this file + """ + self._file_handler_transaction_log.log_created_file( + file_name=output_file_name, file_metadata=file_metadata + ) + + if not self.dry_run: + output_file_path = Path(self.output_directory) / output_file_name + os.makedirs(os.path.dirname(output_file_path), exist_ok=True) + self.copy( + src_file_path=Path(self.working_directory) / file_name, + dst_file_path=output_file_path, + ) + + def delete_file_from_output_directory(self, file_name: str): + """ + Deletes a file from the output directory. All file deletions should use this function + for tracking and handling dry-run logic. + + Parameters + ---------- + file_name + File in the output directory to delete + """ + file_path = Path(self.output_directory) / file_name + exists = os.path.isfile(file_path) + + if exists: + self._file_handler_transaction_log.log_removed_file(file_name) + if not self.dry_run: + self.delete(file_path=file_path) diff --git a/src/ytdl_sub/utils/logger.py b/src/ytdl_sub/utils/logger.py index 90dfc49b..2d811de3 100644 --- a/src/ytdl_sub/utils/logger.py +++ b/src/ytdl_sub/utils/logger.py @@ -1,13 +1,14 @@ import contextlib import io import logging -import os import sys import tempfile from dataclasses import dataclass from typing import List from typing import Optional +from ytdl_sub.utils.file_handler import FileHandler + @dataclass class LoggerLevel: @@ -201,5 +202,5 @@ class Logger: """ cls._DEBUG_LOGGER_FILE.close() - if delete_debug_file and os.path.isfile(cls.debug_log_filename()): - os.remove(cls.debug_log_filename()) + if delete_debug_file: + FileHandler.delete(cls.debug_log_filename()) diff --git a/src/ytdl_sub/ytdl_additions/enhanced_download_archive.py b/src/ytdl_sub/ytdl_additions/enhanced_download_archive.py index 56e71d10..cc2ca39d 100644 --- a/src/ytdl_sub/ytdl_additions/enhanced_download_archive.py +++ b/src/ytdl_sub/ytdl_additions/enhanced_download_archive.py @@ -13,6 +13,9 @@ from typing import Set from yt_dlp import DateRange from ytdl_sub.entries.entry import Entry +from ytdl_sub.utils.file_handler import FileHandler +from ytdl_sub.utils.file_handler import FileHandlerTransactionLog +from ytdl_sub.utils.file_handler import FileMetadata from ytdl_sub.utils.logger import Logger @@ -210,7 +213,7 @@ class DownloadMappings: entry Entry that this file belongs to entry_file_path - Relative path to the file that belongs to the entry + Relative path to the file that lives in the output directory Returns ------- @@ -334,16 +337,52 @@ class EnhancedDownloadArchive: 6. ( Delete the working directory ) """ - def __init__(self, subscription_name: str, working_directory: str, output_directory: str): + def __init__( + self, + subscription_name: str, + working_directory: str, + output_directory: str, + dry_run: bool = False, + ): self.subscription_name = subscription_name - self.working_directory = working_directory - self.output_directory = output_directory - - self._download_archive: Optional[DownloadArchive] = None - self._download_mapping: Optional[DownloadMappings] = None + self._file_handler = FileHandler( + working_directory=working_directory, output_directory=output_directory, dry_run=dry_run + ) + self._download_mapping = DownloadMappings() self._logger = Logger.get(name=subscription_name) + def reinitialize(self, dry_run: bool) -> "EnhancedDownloadArchive": + """ + Re-initialize the enhanced download archive for successive downloads w/the same + subscription. + + Parameters + ---------- + dry_run + Whether to actually move files to the output directory + + Returns + ------- + self + """ + self._file_handler = FileHandler( + working_directory=self.working_directory, + output_directory=self.output_directory, + dry_run=dry_run, + ) + self._download_mapping = DownloadMappings() + return self + + @property + def is_dry_run(self) -> bool: + """ + Returns + ------- + True if this session is a dry-run. False otherwise. + """ + return self._file_handler.dry_run + @property def archive_file_name(self) -> str: """ @@ -353,6 +392,24 @@ class EnhancedDownloadArchive: """ return f".ytdl-subscribe-{self.subscription_name}-download-archive.txt" + @property + def working_directory(self) -> str: + """ + Returns + ------- + Path to the working directory + """ + return self._file_handler.working_directory + + @property + def output_directory(self) -> str: + """ + Returns + ------- + Path to the output directory + """ + return self._file_handler.output_directory + @property def _mapping_file_name(self) -> str: """ @@ -363,7 +420,7 @@ class EnhancedDownloadArchive: return f".ytdl-sub-{self.subscription_name}-download-archive.json" @property - def _mapping_output_file_path(self): + def _mapping_output_file_path(self) -> str: """ Returns ------- @@ -371,6 +428,15 @@ class EnhancedDownloadArchive: """ return str(Path(self.output_directory) / self._mapping_file_name) + @property + def _mapping_working_file_path(self) -> str: + """ + Returns + ------- + The download mapping's file path in the working directory. + """ + return str(Path(self.working_directory) / self._mapping_file_name) + @property def _archive_working_file_path(self) -> str: """ @@ -399,7 +465,6 @@ class EnhancedDownloadArchive: def _load(self) -> "EnhancedDownloadArchive": """ Tries to load download mappings if they are present in the output directory. - If they are not, initialize an empty mapping. Returns ------- @@ -410,14 +475,9 @@ class EnhancedDownloadArchive: self._download_mapping = DownloadMappings.from_file( json_file_path=self._mapping_output_file_path ) - # Otherwise, init an empty download mappings object. Keep _download_archive as None to - # indicate it was not loaded - else: - self._download_mapping = DownloadMappings() - return self - def _copy_to_working_directory(self) -> "EnhancedDownloadArchive": + def _copy_mapping_to_working_directory(self) -> "EnhancedDownloadArchive": """ If the mapping is not empty, create a download archive from it and save it into the working directory. This will tell YTDL to not redownload already downloaded entries. @@ -446,7 +506,7 @@ class EnhancedDownloadArchive: self """ self._load() - self._copy_to_working_directory() + self._copy_mapping_to_working_directory() return self def remove_stale_files(self, date_range: DateRange) -> "EnhancedDownloadArchive": @@ -468,12 +528,8 @@ class EnhancedDownloadArchive: ) for uid, mapping in stale_mappings.items(): - self._logger.info("[%s] Removing the following stale file(s):", uid) for file_name in mapping.file_names: - file_path = Path(self.output_directory) / Path(file_name) - self._logger.info(" - %s", file_path) - if os.path.exists(file_path): - os.remove(file_path) + self._file_handler.delete_file_from_output_directory(file_name=file_name) self.mapping.remove_entry(entry_id=uid) @@ -487,17 +543,96 @@ class EnhancedDownloadArchive: ------- self """ - # TODO: Make sure this logic is actually right... - # Load the download archive from the working directory, which should contain any past - # and new entries downloaded in this session - self._download_archive = DownloadArchive.from_file(self._archive_working_file_path) - - # Keep the download archive in sync with the mapping - for entry_id in self.mapping.entry_ids: - if not self._download_archive.contains(entry_id): - self._download_archive.remove_entry(entry_id) - - # Save the updated mapping file to the output directory - self._download_mapping.to_file(output_json_file=self._mapping_output_file_path) - + self._download_mapping.to_file(output_json_file=self._mapping_working_file_path) + self.save_file_to_output_directory(file_name=self._mapping_file_name) return self + + def save_file_to_output_directory( + self, + file_name: str, + file_metadata: Optional[FileMetadata] = None, + output_file_name: Optional[str] = None, + entry: Optional[Entry] = None, + ): + """ + Saves a file from the working directory to the output directory and record it in the + transaction log. + + Parameters + ---------- + file_name + Name of the file to move (does not include working directory path) + file_metadata + Optional. Metadata to record to the transaction log for this file + output_file_name + Optional. Final name of the file in the output directory (does not include output + directory path). If None, use the same working_directory file_name + entry + Optional. Entry that this file belongs to + """ + if output_file_name is None: + output_file_name = file_name + + if entry: + self.mapping.add_entry(entry=entry, entry_file_path=output_file_name) + + self._file_handler.copy_file_to_output_directory( + file_name=file_name, file_metadata=file_metadata, output_file_name=output_file_name + ) + + def get_file_handler_transaction_log(self) -> FileHandlerTransactionLog: + """ + Returns + ------- + File handler transaction log for this session + """ + return self._file_handler.file_handler_transaction_log + + +class DownloadArchiver: + """ + Used for any class that saves files. Does not allow direct access to output_directory, + forcing the user of the class to use ``save_file`` so it gets archived and avoids any writes + during dry-run. + """ + + def __init__(self, enhanced_download_archive: EnhancedDownloadArchive): + self.__enhanced_download_archive = enhanced_download_archive + + @property + def working_directory(self) -> str: + """ + Returns + ------- + Path to the working directory + """ + return self.__enhanced_download_archive.working_directory + + @property + def is_dry_run(self) -> bool: + """ + Returns + ------- + True if this session is a dry-run. False otherwise. + """ + return self.__enhanced_download_archive.is_dry_run + + def save_file( + self, file_name: str, output_file_name: Optional[str] = None, entry: Optional[Entry] = None + ) -> None: + """ + Saves a file in the working directory to the output directory. + + Parameters + ---------- + file_name + Name of the file relative to the working directory + output_file_name + Optional. Final name of the file in the output directory (does not include output + directory path). If None, use the same working_directory file_name + entry + Optional. Entry that the file belongs to + """ + self.__enhanced_download_archive.save_file_to_output_directory( + file_name=file_name, output_file_name=output_file_name, entry=entry + ) diff --git a/tests/e2e/expected_download.py b/tests/e2e/expected_download.py index 65e49f3b..ffc5dde3 100644 --- a/tests/e2e/expected_download.py +++ b/tests/e2e/expected_download.py @@ -6,6 +6,8 @@ from typing import List from typing import Optional from typing import Union +from ytdl_sub.utils.file_handler import FileHandlerTransactionLog + class ExpectedDownload: """ @@ -58,3 +60,13 @@ class ExpectedDownload: f"MD5 hash for {str(relative_path)} does not match: " f"{md5_hash} != {expected_md5_hash}" ) + + def assert_dry_run_files_logged(self, transaction_log: FileHandlerTransactionLog): + assert ( + len(transaction_log.files_created) == self.file_count + ), "Mismatch in number of created files" + + for relative_path in self.expected_md5_file_hashes.keys(): + assert ( + str(relative_path) in transaction_log.files_created + ), f"Expected {str(relative_path)} to be a file but it is not" diff --git a/tests/e2e/soundcloud/test_soundcloud_discography.py b/tests/e2e/soundcloud/test_soundcloud_discography.py index 2679f772..488c5505 100644 --- a/tests/e2e/soundcloud/test_soundcloud_discography.py +++ b/tests/e2e/soundcloud/test_soundcloud_discography.py @@ -29,7 +29,7 @@ def subscription_dict(output_directory, subscription_name): "preset": "sc_discography", "soundcloud": {"url": "https://soundcloud.com/jessebannon"}, # override the output directory with our fixture-generated dir - "output_options": {"output_directory": output_directory + "/{artist_sanitized}"}, + "output_options": {"output_directory": output_directory}, # download the worst format so it is fast "ytdl_options": { "format": "worst[ext=mp3]", @@ -59,7 +59,7 @@ def expected_discography_download(): return ExpectedDownload( expected_md5_file_hashes={ # Download mapping - Path("j_b/.ytdl-sub-jb-download-archive.json"): "ae55de93b71267b5712c9a3d06c07c26", + Path(".ytdl-sub-jb-download-archive.json"): "1a99156e9ece62539fb2608416a07200", # Entry files (singles) Path("j_b/[2021] Baby Santana's Dorian Groove/01 - Baby Santana's Dorian Groove.mp3"): "bffbd558e12c6a9e029dc136a88342c4", @@ -99,3 +99,9 @@ class TestSoundcloudDiscography: ): discography_subscription.download() expected_discography_download.assert_files_exist(relative_directory=output_directory) + + def test_discography_dry_run( + self, discography_subscription, expected_discography_download, output_directory + ): + transaction_log = discography_subscription.download(dry_run=True) + expected_discography_download.assert_dry_run_files_logged(transaction_log=transaction_log) diff --git a/tests/e2e/youtube/test_channel_as_kodi_tv_show.py b/tests/e2e/youtube/test_channel_as_kodi_tv_show.py index f0afb3f6..fcaeb4eb 100644 --- a/tests/e2e/youtube/test_channel_as_kodi_tv_show.py +++ b/tests/e2e/youtube/test_channel_as_kodi_tv_show.py @@ -27,12 +27,12 @@ def config(config_path): @pytest.fixture -def subscription_dict(output_directory, subscription_name): +def subscription_dict(output_directory): return { "preset": "yt_channel_as_tv", "youtube": {"channel_url": "https://youtube.com/channel/UCcRSMoQqXc_JrBZRHDFGbqA"}, # override the output directory with our fixture-generated dir - "output_options": {"output_directory": str(Path(output_directory) / subscription_name)}, + "output_options": {"output_directory": output_directory}, # download the worst format so it is fast "ytdl_options": { "format": "worst[ext=mp4]", @@ -68,61 +68,61 @@ def expected_full_channel_download(): return ExpectedDownload( expected_md5_file_hashes={ # Download mapping - Path("pz/.ytdl-sub-pz-download-archive.json"): "b7e7c19d2cf0277e4e42453a64fbaa90", + Path(".ytdl-sub-pz-download-archive.json"): "b7e7c19d2cf0277e4e42453a64fbaa90", # Output directory files - Path("pz/fanart.jpg"): "e6e323373c8902568e96e374817179cf", - Path("pz/poster.jpg"): "a14c593bcc75bb8d2c7145de4767ad01", - Path("pz/tvshow.nfo"): "83c7db96081ac5bdf289fcf396bec157", + Path("fanart.jpg"): "e6e323373c8902568e96e374817179cf", + Path("poster.jpg"): "a14c593bcc75bb8d2c7145de4767ad01", + Path("tvshow.nfo"): "83c7db96081ac5bdf289fcf396bec157", # Entry files - Path("pz/Season 2010/s2010.e0813 - Oblivion Mod 'Falcor' p.1.jpg"): "b58377dfe7c39527e1990a24b36bbd77", - Path("pz/Season 2010/s2010.e0813 - Oblivion Mod 'Falcor' p.1.mp4"): "931a705864c57d21d6fedebed4af6bbc", - Path("pz/Season 2010/s2010.e0813 - Oblivion Mod 'Falcor' p.1.nfo"): "67d8d71d048039080acbba3bce4febaa", + Path("Season 2010/s2010.e0813 - Oblivion Mod 'Falcor' p.1.jpg"): "b58377dfe7c39527e1990a24b36bbd77", + Path("Season 2010/s2010.e0813 - Oblivion Mod 'Falcor' p.1.mp4"): "931a705864c57d21d6fedebed4af6bbc", + Path("Season 2010/s2010.e0813 - Oblivion Mod 'Falcor' p.1.nfo"): "67d8d71d048039080acbba3bce4febaa", - Path("pz/Season 2010/s2010.e1202 - Oblivion Mod 'Falcor' p.2.jpg"): "a5ee6247c8dce255aec79c9a51d49da4", - Path("pz/Season 2010/s2010.e1202 - Oblivion Mod 'Falcor' p.2.mp4"): "d3469b4dca7139cb3dbc38712b6796bf", - Path("pz/Season 2010/s2010.e1202 - Oblivion Mod 'Falcor' p.2.nfo"): "d81f49cedbd7edaee987521e89b37904", + Path("Season 2010/s2010.e1202 - Oblivion Mod 'Falcor' p.2.jpg"): "a5ee6247c8dce255aec79c9a51d49da4", + Path("Season 2010/s2010.e1202 - Oblivion Mod 'Falcor' p.2.mp4"): "d3469b4dca7139cb3dbc38712b6796bf", + Path("Season 2010/s2010.e1202 - Oblivion Mod 'Falcor' p.2.nfo"): "d81f49cedbd7edaee987521e89b37904", - Path("pz/Season 2011/s2011.e0201 - Jesse's Minecraft Server [Trailer - Feb.1].jpg"): "048a19cf0f674437351872c3f312ebf1", - Path("pz/Season 2011/s2011.e0201 - Jesse's Minecraft Server [Trailer - Feb.1].mp4"): "e66287b9832277b6a4d1554e29d9fdcc", - Path("pz/Season 2011/s2011.e0201 - Jesse's Minecraft Server [Trailer - Feb.1].nfo"): "f7c0de89038f8c491bded8a3968720a2", + Path("Season 2011/s2011.e0201 - Jesse's Minecraft Server [Trailer - Feb.1].jpg"): "048a19cf0f674437351872c3f312ebf1", + Path("Season 2011/s2011.e0201 - Jesse's Minecraft Server [Trailer - Feb.1].mp4"): "e66287b9832277b6a4d1554e29d9fdcc", + Path("Season 2011/s2011.e0201 - Jesse's Minecraft Server [Trailer - Feb.1].nfo"): "f7c0de89038f8c491bded8a3968720a2", - Path("pz/Season 2011/s2011.e0227 - Jesse's Minecraft Server [Trailer - Feb.27].jpg"): None, - Path("pz/Season 2011/s2011.e0227 - Jesse's Minecraft Server [Trailer - Feb.27].mp4"): "04ab5cb3cc12325d0c96a7cd04a8b91d", - Path("pz/Season 2011/s2011.e0227 - Jesse's Minecraft Server [Trailer - Feb.27].nfo"): "ee1eda78fa0980bc703e602b5012dd1f", + Path("Season 2011/s2011.e0227 - Jesse's Minecraft Server [Trailer - Feb.27].jpg"): None, + Path("Season 2011/s2011.e0227 - Jesse's Minecraft Server [Trailer - Feb.27].mp4"): "04ab5cb3cc12325d0c96a7cd04a8b91d", + Path("Season 2011/s2011.e0227 - Jesse's Minecraft Server [Trailer - Feb.27].nfo"): "ee1eda78fa0980bc703e602b5012dd1f", - Path("pz/Season 2011/s2011.e0321 - Jesse's Minecraft Server [Trailer - Mar.21].jpg"): "9baaddc6b62f5b9ae3781eb4eef0e3b3", - Path("pz/Season 2011/s2011.e0321 - Jesse's Minecraft Server [Trailer - Mar.21].mp4"): "025de6099a5c98e6397153c7a62d517d", - Path("pz/Season 2011/s2011.e0321 - Jesse's Minecraft Server [Trailer - Mar.21].nfo"): "61eb6369430da0ab6134d78829a7621b", + Path("Season 2011/s2011.e0321 - Jesse's Minecraft Server [Trailer - Mar.21].jpg"): "9baaddc6b62f5b9ae3781eb4eef0e3b3", + Path("Season 2011/s2011.e0321 - Jesse's Minecraft Server [Trailer - Mar.21].mp4"): "025de6099a5c98e6397153c7a62d517d", + Path("Season 2011/s2011.e0321 - Jesse's Minecraft Server [Trailer - Mar.21].nfo"): "61eb6369430da0ab6134d78829a7621b", - Path("pz/Season 2011/s2011.e0529 - Project Zombie _Official Trailer_ (IP - mc.projectzombie.beastnode.net).jpg"): "ce1df7f623fffaefe04606ecbafcfec6", - Path("pz/Season 2011/s2011.e0529 - Project Zombie _Official Trailer_ (IP - mc.projectzombie.beastnode.net).mp4"): "3d9c19835b03355d6fd5d00cd59dbe5b", - Path("pz/Season 2011/s2011.e0529 - Project Zombie _Official Trailer_ (IP - mc.projectzombie.beastnode.net).nfo"): "60f72b99f5c69f9e03a071a12160928f", + Path("Season 2011/s2011.e0529 - Project Zombie _Official Trailer_ (IP - mc.projectzombie.beastnode.net).jpg"): "ce1df7f623fffaefe04606ecbafcfec6", + Path("Season 2011/s2011.e0529 - Project Zombie _Official Trailer_ (IP - mc.projectzombie.beastnode.net).mp4"): "3d9c19835b03355d6fd5d00cd59dbe5b", + Path("Season 2011/s2011.e0529 - Project Zombie _Official Trailer_ (IP - mc.projectzombie.beastnode.net).nfo"): "60f72b99f5c69f9e03a071a12160928f", - Path("pz/Season 2011/s2011.e0630 - Project Zombie _Fin.jpg"): "bc3f511915869720c37617a7de706b2b", - Path("pz/Season 2011/s2011.e0630 - Project Zombie _Fin.mp4"): "4971cb2d4fa29460361031f3fa8e1ea9", - Path("pz/Season 2011/s2011.e0630 - Project Zombie _Fin.nfo"): "a7b5d9e57d20852f5daf360a1373bb7a", + Path("Season 2011/s2011.e0630 - Project Zombie _Fin.jpg"): "bc3f511915869720c37617a7de706b2b", + Path("Season 2011/s2011.e0630 - Project Zombie _Fin.mp4"): "4971cb2d4fa29460361031f3fa8e1ea9", + Path("Season 2011/s2011.e0630 - Project Zombie _Fin.nfo"): "a7b5d9e57d20852f5daf360a1373bb7a", - Path("pz/Season 2011/s2011.e1121 - Skyrim 'Ultra HD w_Mods' [PC].jpg"): "12babdb3b86cd868b90b60d013295f66", - Path("pz/Season 2011/s2011.e1121 - Skyrim 'Ultra HD w_Mods' [PC].mp4"): "55e9b0add08c48c9c66105da0def2426", - Path("pz/Season 2011/s2011.e1121 - Skyrim 'Ultra HD w_Mods' [PC].nfo"): "fe60e2b6b564f9316b6c7c183e1cf300", + Path("Season 2011/s2011.e1121 - Skyrim 'Ultra HD w_Mods' [PC].jpg"): "12babdb3b86cd868b90b60d013295f66", + Path("Season 2011/s2011.e1121 - Skyrim 'Ultra HD w_Mods' [PC].mp4"): "55e9b0add08c48c9c66105da0def2426", + Path("Season 2011/s2011.e1121 - Skyrim 'Ultra HD w_Mods' [PC].nfo"): "fe60e2b6b564f9316b6c7c183e1cf300", - Path("pz/Season 2012/s2012.e0123 - Project Zombie _Map Trailer.jpg"): "82d303e16aba75acdde30b15c4154231", - Path("pz/Season 2012/s2012.e0123 - Project Zombie _Map Trailer.mp4"): "65e4ce53ed5ec4139995469f99477a50", - Path("pz/Season 2012/s2012.e0123 - Project Zombie _Map Trailer.nfo"): "c8900adcca83c473c79a4afbc7ad2de1", + Path("Season 2012/s2012.e0123 - Project Zombie _Map Trailer.jpg"): "82d303e16aba75acdde30b15c4154231", + Path("Season 2012/s2012.e0123 - Project Zombie _Map Trailer.mp4"): "65e4ce53ed5ec4139995469f99477a50", + Path("Season 2012/s2012.e0123 - Project Zombie _Map Trailer.nfo"): "c8900adcca83c473c79a4afbc7ad2de1", - Path("pz/Season 2013/s2013.e0719 - Project Zombie Rewind _Trailer.jpg"): "83b1af4c3614d262b2ad419586fff730", - Path("pz/Season 2013/s2013.e0719 - Project Zombie Rewind _Trailer.mp4"): "18620a8257a686beda65e54add4d4cd1", - Path("pz/Season 2013/s2013.e0719 - Project Zombie Rewind _Trailer.nfo"): "1c993c41d4308a6049333154d0adee16", + Path("Season 2013/s2013.e0719 - Project Zombie Rewind _Trailer.jpg"): "83b1af4c3614d262b2ad419586fff730", + Path("Season 2013/s2013.e0719 - Project Zombie Rewind _Trailer.mp4"): "18620a8257a686beda65e54add4d4cd1", + Path("Season 2013/s2013.e0719 - Project Zombie Rewind _Trailer.nfo"): "1c993c41d4308a6049333154d0adee16", - Path("pz/Season 2018/s2018.e1029 - Jesse's Minecraft Server _ Teaser Trailer.jpg"): "2a24de903059f48c7d0df0476046c975", - Path("pz/Season 2018/s2018.e1029 - Jesse's Minecraft Server _ Teaser Trailer.mp4"): "82f6ee7253e1dbb83ae7215af08ffacc", - Path("pz/Season 2018/s2018.e1029 - Jesse's Minecraft Server _ Teaser Trailer.nfo"): "cc7886aae3af6b7b0facd82f95390242", + Path("Season 2018/s2018.e1029 - Jesse's Minecraft Server _ Teaser Trailer.jpg"): "2a24de903059f48c7d0df0476046c975", + Path("Season 2018/s2018.e1029 - Jesse's Minecraft Server _ Teaser Trailer.mp4"): "82f6ee7253e1dbb83ae7215af08ffacc", + Path("Season 2018/s2018.e1029 - Jesse's Minecraft Server _ Teaser Trailer.nfo"): "cc7886aae3af6b7b0facd82f95390242", - Path("pz/Season 2018/s2018.e1102 - Jesse's Minecraft Server _ IP mc.jesse.id.jpg"): "c8baea83b9edeb081657f1130a1031f7", - Path("pz/Season 2018/s2018.e1102 - Jesse's Minecraft Server _ IP mc.jesse.id.mp4"): "e733b4cc385b953b08c8eb0f47e03c1e", - Path("pz/Season 2018/s2018.e1102 - Jesse's Minecraft Server _ IP mc.jesse.id.nfo"): "2b3ccb3f1ef81ee49fe1afb88f275a09", + Path("Season 2018/s2018.e1102 - Jesse's Minecraft Server _ IP mc.jesse.id.jpg"): "c8baea83b9edeb081657f1130a1031f7", + Path("Season 2018/s2018.e1102 - Jesse's Minecraft Server _ IP mc.jesse.id.mp4"): "e733b4cc385b953b08c8eb0f47e03c1e", + Path("Season 2018/s2018.e1102 - Jesse's Minecraft Server _ IP mc.jesse.id.nfo"): "2b3ccb3f1ef81ee49fe1afb88f275a09", } ) # fmt: on @@ -164,21 +164,21 @@ def expected_recent_channel_download(): return ExpectedDownload( expected_md5_file_hashes={ # Download mapping - Path("pz/.ytdl-sub-pz-download-archive.json"): "b1675ca4d9f0d4b9c2102b6749e4cdfd", + Path(".ytdl-sub-pz-download-archive.json"): "b1675ca4d9f0d4b9c2102b6749e4cdfd", # Output directory files - Path("pz/fanart.jpg"): "e6e323373c8902568e96e374817179cf", - Path("pz/poster.jpg"): "a14c593bcc75bb8d2c7145de4767ad01", - Path("pz/tvshow.nfo"): "83c7db96081ac5bdf289fcf396bec157", + Path("fanart.jpg"): "e6e323373c8902568e96e374817179cf", + Path("poster.jpg"): "a14c593bcc75bb8d2c7145de4767ad01", + Path("tvshow.nfo"): "83c7db96081ac5bdf289fcf396bec157", # Recent Entry files - Path("pz/Season 2018/s2018.e1029 - Jesse's Minecraft Server _ Teaser Trailer.jpg"): "2a24de903059f48c7d0df0476046c975", - Path("pz/Season 2018/s2018.e1029 - Jesse's Minecraft Server _ Teaser Trailer.mp4"): "82f6ee7253e1dbb83ae7215af08ffacc", - Path("pz/Season 2018/s2018.e1029 - Jesse's Minecraft Server _ Teaser Trailer.nfo"): "cc7886aae3af6b7b0facd82f95390242", + Path("Season 2018/s2018.e1029 - Jesse's Minecraft Server _ Teaser Trailer.jpg"): "2a24de903059f48c7d0df0476046c975", + Path("Season 2018/s2018.e1029 - Jesse's Minecraft Server _ Teaser Trailer.mp4"): "82f6ee7253e1dbb83ae7215af08ffacc", + Path("Season 2018/s2018.e1029 - Jesse's Minecraft Server _ Teaser Trailer.nfo"): "cc7886aae3af6b7b0facd82f95390242", - Path("pz/Season 2018/s2018.e1102 - Jesse's Minecraft Server _ IP mc.jesse.id.jpg"): "c8baea83b9edeb081657f1130a1031f7", - Path("pz/Season 2018/s2018.e1102 - Jesse's Minecraft Server _ IP mc.jesse.id.mp4"): "e733b4cc385b953b08c8eb0f47e03c1e", - Path("pz/Season 2018/s2018.e1102 - Jesse's Minecraft Server _ IP mc.jesse.id.nfo"): "2b3ccb3f1ef81ee49fe1afb88f275a09", + Path("Season 2018/s2018.e1102 - Jesse's Minecraft Server _ IP mc.jesse.id.jpg"): "c8baea83b9edeb081657f1130a1031f7", + Path("Season 2018/s2018.e1102 - Jesse's Minecraft Server _ IP mc.jesse.id.mp4"): "e733b4cc385b953b08c8eb0f47e03c1e", + Path("Season 2018/s2018.e1102 - Jesse's Minecraft Server _ IP mc.jesse.id.nfo"): "2b3ccb3f1ef81ee49fe1afb88f275a09", } ) # fmt: on @@ -222,12 +222,12 @@ def expected_recent_channel_no_vids_in_range_download(): return ExpectedDownload( expected_md5_file_hashes={ # Download mapping - Path("pz/.ytdl-sub-pz-download-archive.json"): "99914b932bd37a50b983c5e7c90ae93b", + Path(".ytdl-sub-pz-download-archive.json"): "99914b932bd37a50b983c5e7c90ae93b", # Output directory files - Path("pz/fanart.jpg"): "e6e323373c8902568e96e374817179cf", - Path("pz/poster.jpg"): "a14c593bcc75bb8d2c7145de4767ad01", - Path("pz/tvshow.nfo"): "83c7db96081ac5bdf289fcf396bec157", + Path("fanart.jpg"): "e6e323373c8902568e96e374817179cf", + Path("poster.jpg"): "a14c593bcc75bb8d2c7145de4767ad01", + Path("tvshow.nfo"): "83c7db96081ac5bdf289fcf396bec157", } ) # fmt: on @@ -269,17 +269,17 @@ def expected_rolling_recent_channel_download(): return ExpectedDownload( expected_md5_file_hashes={ # Download mapping - Path("pz/.ytdl-sub-pz-download-archive.json"): "9ae3463bd2dc39830003aba68a276df4", + Path(".ytdl-sub-pz-download-archive.json"): "9ae3463bd2dc39830003aba68a276df4", # Output directory files - Path("pz/fanart.jpg"): "e6e323373c8902568e96e374817179cf", - Path("pz/poster.jpg"): "a14c593bcc75bb8d2c7145de4767ad01", - Path("pz/tvshow.nfo"): "83c7db96081ac5bdf289fcf396bec157", + Path("fanart.jpg"): "e6e323373c8902568e96e374817179cf", + Path("poster.jpg"): "a14c593bcc75bb8d2c7145de4767ad01", + Path("tvshow.nfo"): "83c7db96081ac5bdf289fcf396bec157", # Rolling Recent Entry files - Path("pz/Season 2018/s2018.e1102 - Jesse's Minecraft Server _ IP mc.jesse.id.jpg"): "c8baea83b9edeb081657f1130a1031f7", - Path("pz/Season 2018/s2018.e1102 - Jesse's Minecraft Server _ IP mc.jesse.id.mp4"): "e733b4cc385b953b08c8eb0f47e03c1e", - Path("pz/Season 2018/s2018.e1102 - Jesse's Minecraft Server _ IP mc.jesse.id.nfo"): "2b3ccb3f1ef81ee49fe1afb88f275a09", + Path("Season 2018/s2018.e1102 - Jesse's Minecraft Server _ IP mc.jesse.id.jpg"): "c8baea83b9edeb081657f1130a1031f7", + Path("Season 2018/s2018.e1102 - Jesse's Minecraft Server _ IP mc.jesse.id.mp4"): "e733b4cc385b953b08c8eb0f47e03c1e", + Path("Season 2018/s2018.e1102 - Jesse's Minecraft Server _ IP mc.jesse.id.nfo"): "2b3ccb3f1ef81ee49fe1afb88f275a09", } ) # fmt: on @@ -297,6 +297,12 @@ class TestChannelAsKodiTvShow: full_channel_subscription.download() expected_full_channel_download.assert_files_exist(relative_directory=output_directory) + def test_full_channel_dry_run( + self, full_channel_subscription, expected_full_channel_download, output_directory + ): + transaction_log = full_channel_subscription.download(dry_run=True) + expected_full_channel_download.assert_dry_run_files_logged(transaction_log=transaction_log) + def test_recent_channel_download( self, recent_channel_subscription, expected_recent_channel_download, output_directory ): @@ -311,6 +317,14 @@ class TestChannelAsKodiTvShow: recent_channel_subscription.download() expected_recent_channel_download.assert_files_exist(relative_directory=output_directory) + def test_recent_channel_dry_run( + self, recent_channel_subscription, expected_recent_channel_download, output_directory + ): + transaction_log = recent_channel_subscription.download(dry_run=True) + expected_recent_channel_download.assert_dry_run_files_logged( + transaction_log=transaction_log + ) + def test_recent_channel_download__no_vids_in_range( self, recent_channel_no_vids_in_range_subscription, @@ -328,6 +342,17 @@ class TestChannelAsKodiTvShow: relative_directory=output_directory ) + def test_recent_channel_dry_run__no_vids_in_range( + self, + recent_channel_no_vids_in_range_subscription, + expected_recent_channel_no_vids_in_range_download, + output_directory, + ): + transaction_log = recent_channel_no_vids_in_range_subscription.download(dry_run=True) + expected_recent_channel_no_vids_in_range_download.assert_dry_run_files_logged( + transaction_log=transaction_log + ) + def test_rolling_recent_channel_download( self, recent_channel_subscription, diff --git a/tests/e2e/youtube/test_merge_playlist.py b/tests/e2e/youtube/test_merge_playlist.py index 9d9ee8dc..e9bade9b 100644 --- a/tests/e2e/youtube/test_merge_playlist.py +++ b/tests/e2e/youtube/test_merge_playlist.py @@ -81,8 +81,14 @@ class TestYoutubeMergePlaylist: files exist and have the expected md5 file hashes. """ - def test_playlist_download( + def test_merge_playlist_download( self, playlist_subscription, expected_playlist_download, output_directory ): playlist_subscription.download() expected_playlist_download.assert_files_exist(relative_directory=output_directory) + + def test_merge_playlist_dry_run( + self, playlist_subscription, expected_playlist_download, output_directory + ): + transaction_log = playlist_subscription.download(dry_run=True) + expected_playlist_download.assert_dry_run_files_logged(transaction_log=transaction_log) diff --git a/tests/e2e/youtube/test_playlist_as_kodi_music_videos.py b/tests/e2e/youtube/test_playlist_as_kodi_music_videos.py index 39e64400..f8f09699 100644 --- a/tests/e2e/youtube/test_playlist_as_kodi_music_videos.py +++ b/tests/e2e/youtube/test_playlist_as_kodi_music_videos.py @@ -148,8 +148,22 @@ class TestPlaylistAsKodiMusicVideo: playlist_subscription.download() expected_playlist_download.assert_files_exist(relative_directory=output_directory) + def test_playlist_dry_run( + self, playlist_subscription, expected_playlist_download, output_directory + ): + file_transaction_log = playlist_subscription.download(dry_run=True) + expected_playlist_download.assert_dry_run_files_logged(transaction_log=file_transaction_log) + def test_single_video_download( self, single_video_subscription, expected_single_video_download, output_directory ): single_video_subscription.download() expected_single_video_download.assert_files_exist(relative_directory=output_directory) + + def test_single_video_dry_run( + self, single_video_subscription, expected_single_video_download, output_directory + ): + file_transaction_log = single_video_subscription.download(dry_run=True) + expected_single_video_download.assert_dry_run_files_logged( + transaction_log=file_transaction_log + ) diff --git a/tests/e2e/youtube/test_split_video.py b/tests/e2e/youtube/test_split_video.py index 4a71de93..d0602fa5 100644 --- a/tests/e2e/youtube/test_split_video.py +++ b/tests/e2e/youtube/test_split_video.py @@ -119,8 +119,12 @@ class TestPlaylistAsKodiMusicVideo: files exist and have the expected md5 file hashes. """ - def test_single_video_download( + def test_split_video_download( self, single_video_subscription, expected_single_video_download, output_directory ): single_video_subscription.download() expected_single_video_download.assert_files_exist(relative_directory=output_directory) + + def test_split_video_dry_run(self, single_video_subscription, expected_single_video_download): + transaction_log = single_video_subscription.download(dry_run=True) + expected_single_video_download.assert_dry_run_files_logged(transaction_log=transaction_log)