Add --dry-run argument logic (#82)

* dry-run huge refactor

* small fixes, all tests passing

* file handler, transaction log, metadata

* dry-run actually dry running

* working tests, still have more

* Everything passing besides channel

* fix youtube channel thumbnail issue, need to fix fixtures on test

* lint 10, channel tests passing. Need dry-run for split and merge video

* all tests passing, lint 10, should test dry-run arg
This commit is contained in:
Jesse Bannon 2022-07-02 11:24:46 -07:00 committed by GitHub
parent e5b925fb57
commit 9df40497e2
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
22 changed files with 701 additions and 261 deletions

View file

@ -43,9 +43,9 @@ presets:
# store previously downloaded song IDs to tell YTDL not to re-download
# them on a successive invocation.
output_options:
output_directory: "/{music_directory}/{artist_sanitized}"
file_name: "{album_directory_name}/{track_number_padded} - {title_sanitized}.{ext}"
thumbnail_name: "{album_directory_name}/folder.jpg"
output_directory: "{music_directory}"
file_name: "{artist_sanitized}/{album_directory_name}/{track_number_padded} - {title_sanitized}.{ext}"
thumbnail_name: "{artist_sanitized}/{album_directory_name}/folder.jpg"
maintain_download_archive: True
# For each song downloaded, populate the audio file with music tags.

View file

@ -1,9 +1,9 @@
import argparse
from ytdl_sub.utils.logger import LoggerLevels
###################################################################################################
# GLOBAL PARSER
from ytdl_sub.utils.logger import LoggerLevels
parser = argparse.ArgumentParser(
description="ytdl-sub: Automate download and adding metadata with YoutubeDL"
)
@ -15,6 +15,11 @@ parser.add_argument(
help="path to the config yaml, uses config.yaml if not provided",
default="config.yaml",
)
parser.add_argument(
"--dry-run",
action="store_true",
help="does not perform any video downloads or writes to output directories",
)
parser.add_argument(
"--log-level",
metavar="|".join(LoggerLevels.names()),

View file

@ -20,6 +20,8 @@ from ytdl_sub.entries.base_entry import BaseEntry
from ytdl_sub.entries.entry import Entry
from ytdl_sub.utils.logger import Logger
from ytdl_sub.validators.strict_dict_validator import StrictDictValidator
from ytdl_sub.ytdl_additions.enhanced_download_archive import DownloadArchiver
from ytdl_sub.ytdl_additions.enhanced_download_archive import EnhancedDownloadArchive
logger = Logger.get(name="downloader")
@ -35,7 +37,7 @@ DownloaderEntryT = TypeVar("DownloaderEntryT", bound=Entry)
DownloaderParentEntryT = TypeVar("DownloaderParentEntryT", bound=BaseEntry)
class Downloader(Generic[DownloaderOptionsT, DownloaderEntryT], ABC):
class Downloader(DownloadArchiver, Generic[DownloaderOptionsT, DownloaderEntryT], ABC):
"""
Class that interacts with ytdl to perform the download of metadata and content,
and should translate that to list of Entry objects.
@ -66,7 +68,6 @@ class Downloader(Generic[DownloaderOptionsT, DownloaderEntryT], ABC):
cls,
working_directory: str,
ytdl_options: Optional[Dict],
download_archive_file_name: Optional[str],
) -> Dict:
"""Configure the ytdl options for the downloader"""
if ytdl_options is None:
@ -81,39 +82,29 @@ class Downloader(Generic[DownloaderOptionsT, DownloaderEntryT], ABC):
# Overwrite the output location with the specified working directory
ytdl_options["outtmpl"] = str(Path(working_directory) / "%(id)s.%(ext)s")
# If a download archive file name is provided, set it to that
if download_archive_file_name:
ytdl_options["download_archive"] = str(
Path(working_directory) / download_archive_file_name
)
return ytdl_options
def __init__(
self,
working_directory: str,
download_options: DownloaderOptionsT,
enhanced_download_archive: EnhancedDownloadArchive,
ytdl_options: Optional[Dict] = None,
download_archive_file_name: Optional[str] = None,
):
"""
Parameters
----------
working_directory
Path to the working directory
download_options
Options validator for this downloader
enhanced_download_archive
Download archive
ytdl_options
YTDL options validator
download_archive_file_name
Optional. Name of the download archive file that should reside in the working directory
"""
self.working_directory = working_directory
DownloadArchiver.__init__(self=self, enhanced_download_archive=enhanced_download_archive)
self.download_options = download_options
self.ytdl_options = self._configure_ytdl_options(
ytdl_options=ytdl_options,
working_directory=self.working_directory,
download_archive_file_name=download_archive_file_name,
)
@contextmanager
@ -129,6 +120,15 @@ class Downloader(Generic[DownloaderOptionsT, DownloaderEntryT], ABC):
with ytdl.YoutubeDL(ytdl_options) as ytdl_downloader:
yield ytdl_downloader
@property
def is_dry_run(self) -> bool:
"""
Returns
-------
True if dry-run is enabled. False otherwise.
"""
return self.ytdl_options.get("skip_download", False)
def extract_info(self, ytdl_options_overrides: Optional[Dict] = None, **kwargs) -> Dict:
"""
Wrapper around yt_dlp.YoutubeDL.YoutubeDL.extract_info
@ -200,19 +200,14 @@ class Downloader(Generic[DownloaderOptionsT, DownloaderEntryT], ABC):
def download(self) -> List[DownloaderEntryT]:
"""The function to perform the download of all media entries"""
def post_download(self, overrides: Overrides, output_directory: str):
def post_download(self, overrides: Overrides):
"""
After all media entries have been downloaded, post processed, and moved to the output
directory, run this function. This lets the downloader add any extra files directly to the
output directory, for things like YT channel image, banner.
This ideally should not perform any extra downloads, but rather, use the content already
downloaded in the working directory and use it in the output directory.
Parameters
----------
overrides:
Subscription overrides
output_directory:
Output directory to potentially store extra files downloaded
"""

View file

@ -100,8 +100,7 @@ class YoutubeMergePlaylistDownloader(
},
)
@classmethod
def _add_chapters(cls, merged_video: YoutubeVideo) -> None:
def _add_chapters(self, merged_video: YoutubeVideo) -> None:
titles: List[str] = []
timestamps: List[Timestamp] = []
@ -112,11 +111,13 @@ class YoutubeMergePlaylistDownloader(
current_timestamp_sec += video_entry["duration"]
add_ffmpeg_metadata(
file_path=merged_video.get_download_file_path(),
chapters=Chapters(timestamps=timestamps, titles=titles),
file_duration_sec=merged_video.kwargs("duration"),
)
# TODO: return chapter metadata here
if not self.is_dry_run:
add_ffmpeg_metadata(
file_path=merged_video.get_download_file_path(),
chapters=Chapters(timestamps=timestamps, titles=titles),
file_duration_sec=merged_video.kwargs("duration"),
)
def _to_merged_video(self, entry_dict: Dict) -> YoutubeVideo:
"""
@ -130,7 +131,11 @@ class YoutubeMergePlaylistDownloader(
entry_dict["duration"] = sum(
playlist_entry["duration"] for playlist_entry in entry_dict["entries"]
)
entry_dict["ext"] = entry_dict["requested_downloads"][0]["ext"]
entry_dict["ext"] = (
entry_dict["requested_downloads"][0]["ext"]
if "requested_downloads" in entry_dict
else "mkv"
)
return YoutubeVideo(entry_dict=entry_dict, working_directory=self.working_directory)
def download(self) -> List[YoutubeVideo]:

View file

@ -1,7 +1,6 @@
import copy
import re
from pathlib import Path
from shutil import copyfile
from typing import Dict
from typing import List
@ -12,8 +11,6 @@ from ytdl_sub.entries.youtube import YoutubeVideo
from ytdl_sub.utils.chapters import Chapters
from ytdl_sub.utils.chapters import Timestamp
from ytdl_sub.utils.ffmpeg import FFMPEG
from ytdl_sub.utils.thumbnail import convert_download_thumbnail
from ytdl_sub.validators.validators import StringValidator
# Captures the following formats:
# 0:00 title
@ -21,6 +18,10 @@ from ytdl_sub.validators.validators import StringValidator
# 1:00:00 title
# 01:00:00 title
# where capture group 1 and 2 are the timestamp and title, respectively
from ytdl_sub.utils.file_handler import FileHandler
from ytdl_sub.utils.thumbnail import convert_download_thumbnail
from ytdl_sub.validators.validators import StringValidator
_SPLIT_TIMESTAMP_REGEX = re.compile(r"^((?:\d\d:)?(?:\d:)?(?:\d)?\d:\d\d) (.+)$")
@ -154,24 +155,27 @@ class YoutubeSplitVideoDownloader(
for idx, title in enumerate(chapters.titles):
new_uid = _split_video_uid(source_uid=entry.uid, idx=idx)
# Get the input/output file paths
input_file = entry.get_download_file_path()
output_file = str(Path(self.working_directory) / f"{new_uid}.{entry.ext}")
output_thumbnail_file = str(
Path(self.working_directory) / f"{new_uid}.{entry.thumbnail_ext}"
)
if not self.is_dry_run:
# Get the input/output file paths
input_file = entry.get_download_file_path()
output_file = str(Path(self.working_directory) / f"{new_uid}.{entry.ext}")
# Run ffmpeg to create the split the video
FFMPEG.run(
_split_video_ffmpeg_cmd(
input_file=input_file,
output_file=output_file,
timestamps=chapters.timestamps,
idx=idx,
# Run ffmpeg to create the split the video
FFMPEG.run(
_split_video_ffmpeg_cmd(
input_file=input_file,
output_file=output_file,
timestamps=chapters.timestamps,
idx=idx,
)
)
# Copy the original vid thumbnail to the working directory with the new uid. This so
# downstream logic thinks this split video has its own thumbnail
FileHandler.copy(
src_file_path=entry.get_download_thumbnail_path(),
dst_file_path=Path(self.working_directory) / f"{new_uid}.{entry.thumbnail_ext}",
)
# Copy the thumbnail
copyfile(src=entry.get_download_thumbnail_path(), dst=output_thumbnail_file)
# Format the split video as a YoutubePlaylistVideo
split_videos.append(

View file

@ -20,6 +20,7 @@ from ytdl_sub.validators.string_formatter_validators import OverridesStringForma
from ytdl_sub.validators.url_validator import YoutubeChannelUrlValidator
from ytdl_sub.validators.url_validator import YoutubePlaylistUrlValidator
from ytdl_sub.validators.url_validator import YoutubeVideoUrlValidator
from ytdl_sub.ytdl_additions.enhanced_download_archive import EnhancedDownloadArchive
logger = Logger.get()
@ -290,16 +291,14 @@ class YoutubeChannelDownloader(YoutubeDownloader[YoutubeChannelDownloaderOptions
def __init__(
self,
working_directory: str,
download_options: DownloaderOptionsT,
enhanced_download_archive: EnhancedDownloadArchive,
ytdl_options: Optional[Dict] = None,
download_archive_file_name: Optional[str] = None,
):
super().__init__(
working_directory=working_directory,
download_options=download_options,
enhanced_download_archive=enhanced_download_archive,
ytdl_options=ytdl_options,
download_archive_file_name=download_archive_file_name,
)
self.channel: Optional[YoutubeChannel] = None
@ -354,7 +353,7 @@ class YoutubeChannelDownloader(YoutubeDownloader[YoutubeChannelDownloaderOptions
thumbnail_url=thumbnail_url, output_thumbnail_path=output_thumbnail_path
)
def post_download(self, overrides: Overrides, output_directory: str):
def post_download(self, overrides: Overrides):
"""
Downloads and moves channel avatar and banner images to the output directory.
@ -362,17 +361,17 @@ class YoutubeChannelDownloader(YoutubeDownloader[YoutubeChannelDownloaderOptions
----------
overrides
Overrides that can contain variables in the avatar or banner file path
output_directory
Output directory path
"""
avatar_thumbnail_name = overrides.apply_formatter(self.download_options.channel_avatar_path)
self._download_thumbnail(
thumbnail_url=self.channel.avatar_thumbnail_url(),
output_thumbnail_path=str(Path(output_directory) / avatar_thumbnail_name),
output_thumbnail_path=str(Path(self.working_directory) / avatar_thumbnail_name),
)
self.save_file(file_name=avatar_thumbnail_name)
banner_thumbnail_name = overrides.apply_formatter(self.download_options.channel_banner_path)
self._download_thumbnail(
thumbnail_url=self.channel.banner_thumbnail_url(),
output_thumbnail_path=str(Path(output_directory) / banner_thumbnail_name),
output_thumbnail_path=str(Path(self.working_directory) / banner_thumbnail_name),
)
self.save_file(file_name=banner_thumbnail_name)

View file

@ -11,13 +11,29 @@ class Entry(EntryVariables, BaseEntry):
Entry object to represent a single media object returned from yt-dlp.
"""
def get_download_file_name(self) -> str:
"""
Returns
-------
The entry's file name
"""
return f"{self.uid}.{self.ext}"
def get_download_file_path(self) -> str:
"""Returns the entry's file path to where it was downloaded"""
return str(Path(self.working_directory()) / f"{self.uid}.{self.ext}")
return str(Path(self.working_directory()) / self.get_download_file_name())
def get_download_thumbnail_name(self) -> str:
"""
Returns
-------
The download thumbnail's file name
"""
return f"{self.uid}.{self.thumbnail_ext}"
def get_download_thumbnail_path(self) -> str:
"""Returns the entry's thumbnail's file path to where it was downloaded"""
return str(Path(self.working_directory()) / f"{self.uid}.{self.thumbnail_ext}")
return str(Path(self.working_directory()) / self.get_download_thumbnail_name())
@final
def to_dict(self) -> Dict[str, str]:

View file

@ -17,8 +17,12 @@ def _download_subscriptions_from_yaml_files(config: ConfigFile, args: argparse.N
"""
Downloads all subscriptions from one or many subscription yaml files.
:param config: Configuration file
:param args: Arguments from argparse
Parameters
----------
config
Configuration file
args
Arguments from argparse
"""
preset_paths: List[str] = args.subscription_paths
presets: List[Preset] = []
@ -30,15 +34,23 @@ def _download_subscriptions_from_yaml_files(config: ConfigFile, args: argparse.N
subscription = Subscription.from_preset(preset=preset, config=config)
logger.info("Beginning subscription download for %s", subscription.name)
subscription.download()
subscription.download(dry_run=args.dry_run)
def _download_subscription_from_cli(config: ConfigFile, extra_args: List[str]) -> None:
def _download_subscription_from_cli(
config: ConfigFile, args: argparse.Namespace, extra_args: List[str]
) -> None:
"""
Downloads a one-off subscription using the CLI
:param config: Configuration file
:param extra_args: Extra arguments from argparse that contain dynamic subscription options
Parameters
----------
config
Configuration file
args
Arguments from argparse
extra_args
Extra arguments from argparse that contain dynamic subscription options
"""
dl_args_parser = DownloadArgsParser(
extra_arguments=extra_args, config_options=config.config_options
@ -57,7 +69,7 @@ def _download_subscription_from_cli(config: ConfigFile, extra_args: List[str]) -
config=config,
)
subscription.download()
subscription.download(dry_run=args.dry_run)
def _main():
@ -80,7 +92,7 @@ def _main():
# One-off download
if args.subparser == "dl":
_download_subscription_from_cli(config=config, extra_args=extra_args)
_download_subscription_from_cli(config=config, args=args, extra_args=extra_args)
logger.info("Download complete!")
# Ran successfully, so we can delete the debug file

View file

@ -1,8 +1,11 @@
from typing import Dict
import mediafile
from ytdl_sub.entries.entry import Entry
from ytdl_sub.plugins.plugin import Plugin
from ytdl_sub.plugins.plugin import PluginOptions
from ytdl_sub.utils.file_handler import FileMetadata
from ytdl_sub.validators.string_formatter_validators import DictFormatterValidator
from ytdl_sub.validators.validators import StringValidator
@ -57,17 +60,28 @@ class MusicTagsPlugin(Plugin[MusicTagsOptions]):
"""
Tags the entry's audio file using values defined in the metadata options
"""
audio_file = mediafile.MediaFile(entry.get_download_file_path())
for tag, tag_formatter in self.plugin_options.tags.dict.items():
if tag not in audio_file.fields():
supported_fields = list(mediafile.MediaFile.sorted_fields())
tags_to_write: Dict[str, str] = {}
for tag_name, tag_formatter in self.plugin_options.tags.dict.items():
if tag_name not in supported_fields:
# TODO: Add support for custom fields
self._logger.warning(
"tag '%s' is not supported for %s files. Supported tags: %s",
tag,
tag_name,
entry.ext,
", ".join(audio_file.sorted_fields()),
", ".join(sorted(supported_fields)),
)
continue
tag_value = self.overrides.apply_formatter(formatter=tag_formatter, entry=entry)
setattr(audio_file, tag, tag_value)
tags_to_write[tag_name] = tag_value
audio_file.save()
# write the actual tags if its not a dry run
if not self.is_dry_run:
audio_file = mediafile.MediaFile(entry.get_download_file_path())
for tag_name, tag_value in tags_to_write.items():
setattr(audio_file, tag_name, tag_value)
audio_file.save()
# report the tags written
return FileMetadata.from_dict(value_dict=tags_to_write, title="Music Tags:")

View file

@ -6,6 +6,7 @@ import dicttoxml
from ytdl_sub.entries.entry import Entry
from ytdl_sub.plugins.plugin import Plugin
from ytdl_sub.plugins.plugin import PluginOptions
from ytdl_sub.utils.file_handler import FileMetadata
from ytdl_sub.validators.string_formatter_validators import DictFormatterValidator
from ytdl_sub.validators.string_formatter_validators import StringFormatterValidator
@ -109,10 +110,12 @@ class NfoTagsPlugin(Plugin[NfoTagsOptions]):
)
# Save the nfo's XML to file
nfo_file_path = Path(self.output_directory) / nfo_file_name
nfo_file_path = Path(self.working_directory) / nfo_file_name
os.makedirs(os.path.dirname(nfo_file_path), exist_ok=True)
with open(nfo_file_path, "wb") as nfo_file:
nfo_file.write(xml)
# Archive the nfo's file name
self.archive_entry_file_name(entry=entry, relative_file_path=nfo_file_name)
self.save_file(file_name=nfo_file_name, entry=entry)
return FileMetadata.from_dict(value_dict={nfo_root: nfo}, title="NFO tags:")

View file

@ -5,6 +5,7 @@ import dicttoxml
from ytdl_sub.plugins.plugin import Plugin
from ytdl_sub.plugins.plugin import PluginOptions
from ytdl_sub.utils.file_handler import FileMetadata
from ytdl_sub.validators.string_formatter_validators import OverridesDictFormatterValidator
from ytdl_sub.validators.string_formatter_validators import OverridesStringFormatterValidator
@ -100,7 +101,10 @@ class OutputDirectoryNfoTagsPlugin(Plugin[OutputDirectoryNfoTagsOptions]):
nfo_file_name = self.overrides.apply_formatter(formatter=self.plugin_options.nfo_name)
# Save the nfo's XML to file
nfo_file_path = Path(self.output_directory) / nfo_file_name
nfo_file_path = Path(self.working_directory) / nfo_file_name
os.makedirs(os.path.dirname(nfo_file_path), exist_ok=True)
with open(nfo_file_path, "wb") as nfo_file:
nfo_file.write(xml)
self.save_file(file_name=nfo_file_name)
return FileMetadata.from_dict(value_dict={nfo_root: nfo}, title="NFO tags:")

View file

@ -7,8 +7,10 @@ from typing import final
from ytdl_sub.config.preset_options import Overrides
from ytdl_sub.entries.entry import Entry
from ytdl_sub.utils.file_handler import FileMetadata
from ytdl_sub.utils.logger import Logger
from ytdl_sub.validators.strict_dict_validator import StrictDictValidator
from ytdl_sub.ytdl_additions.enhanced_download_archive import DownloadArchiver
from ytdl_sub.ytdl_additions.enhanced_download_archive import EnhancedDownloadArchive
@ -21,7 +23,7 @@ class PluginOptions(StrictDictValidator):
PluginOptionsT = TypeVar("PluginOptionsT", bound=PluginOptions)
class Plugin(Generic[PluginOptionsT], ABC):
class Plugin(DownloadArchiver, Generic[PluginOptionsT], ABC):
"""
Class to define the new plugin functionality
"""
@ -32,36 +34,16 @@ class Plugin(Generic[PluginOptionsT], ABC):
def __init__(
self,
plugin_options: PluginOptionsT,
output_directory: str,
overrides: Overrides,
enhanced_download_archive: Optional[EnhancedDownloadArchive],
enhanced_download_archive: EnhancedDownloadArchive,
):
DownloadArchiver.__init__(self=self, enhanced_download_archive=enhanced_download_archive)
self.plugin_options = plugin_options
self.output_directory = output_directory
self.overrides = overrides
self.__enhanced_download_archive = enhanced_download_archive
# TODO pass yaml snake case name in the class somewhere, and use it for the logger
self._logger = Logger.get(self.__class__.__name__)
@final
def archive_entry_file_name(self, entry: Entry, relative_file_path: str) -> None:
"""
Adds an entry and a file name that belongs to it into the archive mapping.
If maintain_download_archive is False for the subscription, this method will do nothing.
Parameters
----------
entry:
Optional. The entry the file belongs to
relative_file_path:
The name of the file path relative to the output directory
"""
if self.__enhanced_download_archive:
self.__enhanced_download_archive.mapping.add_entry(
entry=entry, entry_file_path=relative_file_path
)
def post_process_entry(self, entry: Entry):
def post_process_entry(self, entry: Entry) -> Optional[FileMetadata]:
"""
For each file downloaded, apply post processing to it.
@ -69,6 +51,10 @@ class Plugin(Generic[PluginOptionsT], ABC):
----------
entry:
Entry to post process
Returns
-------
Optional file metadata for the entry media file.
"""
def post_process_subscription(self):

View file

@ -1,9 +1,10 @@
import contextlib
import copy
import os
import shutil
from pathlib import Path
from shutil import copyfile
from typing import List
from typing import Optional
from typing import Tuple
from typing import Type
@ -18,6 +19,8 @@ from ytdl_sub.downloaders.downloader import DownloaderValidator
from ytdl_sub.entries.entry import Entry
from ytdl_sub.plugins.plugin import Plugin
from ytdl_sub.plugins.plugin import PluginOptions
from ytdl_sub.utils.file_handler import FileHandlerTransactionLog
from ytdl_sub.utils.file_handler import FileMetadata
from ytdl_sub.utils.thumbnail import convert_download_thumbnail
from ytdl_sub.ytdl_additions.enhanced_download_archive import EnhancedDownloadArchive
@ -144,30 +147,9 @@ class Subscription:
and self.downloader_class.supports_download_archive
)
def _copy_file_to_output_directory(
self, entry: Entry, source_file_path: str, output_file_name: str
def _copy_entry_files_to_output_directory(
self, entry: Entry, entry_metadata: Optional[FileMetadata] = None
):
"""
Helper function to move a file from the working directory to the output directory.
Will add it to the download archive mapping if the archive is being maintained.
Parameters
----------
entry:
Entry that the file belongs to
source_file_path:
Path to the source file
output_file_name:
Desired output file name with the output directory as its relative directory
"""
destination_file_path = Path(self.output_directory) / Path(output_file_name)
os.makedirs(os.path.dirname(destination_file_path), exist_ok=True)
copyfile(source_file_path, destination_file_path)
if self.maintain_download_archive:
self._enhanced_download_archive.mapping.add_entry(entry, output_file_name)
def _copy_entry_files_to_output_directory(self, entry: Entry):
"""
Helper function to move the media file and optionally thumbnail file to the output directory
for a single entry.
@ -176,15 +158,18 @@ class Subscription:
----------
entry:
The entry with files to move
entry_metadata
Optional. Metadata to record to the transaction log for this entry
"""
# Move the file after all direct file modifications are complete
output_file_name = self.overrides.apply_formatter(
formatter=self.output_options.file_name, entry=entry
)
self._copy_file_to_output_directory(
entry=entry,
source_file_path=entry.get_download_file_path(),
self._enhanced_download_archive.save_file_to_output_directory(
file_name=entry.get_download_file_name(),
file_metadata=entry_metadata,
output_file_name=output_file_name,
entry=entry,
)
if self.output_options.thumbnail_name:
@ -195,10 +180,10 @@ class Subscription:
# We always convert entry thumbnails to jpgs, and is performed here
convert_download_thumbnail(entry=entry)
self._copy_file_to_output_directory(
entry=entry,
source_file_path=entry.get_download_thumbnail_path(),
self._enhanced_download_archive.save_file_to_output_directory(
file_name=entry.get_download_thumbnail_name(),
output_file_name=output_thumbnail_name,
entry=entry,
)
@contextlib.contextmanager
@ -243,45 +228,59 @@ class Subscription:
for plugin_type, plugin_options in self.plugins:
plugin = plugin_type(
plugin_options=plugin_options,
output_directory=self.output_directory,
overrides=self.overrides,
enhanced_download_archive=self._enhanced_download_archive
if self.maintain_download_archive
else None,
enhanced_download_archive=self._enhanced_download_archive,
)
plugins.append(plugin)
return plugins
def download(self):
def download(self, dry_run: bool = False) -> FileHandlerTransactionLog:
"""
Performs the subscription download.
Performs the subscription download
Parameters
----------
dry_run
If true, do not download any video/audio files or move anything to the output
directory.
"""
self._enhanced_download_archive.reinitialize(dry_run=dry_run)
# TODO: Move this logic to separate function
# TODO: set id here as well
ytdl_options = copy.deepcopy(self.ytdl_options.dict)
if dry_run:
ytdl_options["skip_download"] = True
if self.downloader_class.supports_download_archive and self.maintain_download_archive:
ytdl_options["download_archive"] = str(
Path(self.working_directory) / self._enhanced_download_archive.archive_file_name
)
plugins = self._initialize_plugins()
with self._prepare_working_directory(), self._maintain_archive_file():
downloader = self.downloader_class(
working_directory=self.working_directory,
download_options=self.downloader_options,
ytdl_options=self.ytdl_options.dict,
download_archive_file_name=self._enhanced_download_archive.archive_file_name
if self.maintain_download_archive
else None,
enhanced_download_archive=self._enhanced_download_archive,
ytdl_options=ytdl_options,
)
entries = downloader.download()
for plugin in plugins:
for entry in entries:
plugin.post_process_entry(entry)
for entry in downloader.download():
# TODO: Add entry metadata from the downloader.download function
entry_metadata = FileMetadata()
for plugin in plugins:
entry_metadata.extend(plugin.post_process_entry(entry))
self._copy_entry_files_to_output_directory(
entry=entry, entry_metadata=entry_metadata
)
downloader.post_download(overrides=self.overrides)
for plugin in plugins:
plugin.post_process_subscription()
for entry in entries:
self._copy_entry_files_to_output_directory(entry=entry)
downloader.post_download(
overrides=self.overrides, output_directory=self.output_directory
)
return self._enhanced_download_archive.get_file_handler_transaction_log()
@classmethod
def from_preset(cls, preset: Preset, config: ConfigFile) -> "Subscription":

View file

@ -0,0 +1,195 @@
import os
from pathlib import Path
from shutil import copyfile
from typing import Any
from typing import Dict
from typing import List
from typing import Optional
from typing import Set
from typing import Union
class FileMetadata:
"""
Stores pretty-printed information about a file. Each line in the metadata represents a newline
"""
def __init__(self, metadata: Optional[List[str]] = None):
self.metadata: List[str] = metadata if metadata else []
def append(self, line: str) -> "FileMetadata":
"""
Parameters
----------
line
Line of metadata to append
"""
self.metadata.append(line)
return self
def extend(self, other: Optional["FileMetadata"]) -> "FileMetadata":
"""
Parameters
----------
other
Other metadata to extend to this one in its entirety
"""
if other is not None:
self.metadata.extend(other.metadata)
return self
@classmethod
def from_dict(cls, value_dict: Dict[str, Any], title: Optional[str] = None) -> "FileMetadata":
"""
Parameters
----------
value_dict
Dict of things to print indented
title
Optional. Title line to put above the dict
"""
lines: List[str] = []
if title is not None:
lines.append(title)
def _recursive_add_dict_lines(rdict: Dict, indent: int):
for key, value in sorted(rdict.items()):
if isinstance(value, Dict):
_recursive_add_dict_lines(rdict=value, indent=indent + 2)
_indent = " " * indent
lines.append(f"{_indent}{key}: {value}")
_recursive_add_dict_lines(rdict=value_dict, indent=2)
return cls(metadata=lines)
class FileHandlerTransactionLog:
"""
Tracks file 'transactions' performed by a FileHandler
"""
def __init__(self):
self.files_created: Dict[str, FileMetadata] = {}
self.files_removed: Set[str] = set()
def log_created_file(
self, file_name: str, file_metadata: Optional[FileMetadata] = None
) -> "FileHandlerTransactionLog":
"""
Adds a created file to the transaction log
Parameters
----------
file_name
Name of the file in the output directory
file_metadata
Optional. If the file has metadata, add it to the transaction log
"""
if not file_metadata:
file_metadata = FileMetadata()
self.files_created[file_name] = file_metadata
return self
def log_removed_file(self, file_name: str) -> "FileHandlerTransactionLog":
"""
Records a file removed from the output directory
Parameters
----------
file_name
Name of the file in the output directory getting removed
"""
self.files_removed.add(file_name)
return self
class FileHandler:
"""
Performs and tracks all file moving/copying/deleting
"""
def __init__(self, working_directory: str, output_directory: str, dry_run: bool):
self.dry_run = dry_run
self.working_directory = working_directory
self.output_directory = output_directory
self._file_handler_transaction_log = FileHandlerTransactionLog()
@property
def file_handler_transaction_log(self) -> FileHandlerTransactionLog:
"""
Returns
-------
Transaction logs of this file handler
"""
return self._file_handler_transaction_log
@classmethod
def copy(cls, src_file_path: Union[str, Path], dst_file_path: Union[str, Path]):
"""
Parameters
----------
src_file_path
Source file
dst_file_path
Destination file
"""
copyfile(src=src_file_path, dst=dst_file_path)
@classmethod
def delete(cls, file_path: Union[str, Path]):
"""
Parameters
----------
file_path
File to delete
"""
if os.path.isfile(file_path):
os.remove(file_path)
def copy_file_to_output_directory(
self, file_name: str, output_file_name: str, file_metadata: Optional[FileMetadata] = None
):
"""
Copies a file from the working directory to the output directory.
All file copies from working to output directory should use this function for tracking and
handling dry-run logic.
Parameters
----------
file_name
File in the working directory
output_file_name
Desired output file name in the output_directory
file_metadata
Optional. Metadata to record to the transaction log for this file
"""
self._file_handler_transaction_log.log_created_file(
file_name=output_file_name, file_metadata=file_metadata
)
if not self.dry_run:
output_file_path = Path(self.output_directory) / output_file_name
os.makedirs(os.path.dirname(output_file_path), exist_ok=True)
self.copy(
src_file_path=Path(self.working_directory) / file_name,
dst_file_path=output_file_path,
)
def delete_file_from_output_directory(self, file_name: str):
"""
Deletes a file from the output directory. All file deletions should use this function
for tracking and handling dry-run logic.
Parameters
----------
file_name
File in the output directory to delete
"""
file_path = Path(self.output_directory) / file_name
exists = os.path.isfile(file_path)
if exists:
self._file_handler_transaction_log.log_removed_file(file_name)
if not self.dry_run:
self.delete(file_path=file_path)

View file

@ -1,13 +1,14 @@
import contextlib
import io
import logging
import os
import sys
import tempfile
from dataclasses import dataclass
from typing import List
from typing import Optional
from ytdl_sub.utils.file_handler import FileHandler
@dataclass
class LoggerLevel:
@ -201,5 +202,5 @@ class Logger:
"""
cls._DEBUG_LOGGER_FILE.close()
if delete_debug_file and os.path.isfile(cls.debug_log_filename()):
os.remove(cls.debug_log_filename())
if delete_debug_file:
FileHandler.delete(cls.debug_log_filename())

View file

@ -13,6 +13,9 @@ from typing import Set
from yt_dlp import DateRange
from ytdl_sub.entries.entry import Entry
from ytdl_sub.utils.file_handler import FileHandler
from ytdl_sub.utils.file_handler import FileHandlerTransactionLog
from ytdl_sub.utils.file_handler import FileMetadata
from ytdl_sub.utils.logger import Logger
@ -210,7 +213,7 @@ class DownloadMappings:
entry
Entry that this file belongs to
entry_file_path
Relative path to the file that belongs to the entry
Relative path to the file that lives in the output directory
Returns
-------
@ -334,16 +337,52 @@ class EnhancedDownloadArchive:
6. ( Delete the working directory )
"""
def __init__(self, subscription_name: str, working_directory: str, output_directory: str):
def __init__(
self,
subscription_name: str,
working_directory: str,
output_directory: str,
dry_run: bool = False,
):
self.subscription_name = subscription_name
self.working_directory = working_directory
self.output_directory = output_directory
self._download_archive: Optional[DownloadArchive] = None
self._download_mapping: Optional[DownloadMappings] = None
self._file_handler = FileHandler(
working_directory=working_directory, output_directory=output_directory, dry_run=dry_run
)
self._download_mapping = DownloadMappings()
self._logger = Logger.get(name=subscription_name)
def reinitialize(self, dry_run: bool) -> "EnhancedDownloadArchive":
"""
Re-initialize the enhanced download archive for successive downloads w/the same
subscription.
Parameters
----------
dry_run
Whether to actually move files to the output directory
Returns
-------
self
"""
self._file_handler = FileHandler(
working_directory=self.working_directory,
output_directory=self.output_directory,
dry_run=dry_run,
)
self._download_mapping = DownloadMappings()
return self
@property
def is_dry_run(self) -> bool:
"""
Returns
-------
True if this session is a dry-run. False otherwise.
"""
return self._file_handler.dry_run
@property
def archive_file_name(self) -> str:
"""
@ -353,6 +392,24 @@ class EnhancedDownloadArchive:
"""
return f".ytdl-subscribe-{self.subscription_name}-download-archive.txt"
@property
def working_directory(self) -> str:
"""
Returns
-------
Path to the working directory
"""
return self._file_handler.working_directory
@property
def output_directory(self) -> str:
"""
Returns
-------
Path to the output directory
"""
return self._file_handler.output_directory
@property
def _mapping_file_name(self) -> str:
"""
@ -363,7 +420,7 @@ class EnhancedDownloadArchive:
return f".ytdl-sub-{self.subscription_name}-download-archive.json"
@property
def _mapping_output_file_path(self):
def _mapping_output_file_path(self) -> str:
"""
Returns
-------
@ -371,6 +428,15 @@ class EnhancedDownloadArchive:
"""
return str(Path(self.output_directory) / self._mapping_file_name)
@property
def _mapping_working_file_path(self) -> str:
"""
Returns
-------
The download mapping's file path in the working directory.
"""
return str(Path(self.working_directory) / self._mapping_file_name)
@property
def _archive_working_file_path(self) -> str:
"""
@ -399,7 +465,6 @@ class EnhancedDownloadArchive:
def _load(self) -> "EnhancedDownloadArchive":
"""
Tries to load download mappings if they are present in the output directory.
If they are not, initialize an empty mapping.
Returns
-------
@ -410,14 +475,9 @@ class EnhancedDownloadArchive:
self._download_mapping = DownloadMappings.from_file(
json_file_path=self._mapping_output_file_path
)
# Otherwise, init an empty download mappings object. Keep _download_archive as None to
# indicate it was not loaded
else:
self._download_mapping = DownloadMappings()
return self
def _copy_to_working_directory(self) -> "EnhancedDownloadArchive":
def _copy_mapping_to_working_directory(self) -> "EnhancedDownloadArchive":
"""
If the mapping is not empty, create a download archive from it and save it into the
working directory. This will tell YTDL to not redownload already downloaded entries.
@ -446,7 +506,7 @@ class EnhancedDownloadArchive:
self
"""
self._load()
self._copy_to_working_directory()
self._copy_mapping_to_working_directory()
return self
def remove_stale_files(self, date_range: DateRange) -> "EnhancedDownloadArchive":
@ -468,12 +528,8 @@ class EnhancedDownloadArchive:
)
for uid, mapping in stale_mappings.items():
self._logger.info("[%s] Removing the following stale file(s):", uid)
for file_name in mapping.file_names:
file_path = Path(self.output_directory) / Path(file_name)
self._logger.info(" - %s", file_path)
if os.path.exists(file_path):
os.remove(file_path)
self._file_handler.delete_file_from_output_directory(file_name=file_name)
self.mapping.remove_entry(entry_id=uid)
@ -487,17 +543,96 @@ class EnhancedDownloadArchive:
-------
self
"""
# TODO: Make sure this logic is actually right...
# Load the download archive from the working directory, which should contain any past
# and new entries downloaded in this session
self._download_archive = DownloadArchive.from_file(self._archive_working_file_path)
# Keep the download archive in sync with the mapping
for entry_id in self.mapping.entry_ids:
if not self._download_archive.contains(entry_id):
self._download_archive.remove_entry(entry_id)
# Save the updated mapping file to the output directory
self._download_mapping.to_file(output_json_file=self._mapping_output_file_path)
self._download_mapping.to_file(output_json_file=self._mapping_working_file_path)
self.save_file_to_output_directory(file_name=self._mapping_file_name)
return self
def save_file_to_output_directory(
self,
file_name: str,
file_metadata: Optional[FileMetadata] = None,
output_file_name: Optional[str] = None,
entry: Optional[Entry] = None,
):
"""
Saves a file from the working directory to the output directory and record it in the
transaction log.
Parameters
----------
file_name
Name of the file to move (does not include working directory path)
file_metadata
Optional. Metadata to record to the transaction log for this file
output_file_name
Optional. Final name of the file in the output directory (does not include output
directory path). If None, use the same working_directory file_name
entry
Optional. Entry that this file belongs to
"""
if output_file_name is None:
output_file_name = file_name
if entry:
self.mapping.add_entry(entry=entry, entry_file_path=output_file_name)
self._file_handler.copy_file_to_output_directory(
file_name=file_name, file_metadata=file_metadata, output_file_name=output_file_name
)
def get_file_handler_transaction_log(self) -> FileHandlerTransactionLog:
"""
Returns
-------
File handler transaction log for this session
"""
return self._file_handler.file_handler_transaction_log
class DownloadArchiver:
"""
Used for any class that saves files. Does not allow direct access to output_directory,
forcing the user of the class to use ``save_file`` so it gets archived and avoids any writes
during dry-run.
"""
def __init__(self, enhanced_download_archive: EnhancedDownloadArchive):
self.__enhanced_download_archive = enhanced_download_archive
@property
def working_directory(self) -> str:
"""
Returns
-------
Path to the working directory
"""
return self.__enhanced_download_archive.working_directory
@property
def is_dry_run(self) -> bool:
"""
Returns
-------
True if this session is a dry-run. False otherwise.
"""
return self.__enhanced_download_archive.is_dry_run
def save_file(
self, file_name: str, output_file_name: Optional[str] = None, entry: Optional[Entry] = None
) -> None:
"""
Saves a file in the working directory to the output directory.
Parameters
----------
file_name
Name of the file relative to the working directory
output_file_name
Optional. Final name of the file in the output directory (does not include output
directory path). If None, use the same working_directory file_name
entry
Optional. Entry that the file belongs to
"""
self.__enhanced_download_archive.save_file_to_output_directory(
file_name=file_name, output_file_name=output_file_name, entry=entry
)

View file

@ -6,6 +6,8 @@ from typing import List
from typing import Optional
from typing import Union
from ytdl_sub.utils.file_handler import FileHandlerTransactionLog
class ExpectedDownload:
"""
@ -58,3 +60,13 @@ class ExpectedDownload:
f"MD5 hash for {str(relative_path)} does not match: "
f"{md5_hash} != {expected_md5_hash}"
)
def assert_dry_run_files_logged(self, transaction_log: FileHandlerTransactionLog):
assert (
len(transaction_log.files_created) == self.file_count
), "Mismatch in number of created files"
for relative_path in self.expected_md5_file_hashes.keys():
assert (
str(relative_path) in transaction_log.files_created
), f"Expected {str(relative_path)} to be a file but it is not"

View file

@ -29,7 +29,7 @@ def subscription_dict(output_directory, subscription_name):
"preset": "sc_discography",
"soundcloud": {"url": "https://soundcloud.com/jessebannon"},
# override the output directory with our fixture-generated dir
"output_options": {"output_directory": output_directory + "/{artist_sanitized}"},
"output_options": {"output_directory": output_directory},
# download the worst format so it is fast
"ytdl_options": {
"format": "worst[ext=mp3]",
@ -59,7 +59,7 @@ def expected_discography_download():
return ExpectedDownload(
expected_md5_file_hashes={
# Download mapping
Path("j_b/.ytdl-sub-jb-download-archive.json"): "ae55de93b71267b5712c9a3d06c07c26",
Path(".ytdl-sub-jb-download-archive.json"): "1a99156e9ece62539fb2608416a07200",
# Entry files (singles)
Path("j_b/[2021] Baby Santana's Dorian Groove/01 - Baby Santana's Dorian Groove.mp3"): "bffbd558e12c6a9e029dc136a88342c4",
@ -99,3 +99,9 @@ class TestSoundcloudDiscography:
):
discography_subscription.download()
expected_discography_download.assert_files_exist(relative_directory=output_directory)
def test_discography_dry_run(
self, discography_subscription, expected_discography_download, output_directory
):
transaction_log = discography_subscription.download(dry_run=True)
expected_discography_download.assert_dry_run_files_logged(transaction_log=transaction_log)

View file

@ -27,12 +27,12 @@ def config(config_path):
@pytest.fixture
def subscription_dict(output_directory, subscription_name):
def subscription_dict(output_directory):
return {
"preset": "yt_channel_as_tv",
"youtube": {"channel_url": "https://youtube.com/channel/UCcRSMoQqXc_JrBZRHDFGbqA"},
# override the output directory with our fixture-generated dir
"output_options": {"output_directory": str(Path(output_directory) / subscription_name)},
"output_options": {"output_directory": output_directory},
# download the worst format so it is fast
"ytdl_options": {
"format": "worst[ext=mp4]",
@ -68,61 +68,61 @@ def expected_full_channel_download():
return ExpectedDownload(
expected_md5_file_hashes={
# Download mapping
Path("pz/.ytdl-sub-pz-download-archive.json"): "b7e7c19d2cf0277e4e42453a64fbaa90",
Path(".ytdl-sub-pz-download-archive.json"): "b7e7c19d2cf0277e4e42453a64fbaa90",
# Output directory files
Path("pz/fanart.jpg"): "e6e323373c8902568e96e374817179cf",
Path("pz/poster.jpg"): "a14c593bcc75bb8d2c7145de4767ad01",
Path("pz/tvshow.nfo"): "83c7db96081ac5bdf289fcf396bec157",
Path("fanart.jpg"): "e6e323373c8902568e96e374817179cf",
Path("poster.jpg"): "a14c593bcc75bb8d2c7145de4767ad01",
Path("tvshow.nfo"): "83c7db96081ac5bdf289fcf396bec157",
# Entry files
Path("pz/Season 2010/s2010.e0813 - Oblivion Mod 'Falcor' p.1.jpg"): "b58377dfe7c39527e1990a24b36bbd77",
Path("pz/Season 2010/s2010.e0813 - Oblivion Mod 'Falcor' p.1.mp4"): "931a705864c57d21d6fedebed4af6bbc",
Path("pz/Season 2010/s2010.e0813 - Oblivion Mod 'Falcor' p.1.nfo"): "67d8d71d048039080acbba3bce4febaa",
Path("Season 2010/s2010.e0813 - Oblivion Mod 'Falcor' p.1.jpg"): "b58377dfe7c39527e1990a24b36bbd77",
Path("Season 2010/s2010.e0813 - Oblivion Mod 'Falcor' p.1.mp4"): "931a705864c57d21d6fedebed4af6bbc",
Path("Season 2010/s2010.e0813 - Oblivion Mod 'Falcor' p.1.nfo"): "67d8d71d048039080acbba3bce4febaa",
Path("pz/Season 2010/s2010.e1202 - Oblivion Mod 'Falcor' p.2.jpg"): "a5ee6247c8dce255aec79c9a51d49da4",
Path("pz/Season 2010/s2010.e1202 - Oblivion Mod 'Falcor' p.2.mp4"): "d3469b4dca7139cb3dbc38712b6796bf",
Path("pz/Season 2010/s2010.e1202 - Oblivion Mod 'Falcor' p.2.nfo"): "d81f49cedbd7edaee987521e89b37904",
Path("Season 2010/s2010.e1202 - Oblivion Mod 'Falcor' p.2.jpg"): "a5ee6247c8dce255aec79c9a51d49da4",
Path("Season 2010/s2010.e1202 - Oblivion Mod 'Falcor' p.2.mp4"): "d3469b4dca7139cb3dbc38712b6796bf",
Path("Season 2010/s2010.e1202 - Oblivion Mod 'Falcor' p.2.nfo"): "d81f49cedbd7edaee987521e89b37904",
Path("pz/Season 2011/s2011.e0201 - Jesse's Minecraft Server [Trailer - Feb.1].jpg"): "048a19cf0f674437351872c3f312ebf1",
Path("pz/Season 2011/s2011.e0201 - Jesse's Minecraft Server [Trailer - Feb.1].mp4"): "e66287b9832277b6a4d1554e29d9fdcc",
Path("pz/Season 2011/s2011.e0201 - Jesse's Minecraft Server [Trailer - Feb.1].nfo"): "f7c0de89038f8c491bded8a3968720a2",
Path("Season 2011/s2011.e0201 - Jesse's Minecraft Server [Trailer - Feb.1].jpg"): "048a19cf0f674437351872c3f312ebf1",
Path("Season 2011/s2011.e0201 - Jesse's Minecraft Server [Trailer - Feb.1].mp4"): "e66287b9832277b6a4d1554e29d9fdcc",
Path("Season 2011/s2011.e0201 - Jesse's Minecraft Server [Trailer - Feb.1].nfo"): "f7c0de89038f8c491bded8a3968720a2",
Path("pz/Season 2011/s2011.e0227 - Jesse's Minecraft Server [Trailer - Feb.27].jpg"): None,
Path("pz/Season 2011/s2011.e0227 - Jesse's Minecraft Server [Trailer - Feb.27].mp4"): "04ab5cb3cc12325d0c96a7cd04a8b91d",
Path("pz/Season 2011/s2011.e0227 - Jesse's Minecraft Server [Trailer - Feb.27].nfo"): "ee1eda78fa0980bc703e602b5012dd1f",
Path("Season 2011/s2011.e0227 - Jesse's Minecraft Server [Trailer - Feb.27].jpg"): None,
Path("Season 2011/s2011.e0227 - Jesse's Minecraft Server [Trailer - Feb.27].mp4"): "04ab5cb3cc12325d0c96a7cd04a8b91d",
Path("Season 2011/s2011.e0227 - Jesse's Minecraft Server [Trailer - Feb.27].nfo"): "ee1eda78fa0980bc703e602b5012dd1f",
Path("pz/Season 2011/s2011.e0321 - Jesse's Minecraft Server [Trailer - Mar.21].jpg"): "9baaddc6b62f5b9ae3781eb4eef0e3b3",
Path("pz/Season 2011/s2011.e0321 - Jesse's Minecraft Server [Trailer - Mar.21].mp4"): "025de6099a5c98e6397153c7a62d517d",
Path("pz/Season 2011/s2011.e0321 - Jesse's Minecraft Server [Trailer - Mar.21].nfo"): "61eb6369430da0ab6134d78829a7621b",
Path("Season 2011/s2011.e0321 - Jesse's Minecraft Server [Trailer - Mar.21].jpg"): "9baaddc6b62f5b9ae3781eb4eef0e3b3",
Path("Season 2011/s2011.e0321 - Jesse's Minecraft Server [Trailer - Mar.21].mp4"): "025de6099a5c98e6397153c7a62d517d",
Path("Season 2011/s2011.e0321 - Jesse's Minecraft Server [Trailer - Mar.21].nfo"): "61eb6369430da0ab6134d78829a7621b",
Path("pz/Season 2011/s2011.e0529 - Project Zombie _Official Trailer_ (IP - mc.projectzombie.beastnode.net).jpg"): "ce1df7f623fffaefe04606ecbafcfec6",
Path("pz/Season 2011/s2011.e0529 - Project Zombie _Official Trailer_ (IP - mc.projectzombie.beastnode.net).mp4"): "3d9c19835b03355d6fd5d00cd59dbe5b",
Path("pz/Season 2011/s2011.e0529 - Project Zombie _Official Trailer_ (IP - mc.projectzombie.beastnode.net).nfo"): "60f72b99f5c69f9e03a071a12160928f",
Path("Season 2011/s2011.e0529 - Project Zombie _Official Trailer_ (IP - mc.projectzombie.beastnode.net).jpg"): "ce1df7f623fffaefe04606ecbafcfec6",
Path("Season 2011/s2011.e0529 - Project Zombie _Official Trailer_ (IP - mc.projectzombie.beastnode.net).mp4"): "3d9c19835b03355d6fd5d00cd59dbe5b",
Path("Season 2011/s2011.e0529 - Project Zombie _Official Trailer_ (IP - mc.projectzombie.beastnode.net).nfo"): "60f72b99f5c69f9e03a071a12160928f",
Path("pz/Season 2011/s2011.e0630 - Project Zombie _Fin.jpg"): "bc3f511915869720c37617a7de706b2b",
Path("pz/Season 2011/s2011.e0630 - Project Zombie _Fin.mp4"): "4971cb2d4fa29460361031f3fa8e1ea9",
Path("pz/Season 2011/s2011.e0630 - Project Zombie _Fin.nfo"): "a7b5d9e57d20852f5daf360a1373bb7a",
Path("Season 2011/s2011.e0630 - Project Zombie _Fin.jpg"): "bc3f511915869720c37617a7de706b2b",
Path("Season 2011/s2011.e0630 - Project Zombie _Fin.mp4"): "4971cb2d4fa29460361031f3fa8e1ea9",
Path("Season 2011/s2011.e0630 - Project Zombie _Fin.nfo"): "a7b5d9e57d20852f5daf360a1373bb7a",
Path("pz/Season 2011/s2011.e1121 - Skyrim 'Ultra HD w_Mods' [PC].jpg"): "12babdb3b86cd868b90b60d013295f66",
Path("pz/Season 2011/s2011.e1121 - Skyrim 'Ultra HD w_Mods' [PC].mp4"): "55e9b0add08c48c9c66105da0def2426",
Path("pz/Season 2011/s2011.e1121 - Skyrim 'Ultra HD w_Mods' [PC].nfo"): "fe60e2b6b564f9316b6c7c183e1cf300",
Path("Season 2011/s2011.e1121 - Skyrim 'Ultra HD w_Mods' [PC].jpg"): "12babdb3b86cd868b90b60d013295f66",
Path("Season 2011/s2011.e1121 - Skyrim 'Ultra HD w_Mods' [PC].mp4"): "55e9b0add08c48c9c66105da0def2426",
Path("Season 2011/s2011.e1121 - Skyrim 'Ultra HD w_Mods' [PC].nfo"): "fe60e2b6b564f9316b6c7c183e1cf300",
Path("pz/Season 2012/s2012.e0123 - Project Zombie _Map Trailer.jpg"): "82d303e16aba75acdde30b15c4154231",
Path("pz/Season 2012/s2012.e0123 - Project Zombie _Map Trailer.mp4"): "65e4ce53ed5ec4139995469f99477a50",
Path("pz/Season 2012/s2012.e0123 - Project Zombie _Map Trailer.nfo"): "c8900adcca83c473c79a4afbc7ad2de1",
Path("Season 2012/s2012.e0123 - Project Zombie _Map Trailer.jpg"): "82d303e16aba75acdde30b15c4154231",
Path("Season 2012/s2012.e0123 - Project Zombie _Map Trailer.mp4"): "65e4ce53ed5ec4139995469f99477a50",
Path("Season 2012/s2012.e0123 - Project Zombie _Map Trailer.nfo"): "c8900adcca83c473c79a4afbc7ad2de1",
Path("pz/Season 2013/s2013.e0719 - Project Zombie Rewind _Trailer.jpg"): "83b1af4c3614d262b2ad419586fff730",
Path("pz/Season 2013/s2013.e0719 - Project Zombie Rewind _Trailer.mp4"): "18620a8257a686beda65e54add4d4cd1",
Path("pz/Season 2013/s2013.e0719 - Project Zombie Rewind _Trailer.nfo"): "1c993c41d4308a6049333154d0adee16",
Path("Season 2013/s2013.e0719 - Project Zombie Rewind _Trailer.jpg"): "83b1af4c3614d262b2ad419586fff730",
Path("Season 2013/s2013.e0719 - Project Zombie Rewind _Trailer.mp4"): "18620a8257a686beda65e54add4d4cd1",
Path("Season 2013/s2013.e0719 - Project Zombie Rewind _Trailer.nfo"): "1c993c41d4308a6049333154d0adee16",
Path("pz/Season 2018/s2018.e1029 - Jesse's Minecraft Server _ Teaser Trailer.jpg"): "2a24de903059f48c7d0df0476046c975",
Path("pz/Season 2018/s2018.e1029 - Jesse's Minecraft Server _ Teaser Trailer.mp4"): "82f6ee7253e1dbb83ae7215af08ffacc",
Path("pz/Season 2018/s2018.e1029 - Jesse's Minecraft Server _ Teaser Trailer.nfo"): "cc7886aae3af6b7b0facd82f95390242",
Path("Season 2018/s2018.e1029 - Jesse's Minecraft Server _ Teaser Trailer.jpg"): "2a24de903059f48c7d0df0476046c975",
Path("Season 2018/s2018.e1029 - Jesse's Minecraft Server _ Teaser Trailer.mp4"): "82f6ee7253e1dbb83ae7215af08ffacc",
Path("Season 2018/s2018.e1029 - Jesse's Minecraft Server _ Teaser Trailer.nfo"): "cc7886aae3af6b7b0facd82f95390242",
Path("pz/Season 2018/s2018.e1102 - Jesse's Minecraft Server _ IP mc.jesse.id.jpg"): "c8baea83b9edeb081657f1130a1031f7",
Path("pz/Season 2018/s2018.e1102 - Jesse's Minecraft Server _ IP mc.jesse.id.mp4"): "e733b4cc385b953b08c8eb0f47e03c1e",
Path("pz/Season 2018/s2018.e1102 - Jesse's Minecraft Server _ IP mc.jesse.id.nfo"): "2b3ccb3f1ef81ee49fe1afb88f275a09",
Path("Season 2018/s2018.e1102 - Jesse's Minecraft Server _ IP mc.jesse.id.jpg"): "c8baea83b9edeb081657f1130a1031f7",
Path("Season 2018/s2018.e1102 - Jesse's Minecraft Server _ IP mc.jesse.id.mp4"): "e733b4cc385b953b08c8eb0f47e03c1e",
Path("Season 2018/s2018.e1102 - Jesse's Minecraft Server _ IP mc.jesse.id.nfo"): "2b3ccb3f1ef81ee49fe1afb88f275a09",
}
)
# fmt: on
@ -164,21 +164,21 @@ def expected_recent_channel_download():
return ExpectedDownload(
expected_md5_file_hashes={
# Download mapping
Path("pz/.ytdl-sub-pz-download-archive.json"): "b1675ca4d9f0d4b9c2102b6749e4cdfd",
Path(".ytdl-sub-pz-download-archive.json"): "b1675ca4d9f0d4b9c2102b6749e4cdfd",
# Output directory files
Path("pz/fanart.jpg"): "e6e323373c8902568e96e374817179cf",
Path("pz/poster.jpg"): "a14c593bcc75bb8d2c7145de4767ad01",
Path("pz/tvshow.nfo"): "83c7db96081ac5bdf289fcf396bec157",
Path("fanart.jpg"): "e6e323373c8902568e96e374817179cf",
Path("poster.jpg"): "a14c593bcc75bb8d2c7145de4767ad01",
Path("tvshow.nfo"): "83c7db96081ac5bdf289fcf396bec157",
# Recent Entry files
Path("pz/Season 2018/s2018.e1029 - Jesse's Minecraft Server _ Teaser Trailer.jpg"): "2a24de903059f48c7d0df0476046c975",
Path("pz/Season 2018/s2018.e1029 - Jesse's Minecraft Server _ Teaser Trailer.mp4"): "82f6ee7253e1dbb83ae7215af08ffacc",
Path("pz/Season 2018/s2018.e1029 - Jesse's Minecraft Server _ Teaser Trailer.nfo"): "cc7886aae3af6b7b0facd82f95390242",
Path("Season 2018/s2018.e1029 - Jesse's Minecraft Server _ Teaser Trailer.jpg"): "2a24de903059f48c7d0df0476046c975",
Path("Season 2018/s2018.e1029 - Jesse's Minecraft Server _ Teaser Trailer.mp4"): "82f6ee7253e1dbb83ae7215af08ffacc",
Path("Season 2018/s2018.e1029 - Jesse's Minecraft Server _ Teaser Trailer.nfo"): "cc7886aae3af6b7b0facd82f95390242",
Path("pz/Season 2018/s2018.e1102 - Jesse's Minecraft Server _ IP mc.jesse.id.jpg"): "c8baea83b9edeb081657f1130a1031f7",
Path("pz/Season 2018/s2018.e1102 - Jesse's Minecraft Server _ IP mc.jesse.id.mp4"): "e733b4cc385b953b08c8eb0f47e03c1e",
Path("pz/Season 2018/s2018.e1102 - Jesse's Minecraft Server _ IP mc.jesse.id.nfo"): "2b3ccb3f1ef81ee49fe1afb88f275a09",
Path("Season 2018/s2018.e1102 - Jesse's Minecraft Server _ IP mc.jesse.id.jpg"): "c8baea83b9edeb081657f1130a1031f7",
Path("Season 2018/s2018.e1102 - Jesse's Minecraft Server _ IP mc.jesse.id.mp4"): "e733b4cc385b953b08c8eb0f47e03c1e",
Path("Season 2018/s2018.e1102 - Jesse's Minecraft Server _ IP mc.jesse.id.nfo"): "2b3ccb3f1ef81ee49fe1afb88f275a09",
}
)
# fmt: on
@ -222,12 +222,12 @@ def expected_recent_channel_no_vids_in_range_download():
return ExpectedDownload(
expected_md5_file_hashes={
# Download mapping
Path("pz/.ytdl-sub-pz-download-archive.json"): "99914b932bd37a50b983c5e7c90ae93b",
Path(".ytdl-sub-pz-download-archive.json"): "99914b932bd37a50b983c5e7c90ae93b",
# Output directory files
Path("pz/fanart.jpg"): "e6e323373c8902568e96e374817179cf",
Path("pz/poster.jpg"): "a14c593bcc75bb8d2c7145de4767ad01",
Path("pz/tvshow.nfo"): "83c7db96081ac5bdf289fcf396bec157",
Path("fanart.jpg"): "e6e323373c8902568e96e374817179cf",
Path("poster.jpg"): "a14c593bcc75bb8d2c7145de4767ad01",
Path("tvshow.nfo"): "83c7db96081ac5bdf289fcf396bec157",
}
)
# fmt: on
@ -269,17 +269,17 @@ def expected_rolling_recent_channel_download():
return ExpectedDownload(
expected_md5_file_hashes={
# Download mapping
Path("pz/.ytdl-sub-pz-download-archive.json"): "9ae3463bd2dc39830003aba68a276df4",
Path(".ytdl-sub-pz-download-archive.json"): "9ae3463bd2dc39830003aba68a276df4",
# Output directory files
Path("pz/fanart.jpg"): "e6e323373c8902568e96e374817179cf",
Path("pz/poster.jpg"): "a14c593bcc75bb8d2c7145de4767ad01",
Path("pz/tvshow.nfo"): "83c7db96081ac5bdf289fcf396bec157",
Path("fanart.jpg"): "e6e323373c8902568e96e374817179cf",
Path("poster.jpg"): "a14c593bcc75bb8d2c7145de4767ad01",
Path("tvshow.nfo"): "83c7db96081ac5bdf289fcf396bec157",
# Rolling Recent Entry files
Path("pz/Season 2018/s2018.e1102 - Jesse's Minecraft Server _ IP mc.jesse.id.jpg"): "c8baea83b9edeb081657f1130a1031f7",
Path("pz/Season 2018/s2018.e1102 - Jesse's Minecraft Server _ IP mc.jesse.id.mp4"): "e733b4cc385b953b08c8eb0f47e03c1e",
Path("pz/Season 2018/s2018.e1102 - Jesse's Minecraft Server _ IP mc.jesse.id.nfo"): "2b3ccb3f1ef81ee49fe1afb88f275a09",
Path("Season 2018/s2018.e1102 - Jesse's Minecraft Server _ IP mc.jesse.id.jpg"): "c8baea83b9edeb081657f1130a1031f7",
Path("Season 2018/s2018.e1102 - Jesse's Minecraft Server _ IP mc.jesse.id.mp4"): "e733b4cc385b953b08c8eb0f47e03c1e",
Path("Season 2018/s2018.e1102 - Jesse's Minecraft Server _ IP mc.jesse.id.nfo"): "2b3ccb3f1ef81ee49fe1afb88f275a09",
}
)
# fmt: on
@ -297,6 +297,12 @@ class TestChannelAsKodiTvShow:
full_channel_subscription.download()
expected_full_channel_download.assert_files_exist(relative_directory=output_directory)
def test_full_channel_dry_run(
self, full_channel_subscription, expected_full_channel_download, output_directory
):
transaction_log = full_channel_subscription.download(dry_run=True)
expected_full_channel_download.assert_dry_run_files_logged(transaction_log=transaction_log)
def test_recent_channel_download(
self, recent_channel_subscription, expected_recent_channel_download, output_directory
):
@ -311,6 +317,14 @@ class TestChannelAsKodiTvShow:
recent_channel_subscription.download()
expected_recent_channel_download.assert_files_exist(relative_directory=output_directory)
def test_recent_channel_dry_run(
self, recent_channel_subscription, expected_recent_channel_download, output_directory
):
transaction_log = recent_channel_subscription.download(dry_run=True)
expected_recent_channel_download.assert_dry_run_files_logged(
transaction_log=transaction_log
)
def test_recent_channel_download__no_vids_in_range(
self,
recent_channel_no_vids_in_range_subscription,
@ -328,6 +342,17 @@ class TestChannelAsKodiTvShow:
relative_directory=output_directory
)
def test_recent_channel_dry_run__no_vids_in_range(
self,
recent_channel_no_vids_in_range_subscription,
expected_recent_channel_no_vids_in_range_download,
output_directory,
):
transaction_log = recent_channel_no_vids_in_range_subscription.download(dry_run=True)
expected_recent_channel_no_vids_in_range_download.assert_dry_run_files_logged(
transaction_log=transaction_log
)
def test_rolling_recent_channel_download(
self,
recent_channel_subscription,

View file

@ -81,8 +81,14 @@ class TestYoutubeMergePlaylist:
files exist and have the expected md5 file hashes.
"""
def test_playlist_download(
def test_merge_playlist_download(
self, playlist_subscription, expected_playlist_download, output_directory
):
playlist_subscription.download()
expected_playlist_download.assert_files_exist(relative_directory=output_directory)
def test_merge_playlist_dry_run(
self, playlist_subscription, expected_playlist_download, output_directory
):
transaction_log = playlist_subscription.download(dry_run=True)
expected_playlist_download.assert_dry_run_files_logged(transaction_log=transaction_log)

View file

@ -148,8 +148,22 @@ class TestPlaylistAsKodiMusicVideo:
playlist_subscription.download()
expected_playlist_download.assert_files_exist(relative_directory=output_directory)
def test_playlist_dry_run(
self, playlist_subscription, expected_playlist_download, output_directory
):
file_transaction_log = playlist_subscription.download(dry_run=True)
expected_playlist_download.assert_dry_run_files_logged(transaction_log=file_transaction_log)
def test_single_video_download(
self, single_video_subscription, expected_single_video_download, output_directory
):
single_video_subscription.download()
expected_single_video_download.assert_files_exist(relative_directory=output_directory)
def test_single_video_dry_run(
self, single_video_subscription, expected_single_video_download, output_directory
):
file_transaction_log = single_video_subscription.download(dry_run=True)
expected_single_video_download.assert_dry_run_files_logged(
transaction_log=file_transaction_log
)

View file

@ -119,8 +119,12 @@ class TestPlaylistAsKodiMusicVideo:
files exist and have the expected md5 file hashes.
"""
def test_single_video_download(
def test_split_video_download(
self, single_video_subscription, expected_single_video_download, output_directory
):
single_video_subscription.download()
expected_single_video_download.assert_files_exist(relative_directory=output_directory)
def test_split_video_dry_run(self, single_video_subscription, expected_single_video_download):
transaction_log = single_video_subscription.download(dry_run=True)
expected_single_video_download.assert_dry_run_files_logged(transaction_log=transaction_log)