Add --dry-run argument logic (#82)
* dry-run huge refactor * small fixes, all tests passing * file handler, transaction log, metadata * dry-run actually dry running * working tests, still have more * Everything passing besides channel * fix youtube channel thumbnail issue, need to fix fixtures on test * lint 10, channel tests passing. Need dry-run for split and merge video * all tests passing, lint 10, should test dry-run arg
This commit is contained in:
parent
e5b925fb57
commit
9df40497e2
22 changed files with 701 additions and 261 deletions
|
|
@ -43,9 +43,9 @@ presets:
|
|||
# store previously downloaded song IDs to tell YTDL not to re-download
|
||||
# them on a successive invocation.
|
||||
output_options:
|
||||
output_directory: "/{music_directory}/{artist_sanitized}"
|
||||
file_name: "{album_directory_name}/{track_number_padded} - {title_sanitized}.{ext}"
|
||||
thumbnail_name: "{album_directory_name}/folder.jpg"
|
||||
output_directory: "{music_directory}"
|
||||
file_name: "{artist_sanitized}/{album_directory_name}/{track_number_padded} - {title_sanitized}.{ext}"
|
||||
thumbnail_name: "{artist_sanitized}/{album_directory_name}/folder.jpg"
|
||||
maintain_download_archive: True
|
||||
|
||||
# For each song downloaded, populate the audio file with music tags.
|
||||
|
|
|
|||
|
|
@ -1,9 +1,9 @@
|
|||
import argparse
|
||||
|
||||
from ytdl_sub.utils.logger import LoggerLevels
|
||||
|
||||
###################################################################################################
|
||||
# GLOBAL PARSER
|
||||
from ytdl_sub.utils.logger import LoggerLevels
|
||||
|
||||
parser = argparse.ArgumentParser(
|
||||
description="ytdl-sub: Automate download and adding metadata with YoutubeDL"
|
||||
)
|
||||
|
|
@ -15,6 +15,11 @@ parser.add_argument(
|
|||
help="path to the config yaml, uses config.yaml if not provided",
|
||||
default="config.yaml",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--dry-run",
|
||||
action="store_true",
|
||||
help="does not perform any video downloads or writes to output directories",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--log-level",
|
||||
metavar="|".join(LoggerLevels.names()),
|
||||
|
|
|
|||
|
|
@ -20,6 +20,8 @@ from ytdl_sub.entries.base_entry import BaseEntry
|
|||
from ytdl_sub.entries.entry import Entry
|
||||
from ytdl_sub.utils.logger import Logger
|
||||
from ytdl_sub.validators.strict_dict_validator import StrictDictValidator
|
||||
from ytdl_sub.ytdl_additions.enhanced_download_archive import DownloadArchiver
|
||||
from ytdl_sub.ytdl_additions.enhanced_download_archive import EnhancedDownloadArchive
|
||||
|
||||
logger = Logger.get(name="downloader")
|
||||
|
||||
|
|
@ -35,7 +37,7 @@ DownloaderEntryT = TypeVar("DownloaderEntryT", bound=Entry)
|
|||
DownloaderParentEntryT = TypeVar("DownloaderParentEntryT", bound=BaseEntry)
|
||||
|
||||
|
||||
class Downloader(Generic[DownloaderOptionsT, DownloaderEntryT], ABC):
|
||||
class Downloader(DownloadArchiver, Generic[DownloaderOptionsT, DownloaderEntryT], ABC):
|
||||
"""
|
||||
Class that interacts with ytdl to perform the download of metadata and content,
|
||||
and should translate that to list of Entry objects.
|
||||
|
|
@ -66,7 +68,6 @@ class Downloader(Generic[DownloaderOptionsT, DownloaderEntryT], ABC):
|
|||
cls,
|
||||
working_directory: str,
|
||||
ytdl_options: Optional[Dict],
|
||||
download_archive_file_name: Optional[str],
|
||||
) -> Dict:
|
||||
"""Configure the ytdl options for the downloader"""
|
||||
if ytdl_options is None:
|
||||
|
|
@ -81,39 +82,29 @@ class Downloader(Generic[DownloaderOptionsT, DownloaderEntryT], ABC):
|
|||
# Overwrite the output location with the specified working directory
|
||||
ytdl_options["outtmpl"] = str(Path(working_directory) / "%(id)s.%(ext)s")
|
||||
|
||||
# If a download archive file name is provided, set it to that
|
||||
if download_archive_file_name:
|
||||
ytdl_options["download_archive"] = str(
|
||||
Path(working_directory) / download_archive_file_name
|
||||
)
|
||||
|
||||
return ytdl_options
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
working_directory: str,
|
||||
download_options: DownloaderOptionsT,
|
||||
enhanced_download_archive: EnhancedDownloadArchive,
|
||||
ytdl_options: Optional[Dict] = None,
|
||||
download_archive_file_name: Optional[str] = None,
|
||||
):
|
||||
"""
|
||||
Parameters
|
||||
----------
|
||||
working_directory
|
||||
Path to the working directory
|
||||
download_options
|
||||
Options validator for this downloader
|
||||
enhanced_download_archive
|
||||
Download archive
|
||||
ytdl_options
|
||||
YTDL options validator
|
||||
download_archive_file_name
|
||||
Optional. Name of the download archive file that should reside in the working directory
|
||||
"""
|
||||
self.working_directory = working_directory
|
||||
DownloadArchiver.__init__(self=self, enhanced_download_archive=enhanced_download_archive)
|
||||
self.download_options = download_options
|
||||
self.ytdl_options = self._configure_ytdl_options(
|
||||
ytdl_options=ytdl_options,
|
||||
working_directory=self.working_directory,
|
||||
download_archive_file_name=download_archive_file_name,
|
||||
)
|
||||
|
||||
@contextmanager
|
||||
|
|
@ -129,6 +120,15 @@ class Downloader(Generic[DownloaderOptionsT, DownloaderEntryT], ABC):
|
|||
with ytdl.YoutubeDL(ytdl_options) as ytdl_downloader:
|
||||
yield ytdl_downloader
|
||||
|
||||
@property
|
||||
def is_dry_run(self) -> bool:
|
||||
"""
|
||||
Returns
|
||||
-------
|
||||
True if dry-run is enabled. False otherwise.
|
||||
"""
|
||||
return self.ytdl_options.get("skip_download", False)
|
||||
|
||||
def extract_info(self, ytdl_options_overrides: Optional[Dict] = None, **kwargs) -> Dict:
|
||||
"""
|
||||
Wrapper around yt_dlp.YoutubeDL.YoutubeDL.extract_info
|
||||
|
|
@ -200,19 +200,14 @@ class Downloader(Generic[DownloaderOptionsT, DownloaderEntryT], ABC):
|
|||
def download(self) -> List[DownloaderEntryT]:
|
||||
"""The function to perform the download of all media entries"""
|
||||
|
||||
def post_download(self, overrides: Overrides, output_directory: str):
|
||||
def post_download(self, overrides: Overrides):
|
||||
"""
|
||||
After all media entries have been downloaded, post processed, and moved to the output
|
||||
directory, run this function. This lets the downloader add any extra files directly to the
|
||||
output directory, for things like YT channel image, banner.
|
||||
|
||||
This ideally should not perform any extra downloads, but rather, use the content already
|
||||
downloaded in the working directory and use it in the output directory.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
overrides:
|
||||
Subscription overrides
|
||||
output_directory:
|
||||
Output directory to potentially store extra files downloaded
|
||||
"""
|
||||
|
|
|
|||
|
|
@ -100,8 +100,7 @@ class YoutubeMergePlaylistDownloader(
|
|||
},
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def _add_chapters(cls, merged_video: YoutubeVideo) -> None:
|
||||
def _add_chapters(self, merged_video: YoutubeVideo) -> None:
|
||||
titles: List[str] = []
|
||||
timestamps: List[Timestamp] = []
|
||||
|
||||
|
|
@ -112,11 +111,13 @@ class YoutubeMergePlaylistDownloader(
|
|||
|
||||
current_timestamp_sec += video_entry["duration"]
|
||||
|
||||
add_ffmpeg_metadata(
|
||||
file_path=merged_video.get_download_file_path(),
|
||||
chapters=Chapters(timestamps=timestamps, titles=titles),
|
||||
file_duration_sec=merged_video.kwargs("duration"),
|
||||
)
|
||||
# TODO: return chapter metadata here
|
||||
if not self.is_dry_run:
|
||||
add_ffmpeg_metadata(
|
||||
file_path=merged_video.get_download_file_path(),
|
||||
chapters=Chapters(timestamps=timestamps, titles=titles),
|
||||
file_duration_sec=merged_video.kwargs("duration"),
|
||||
)
|
||||
|
||||
def _to_merged_video(self, entry_dict: Dict) -> YoutubeVideo:
|
||||
"""
|
||||
|
|
@ -130,7 +131,11 @@ class YoutubeMergePlaylistDownloader(
|
|||
entry_dict["duration"] = sum(
|
||||
playlist_entry["duration"] for playlist_entry in entry_dict["entries"]
|
||||
)
|
||||
entry_dict["ext"] = entry_dict["requested_downloads"][0]["ext"]
|
||||
entry_dict["ext"] = (
|
||||
entry_dict["requested_downloads"][0]["ext"]
|
||||
if "requested_downloads" in entry_dict
|
||||
else "mkv"
|
||||
)
|
||||
return YoutubeVideo(entry_dict=entry_dict, working_directory=self.working_directory)
|
||||
|
||||
def download(self) -> List[YoutubeVideo]:
|
||||
|
|
|
|||
|
|
@ -1,7 +1,6 @@
|
|||
import copy
|
||||
import re
|
||||
from pathlib import Path
|
||||
from shutil import copyfile
|
||||
from typing import Dict
|
||||
from typing import List
|
||||
|
||||
|
|
@ -12,8 +11,6 @@ from ytdl_sub.entries.youtube import YoutubeVideo
|
|||
from ytdl_sub.utils.chapters import Chapters
|
||||
from ytdl_sub.utils.chapters import Timestamp
|
||||
from ytdl_sub.utils.ffmpeg import FFMPEG
|
||||
from ytdl_sub.utils.thumbnail import convert_download_thumbnail
|
||||
from ytdl_sub.validators.validators import StringValidator
|
||||
|
||||
# Captures the following formats:
|
||||
# 0:00 title
|
||||
|
|
@ -21,6 +18,10 @@ from ytdl_sub.validators.validators import StringValidator
|
|||
# 1:00:00 title
|
||||
# 01:00:00 title
|
||||
# where capture group 1 and 2 are the timestamp and title, respectively
|
||||
from ytdl_sub.utils.file_handler import FileHandler
|
||||
from ytdl_sub.utils.thumbnail import convert_download_thumbnail
|
||||
from ytdl_sub.validators.validators import StringValidator
|
||||
|
||||
_SPLIT_TIMESTAMP_REGEX = re.compile(r"^((?:\d\d:)?(?:\d:)?(?:\d)?\d:\d\d) (.+)$")
|
||||
|
||||
|
||||
|
|
@ -154,24 +155,27 @@ class YoutubeSplitVideoDownloader(
|
|||
for idx, title in enumerate(chapters.titles):
|
||||
new_uid = _split_video_uid(source_uid=entry.uid, idx=idx)
|
||||
|
||||
# Get the input/output file paths
|
||||
input_file = entry.get_download_file_path()
|
||||
output_file = str(Path(self.working_directory) / f"{new_uid}.{entry.ext}")
|
||||
output_thumbnail_file = str(
|
||||
Path(self.working_directory) / f"{new_uid}.{entry.thumbnail_ext}"
|
||||
)
|
||||
if not self.is_dry_run:
|
||||
# Get the input/output file paths
|
||||
input_file = entry.get_download_file_path()
|
||||
output_file = str(Path(self.working_directory) / f"{new_uid}.{entry.ext}")
|
||||
|
||||
# Run ffmpeg to create the split the video
|
||||
FFMPEG.run(
|
||||
_split_video_ffmpeg_cmd(
|
||||
input_file=input_file,
|
||||
output_file=output_file,
|
||||
timestamps=chapters.timestamps,
|
||||
idx=idx,
|
||||
# Run ffmpeg to create the split the video
|
||||
FFMPEG.run(
|
||||
_split_video_ffmpeg_cmd(
|
||||
input_file=input_file,
|
||||
output_file=output_file,
|
||||
timestamps=chapters.timestamps,
|
||||
idx=idx,
|
||||
)
|
||||
)
|
||||
|
||||
# Copy the original vid thumbnail to the working directory with the new uid. This so
|
||||
# downstream logic thinks this split video has its own thumbnail
|
||||
FileHandler.copy(
|
||||
src_file_path=entry.get_download_thumbnail_path(),
|
||||
dst_file_path=Path(self.working_directory) / f"{new_uid}.{entry.thumbnail_ext}",
|
||||
)
|
||||
# Copy the thumbnail
|
||||
copyfile(src=entry.get_download_thumbnail_path(), dst=output_thumbnail_file)
|
||||
|
||||
# Format the split video as a YoutubePlaylistVideo
|
||||
split_videos.append(
|
||||
|
|
|
|||
|
|
@ -20,6 +20,7 @@ from ytdl_sub.validators.string_formatter_validators import OverridesStringForma
|
|||
from ytdl_sub.validators.url_validator import YoutubeChannelUrlValidator
|
||||
from ytdl_sub.validators.url_validator import YoutubePlaylistUrlValidator
|
||||
from ytdl_sub.validators.url_validator import YoutubeVideoUrlValidator
|
||||
from ytdl_sub.ytdl_additions.enhanced_download_archive import EnhancedDownloadArchive
|
||||
|
||||
logger = Logger.get()
|
||||
|
||||
|
|
@ -290,16 +291,14 @@ class YoutubeChannelDownloader(YoutubeDownloader[YoutubeChannelDownloaderOptions
|
|||
|
||||
def __init__(
|
||||
self,
|
||||
working_directory: str,
|
||||
download_options: DownloaderOptionsT,
|
||||
enhanced_download_archive: EnhancedDownloadArchive,
|
||||
ytdl_options: Optional[Dict] = None,
|
||||
download_archive_file_name: Optional[str] = None,
|
||||
):
|
||||
super().__init__(
|
||||
working_directory=working_directory,
|
||||
download_options=download_options,
|
||||
enhanced_download_archive=enhanced_download_archive,
|
||||
ytdl_options=ytdl_options,
|
||||
download_archive_file_name=download_archive_file_name,
|
||||
)
|
||||
self.channel: Optional[YoutubeChannel] = None
|
||||
|
||||
|
|
@ -354,7 +353,7 @@ class YoutubeChannelDownloader(YoutubeDownloader[YoutubeChannelDownloaderOptions
|
|||
thumbnail_url=thumbnail_url, output_thumbnail_path=output_thumbnail_path
|
||||
)
|
||||
|
||||
def post_download(self, overrides: Overrides, output_directory: str):
|
||||
def post_download(self, overrides: Overrides):
|
||||
"""
|
||||
Downloads and moves channel avatar and banner images to the output directory.
|
||||
|
||||
|
|
@ -362,17 +361,17 @@ class YoutubeChannelDownloader(YoutubeDownloader[YoutubeChannelDownloaderOptions
|
|||
----------
|
||||
overrides
|
||||
Overrides that can contain variables in the avatar or banner file path
|
||||
output_directory
|
||||
Output directory path
|
||||
"""
|
||||
avatar_thumbnail_name = overrides.apply_formatter(self.download_options.channel_avatar_path)
|
||||
self._download_thumbnail(
|
||||
thumbnail_url=self.channel.avatar_thumbnail_url(),
|
||||
output_thumbnail_path=str(Path(output_directory) / avatar_thumbnail_name),
|
||||
output_thumbnail_path=str(Path(self.working_directory) / avatar_thumbnail_name),
|
||||
)
|
||||
self.save_file(file_name=avatar_thumbnail_name)
|
||||
|
||||
banner_thumbnail_name = overrides.apply_formatter(self.download_options.channel_banner_path)
|
||||
self._download_thumbnail(
|
||||
thumbnail_url=self.channel.banner_thumbnail_url(),
|
||||
output_thumbnail_path=str(Path(output_directory) / banner_thumbnail_name),
|
||||
output_thumbnail_path=str(Path(self.working_directory) / banner_thumbnail_name),
|
||||
)
|
||||
self.save_file(file_name=banner_thumbnail_name)
|
||||
|
|
|
|||
|
|
@ -11,13 +11,29 @@ class Entry(EntryVariables, BaseEntry):
|
|||
Entry object to represent a single media object returned from yt-dlp.
|
||||
"""
|
||||
|
||||
def get_download_file_name(self) -> str:
|
||||
"""
|
||||
Returns
|
||||
-------
|
||||
The entry's file name
|
||||
"""
|
||||
return f"{self.uid}.{self.ext}"
|
||||
|
||||
def get_download_file_path(self) -> str:
|
||||
"""Returns the entry's file path to where it was downloaded"""
|
||||
return str(Path(self.working_directory()) / f"{self.uid}.{self.ext}")
|
||||
return str(Path(self.working_directory()) / self.get_download_file_name())
|
||||
|
||||
def get_download_thumbnail_name(self) -> str:
|
||||
"""
|
||||
Returns
|
||||
-------
|
||||
The download thumbnail's file name
|
||||
"""
|
||||
return f"{self.uid}.{self.thumbnail_ext}"
|
||||
|
||||
def get_download_thumbnail_path(self) -> str:
|
||||
"""Returns the entry's thumbnail's file path to where it was downloaded"""
|
||||
return str(Path(self.working_directory()) / f"{self.uid}.{self.thumbnail_ext}")
|
||||
return str(Path(self.working_directory()) / self.get_download_thumbnail_name())
|
||||
|
||||
@final
|
||||
def to_dict(self) -> Dict[str, str]:
|
||||
|
|
|
|||
|
|
@ -17,8 +17,12 @@ def _download_subscriptions_from_yaml_files(config: ConfigFile, args: argparse.N
|
|||
"""
|
||||
Downloads all subscriptions from one or many subscription yaml files.
|
||||
|
||||
:param config: Configuration file
|
||||
:param args: Arguments from argparse
|
||||
Parameters
|
||||
----------
|
||||
config
|
||||
Configuration file
|
||||
args
|
||||
Arguments from argparse
|
||||
"""
|
||||
preset_paths: List[str] = args.subscription_paths
|
||||
presets: List[Preset] = []
|
||||
|
|
@ -30,15 +34,23 @@ def _download_subscriptions_from_yaml_files(config: ConfigFile, args: argparse.N
|
|||
subscription = Subscription.from_preset(preset=preset, config=config)
|
||||
|
||||
logger.info("Beginning subscription download for %s", subscription.name)
|
||||
subscription.download()
|
||||
subscription.download(dry_run=args.dry_run)
|
||||
|
||||
|
||||
def _download_subscription_from_cli(config: ConfigFile, extra_args: List[str]) -> None:
|
||||
def _download_subscription_from_cli(
|
||||
config: ConfigFile, args: argparse.Namespace, extra_args: List[str]
|
||||
) -> None:
|
||||
"""
|
||||
Downloads a one-off subscription using the CLI
|
||||
|
||||
:param config: Configuration file
|
||||
:param extra_args: Extra arguments from argparse that contain dynamic subscription options
|
||||
Parameters
|
||||
----------
|
||||
config
|
||||
Configuration file
|
||||
args
|
||||
Arguments from argparse
|
||||
extra_args
|
||||
Extra arguments from argparse that contain dynamic subscription options
|
||||
"""
|
||||
dl_args_parser = DownloadArgsParser(
|
||||
extra_arguments=extra_args, config_options=config.config_options
|
||||
|
|
@ -57,7 +69,7 @@ def _download_subscription_from_cli(config: ConfigFile, extra_args: List[str]) -
|
|||
config=config,
|
||||
)
|
||||
|
||||
subscription.download()
|
||||
subscription.download(dry_run=args.dry_run)
|
||||
|
||||
|
||||
def _main():
|
||||
|
|
@ -80,7 +92,7 @@ def _main():
|
|||
|
||||
# One-off download
|
||||
if args.subparser == "dl":
|
||||
_download_subscription_from_cli(config=config, extra_args=extra_args)
|
||||
_download_subscription_from_cli(config=config, args=args, extra_args=extra_args)
|
||||
logger.info("Download complete!")
|
||||
|
||||
# Ran successfully, so we can delete the debug file
|
||||
|
|
|
|||
|
|
@ -1,8 +1,11 @@
|
|||
from typing import Dict
|
||||
|
||||
import mediafile
|
||||
|
||||
from ytdl_sub.entries.entry import Entry
|
||||
from ytdl_sub.plugins.plugin import Plugin
|
||||
from ytdl_sub.plugins.plugin import PluginOptions
|
||||
from ytdl_sub.utils.file_handler import FileMetadata
|
||||
from ytdl_sub.validators.string_formatter_validators import DictFormatterValidator
|
||||
from ytdl_sub.validators.validators import StringValidator
|
||||
|
||||
|
|
@ -57,17 +60,28 @@ class MusicTagsPlugin(Plugin[MusicTagsOptions]):
|
|||
"""
|
||||
Tags the entry's audio file using values defined in the metadata options
|
||||
"""
|
||||
audio_file = mediafile.MediaFile(entry.get_download_file_path())
|
||||
for tag, tag_formatter in self.plugin_options.tags.dict.items():
|
||||
if tag not in audio_file.fields():
|
||||
supported_fields = list(mediafile.MediaFile.sorted_fields())
|
||||
tags_to_write: Dict[str, str] = {}
|
||||
for tag_name, tag_formatter in self.plugin_options.tags.dict.items():
|
||||
if tag_name not in supported_fields:
|
||||
# TODO: Add support for custom fields
|
||||
self._logger.warning(
|
||||
"tag '%s' is not supported for %s files. Supported tags: %s",
|
||||
tag,
|
||||
tag_name,
|
||||
entry.ext,
|
||||
", ".join(audio_file.sorted_fields()),
|
||||
", ".join(sorted(supported_fields)),
|
||||
)
|
||||
continue
|
||||
|
||||
tag_value = self.overrides.apply_formatter(formatter=tag_formatter, entry=entry)
|
||||
setattr(audio_file, tag, tag_value)
|
||||
tags_to_write[tag_name] = tag_value
|
||||
|
||||
audio_file.save()
|
||||
# write the actual tags if its not a dry run
|
||||
if not self.is_dry_run:
|
||||
audio_file = mediafile.MediaFile(entry.get_download_file_path())
|
||||
for tag_name, tag_value in tags_to_write.items():
|
||||
setattr(audio_file, tag_name, tag_value)
|
||||
audio_file.save()
|
||||
|
||||
# report the tags written
|
||||
return FileMetadata.from_dict(value_dict=tags_to_write, title="Music Tags:")
|
||||
|
|
|
|||
|
|
@ -6,6 +6,7 @@ import dicttoxml
|
|||
from ytdl_sub.entries.entry import Entry
|
||||
from ytdl_sub.plugins.plugin import Plugin
|
||||
from ytdl_sub.plugins.plugin import PluginOptions
|
||||
from ytdl_sub.utils.file_handler import FileMetadata
|
||||
from ytdl_sub.validators.string_formatter_validators import DictFormatterValidator
|
||||
from ytdl_sub.validators.string_formatter_validators import StringFormatterValidator
|
||||
|
||||
|
|
@ -109,10 +110,12 @@ class NfoTagsPlugin(Plugin[NfoTagsOptions]):
|
|||
)
|
||||
|
||||
# Save the nfo's XML to file
|
||||
nfo_file_path = Path(self.output_directory) / nfo_file_name
|
||||
nfo_file_path = Path(self.working_directory) / nfo_file_name
|
||||
os.makedirs(os.path.dirname(nfo_file_path), exist_ok=True)
|
||||
with open(nfo_file_path, "wb") as nfo_file:
|
||||
nfo_file.write(xml)
|
||||
|
||||
# Archive the nfo's file name
|
||||
self.archive_entry_file_name(entry=entry, relative_file_path=nfo_file_name)
|
||||
self.save_file(file_name=nfo_file_name, entry=entry)
|
||||
|
||||
return FileMetadata.from_dict(value_dict={nfo_root: nfo}, title="NFO tags:")
|
||||
|
|
|
|||
|
|
@ -5,6 +5,7 @@ import dicttoxml
|
|||
|
||||
from ytdl_sub.plugins.plugin import Plugin
|
||||
from ytdl_sub.plugins.plugin import PluginOptions
|
||||
from ytdl_sub.utils.file_handler import FileMetadata
|
||||
from ytdl_sub.validators.string_formatter_validators import OverridesDictFormatterValidator
|
||||
from ytdl_sub.validators.string_formatter_validators import OverridesStringFormatterValidator
|
||||
|
||||
|
|
@ -100,7 +101,10 @@ class OutputDirectoryNfoTagsPlugin(Plugin[OutputDirectoryNfoTagsOptions]):
|
|||
nfo_file_name = self.overrides.apply_formatter(formatter=self.plugin_options.nfo_name)
|
||||
|
||||
# Save the nfo's XML to file
|
||||
nfo_file_path = Path(self.output_directory) / nfo_file_name
|
||||
nfo_file_path = Path(self.working_directory) / nfo_file_name
|
||||
os.makedirs(os.path.dirname(nfo_file_path), exist_ok=True)
|
||||
with open(nfo_file_path, "wb") as nfo_file:
|
||||
nfo_file.write(xml)
|
||||
|
||||
self.save_file(file_name=nfo_file_name)
|
||||
return FileMetadata.from_dict(value_dict={nfo_root: nfo}, title="NFO tags:")
|
||||
|
|
|
|||
|
|
@ -7,8 +7,10 @@ from typing import final
|
|||
|
||||
from ytdl_sub.config.preset_options import Overrides
|
||||
from ytdl_sub.entries.entry import Entry
|
||||
from ytdl_sub.utils.file_handler import FileMetadata
|
||||
from ytdl_sub.utils.logger import Logger
|
||||
from ytdl_sub.validators.strict_dict_validator import StrictDictValidator
|
||||
from ytdl_sub.ytdl_additions.enhanced_download_archive import DownloadArchiver
|
||||
from ytdl_sub.ytdl_additions.enhanced_download_archive import EnhancedDownloadArchive
|
||||
|
||||
|
||||
|
|
@ -21,7 +23,7 @@ class PluginOptions(StrictDictValidator):
|
|||
PluginOptionsT = TypeVar("PluginOptionsT", bound=PluginOptions)
|
||||
|
||||
|
||||
class Plugin(Generic[PluginOptionsT], ABC):
|
||||
class Plugin(DownloadArchiver, Generic[PluginOptionsT], ABC):
|
||||
"""
|
||||
Class to define the new plugin functionality
|
||||
"""
|
||||
|
|
@ -32,36 +34,16 @@ class Plugin(Generic[PluginOptionsT], ABC):
|
|||
def __init__(
|
||||
self,
|
||||
plugin_options: PluginOptionsT,
|
||||
output_directory: str,
|
||||
overrides: Overrides,
|
||||
enhanced_download_archive: Optional[EnhancedDownloadArchive],
|
||||
enhanced_download_archive: EnhancedDownloadArchive,
|
||||
):
|
||||
DownloadArchiver.__init__(self=self, enhanced_download_archive=enhanced_download_archive)
|
||||
self.plugin_options = plugin_options
|
||||
self.output_directory = output_directory
|
||||
self.overrides = overrides
|
||||
self.__enhanced_download_archive = enhanced_download_archive
|
||||
# TODO pass yaml snake case name in the class somewhere, and use it for the logger
|
||||
self._logger = Logger.get(self.__class__.__name__)
|
||||
|
||||
@final
|
||||
def archive_entry_file_name(self, entry: Entry, relative_file_path: str) -> None:
|
||||
"""
|
||||
Adds an entry and a file name that belongs to it into the archive mapping.
|
||||
If maintain_download_archive is False for the subscription, this method will do nothing.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
entry:
|
||||
Optional. The entry the file belongs to
|
||||
relative_file_path:
|
||||
The name of the file path relative to the output directory
|
||||
"""
|
||||
if self.__enhanced_download_archive:
|
||||
self.__enhanced_download_archive.mapping.add_entry(
|
||||
entry=entry, entry_file_path=relative_file_path
|
||||
)
|
||||
|
||||
def post_process_entry(self, entry: Entry):
|
||||
def post_process_entry(self, entry: Entry) -> Optional[FileMetadata]:
|
||||
"""
|
||||
For each file downloaded, apply post processing to it.
|
||||
|
||||
|
|
@ -69,6 +51,10 @@ class Plugin(Generic[PluginOptionsT], ABC):
|
|||
----------
|
||||
entry:
|
||||
Entry to post process
|
||||
|
||||
Returns
|
||||
-------
|
||||
Optional file metadata for the entry media file.
|
||||
"""
|
||||
|
||||
def post_process_subscription(self):
|
||||
|
|
|
|||
|
|
@ -1,9 +1,10 @@
|
|||
import contextlib
|
||||
import copy
|
||||
import os
|
||||
import shutil
|
||||
from pathlib import Path
|
||||
from shutil import copyfile
|
||||
from typing import List
|
||||
from typing import Optional
|
||||
from typing import Tuple
|
||||
from typing import Type
|
||||
|
||||
|
|
@ -18,6 +19,8 @@ from ytdl_sub.downloaders.downloader import DownloaderValidator
|
|||
from ytdl_sub.entries.entry import Entry
|
||||
from ytdl_sub.plugins.plugin import Plugin
|
||||
from ytdl_sub.plugins.plugin import PluginOptions
|
||||
from ytdl_sub.utils.file_handler import FileHandlerTransactionLog
|
||||
from ytdl_sub.utils.file_handler import FileMetadata
|
||||
from ytdl_sub.utils.thumbnail import convert_download_thumbnail
|
||||
from ytdl_sub.ytdl_additions.enhanced_download_archive import EnhancedDownloadArchive
|
||||
|
||||
|
|
@ -144,30 +147,9 @@ class Subscription:
|
|||
and self.downloader_class.supports_download_archive
|
||||
)
|
||||
|
||||
def _copy_file_to_output_directory(
|
||||
self, entry: Entry, source_file_path: str, output_file_name: str
|
||||
def _copy_entry_files_to_output_directory(
|
||||
self, entry: Entry, entry_metadata: Optional[FileMetadata] = None
|
||||
):
|
||||
"""
|
||||
Helper function to move a file from the working directory to the output directory.
|
||||
Will add it to the download archive mapping if the archive is being maintained.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
entry:
|
||||
Entry that the file belongs to
|
||||
source_file_path:
|
||||
Path to the source file
|
||||
output_file_name:
|
||||
Desired output file name with the output directory as its relative directory
|
||||
"""
|
||||
destination_file_path = Path(self.output_directory) / Path(output_file_name)
|
||||
os.makedirs(os.path.dirname(destination_file_path), exist_ok=True)
|
||||
copyfile(source_file_path, destination_file_path)
|
||||
|
||||
if self.maintain_download_archive:
|
||||
self._enhanced_download_archive.mapping.add_entry(entry, output_file_name)
|
||||
|
||||
def _copy_entry_files_to_output_directory(self, entry: Entry):
|
||||
"""
|
||||
Helper function to move the media file and optionally thumbnail file to the output directory
|
||||
for a single entry.
|
||||
|
|
@ -176,15 +158,18 @@ class Subscription:
|
|||
----------
|
||||
entry:
|
||||
The entry with files to move
|
||||
entry_metadata
|
||||
Optional. Metadata to record to the transaction log for this entry
|
||||
"""
|
||||
# Move the file after all direct file modifications are complete
|
||||
output_file_name = self.overrides.apply_formatter(
|
||||
formatter=self.output_options.file_name, entry=entry
|
||||
)
|
||||
self._copy_file_to_output_directory(
|
||||
entry=entry,
|
||||
source_file_path=entry.get_download_file_path(),
|
||||
self._enhanced_download_archive.save_file_to_output_directory(
|
||||
file_name=entry.get_download_file_name(),
|
||||
file_metadata=entry_metadata,
|
||||
output_file_name=output_file_name,
|
||||
entry=entry,
|
||||
)
|
||||
|
||||
if self.output_options.thumbnail_name:
|
||||
|
|
@ -195,10 +180,10 @@ class Subscription:
|
|||
# We always convert entry thumbnails to jpgs, and is performed here
|
||||
convert_download_thumbnail(entry=entry)
|
||||
|
||||
self._copy_file_to_output_directory(
|
||||
entry=entry,
|
||||
source_file_path=entry.get_download_thumbnail_path(),
|
||||
self._enhanced_download_archive.save_file_to_output_directory(
|
||||
file_name=entry.get_download_thumbnail_name(),
|
||||
output_file_name=output_thumbnail_name,
|
||||
entry=entry,
|
||||
)
|
||||
|
||||
@contextlib.contextmanager
|
||||
|
|
@ -243,45 +228,59 @@ class Subscription:
|
|||
for plugin_type, plugin_options in self.plugins:
|
||||
plugin = plugin_type(
|
||||
plugin_options=plugin_options,
|
||||
output_directory=self.output_directory,
|
||||
overrides=self.overrides,
|
||||
enhanced_download_archive=self._enhanced_download_archive
|
||||
if self.maintain_download_archive
|
||||
else None,
|
||||
enhanced_download_archive=self._enhanced_download_archive,
|
||||
)
|
||||
|
||||
plugins.append(plugin)
|
||||
|
||||
return plugins
|
||||
|
||||
def download(self):
|
||||
def download(self, dry_run: bool = False) -> FileHandlerTransactionLog:
|
||||
"""
|
||||
Performs the subscription download.
|
||||
Performs the subscription download
|
||||
|
||||
Parameters
|
||||
----------
|
||||
dry_run
|
||||
If true, do not download any video/audio files or move anything to the output
|
||||
directory.
|
||||
"""
|
||||
self._enhanced_download_archive.reinitialize(dry_run=dry_run)
|
||||
|
||||
# TODO: Move this logic to separate function
|
||||
# TODO: set id here as well
|
||||
ytdl_options = copy.deepcopy(self.ytdl_options.dict)
|
||||
if dry_run:
|
||||
ytdl_options["skip_download"] = True
|
||||
if self.downloader_class.supports_download_archive and self.maintain_download_archive:
|
||||
ytdl_options["download_archive"] = str(
|
||||
Path(self.working_directory) / self._enhanced_download_archive.archive_file_name
|
||||
)
|
||||
|
||||
plugins = self._initialize_plugins()
|
||||
with self._prepare_working_directory(), self._maintain_archive_file():
|
||||
downloader = self.downloader_class(
|
||||
working_directory=self.working_directory,
|
||||
download_options=self.downloader_options,
|
||||
ytdl_options=self.ytdl_options.dict,
|
||||
download_archive_file_name=self._enhanced_download_archive.archive_file_name
|
||||
if self.maintain_download_archive
|
||||
else None,
|
||||
enhanced_download_archive=self._enhanced_download_archive,
|
||||
ytdl_options=ytdl_options,
|
||||
)
|
||||
|
||||
entries = downloader.download()
|
||||
for plugin in plugins:
|
||||
for entry in entries:
|
||||
plugin.post_process_entry(entry)
|
||||
for entry in downloader.download():
|
||||
# TODO: Add entry metadata from the downloader.download function
|
||||
entry_metadata = FileMetadata()
|
||||
for plugin in plugins:
|
||||
entry_metadata.extend(plugin.post_process_entry(entry))
|
||||
|
||||
self._copy_entry_files_to_output_directory(
|
||||
entry=entry, entry_metadata=entry_metadata
|
||||
)
|
||||
|
||||
downloader.post_download(overrides=self.overrides)
|
||||
for plugin in plugins:
|
||||
plugin.post_process_subscription()
|
||||
|
||||
for entry in entries:
|
||||
self._copy_entry_files_to_output_directory(entry=entry)
|
||||
|
||||
downloader.post_download(
|
||||
overrides=self.overrides, output_directory=self.output_directory
|
||||
)
|
||||
return self._enhanced_download_archive.get_file_handler_transaction_log()
|
||||
|
||||
@classmethod
|
||||
def from_preset(cls, preset: Preset, config: ConfigFile) -> "Subscription":
|
||||
|
|
|
|||
195
src/ytdl_sub/utils/file_handler.py
Normal file
195
src/ytdl_sub/utils/file_handler.py
Normal file
|
|
@ -0,0 +1,195 @@
|
|||
import os
|
||||
from pathlib import Path
|
||||
from shutil import copyfile
|
||||
from typing import Any
|
||||
from typing import Dict
|
||||
from typing import List
|
||||
from typing import Optional
|
||||
from typing import Set
|
||||
from typing import Union
|
||||
|
||||
|
||||
class FileMetadata:
|
||||
"""
|
||||
Stores pretty-printed information about a file. Each line in the metadata represents a newline
|
||||
"""
|
||||
|
||||
def __init__(self, metadata: Optional[List[str]] = None):
|
||||
self.metadata: List[str] = metadata if metadata else []
|
||||
|
||||
def append(self, line: str) -> "FileMetadata":
|
||||
"""
|
||||
Parameters
|
||||
----------
|
||||
line
|
||||
Line of metadata to append
|
||||
"""
|
||||
self.metadata.append(line)
|
||||
return self
|
||||
|
||||
def extend(self, other: Optional["FileMetadata"]) -> "FileMetadata":
|
||||
"""
|
||||
Parameters
|
||||
----------
|
||||
other
|
||||
Other metadata to extend to this one in its entirety
|
||||
"""
|
||||
if other is not None:
|
||||
self.metadata.extend(other.metadata)
|
||||
return self
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls, value_dict: Dict[str, Any], title: Optional[str] = None) -> "FileMetadata":
|
||||
"""
|
||||
Parameters
|
||||
----------
|
||||
value_dict
|
||||
Dict of things to print indented
|
||||
title
|
||||
Optional. Title line to put above the dict
|
||||
"""
|
||||
lines: List[str] = []
|
||||
if title is not None:
|
||||
lines.append(title)
|
||||
|
||||
def _recursive_add_dict_lines(rdict: Dict, indent: int):
|
||||
for key, value in sorted(rdict.items()):
|
||||
if isinstance(value, Dict):
|
||||
_recursive_add_dict_lines(rdict=value, indent=indent + 2)
|
||||
|
||||
_indent = " " * indent
|
||||
lines.append(f"{_indent}{key}: {value}")
|
||||
|
||||
_recursive_add_dict_lines(rdict=value_dict, indent=2)
|
||||
return cls(metadata=lines)
|
||||
|
||||
|
||||
class FileHandlerTransactionLog:
|
||||
"""
|
||||
Tracks file 'transactions' performed by a FileHandler
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
self.files_created: Dict[str, FileMetadata] = {}
|
||||
self.files_removed: Set[str] = set()
|
||||
|
||||
def log_created_file(
|
||||
self, file_name: str, file_metadata: Optional[FileMetadata] = None
|
||||
) -> "FileHandlerTransactionLog":
|
||||
"""
|
||||
Adds a created file to the transaction log
|
||||
|
||||
Parameters
|
||||
----------
|
||||
file_name
|
||||
Name of the file in the output directory
|
||||
file_metadata
|
||||
Optional. If the file has metadata, add it to the transaction log
|
||||
"""
|
||||
if not file_metadata:
|
||||
file_metadata = FileMetadata()
|
||||
|
||||
self.files_created[file_name] = file_metadata
|
||||
return self
|
||||
|
||||
def log_removed_file(self, file_name: str) -> "FileHandlerTransactionLog":
|
||||
"""
|
||||
Records a file removed from the output directory
|
||||
Parameters
|
||||
----------
|
||||
file_name
|
||||
Name of the file in the output directory getting removed
|
||||
"""
|
||||
self.files_removed.add(file_name)
|
||||
return self
|
||||
|
||||
|
||||
class FileHandler:
|
||||
"""
|
||||
Performs and tracks all file moving/copying/deleting
|
||||
"""
|
||||
|
||||
def __init__(self, working_directory: str, output_directory: str, dry_run: bool):
|
||||
self.dry_run = dry_run
|
||||
self.working_directory = working_directory
|
||||
self.output_directory = output_directory
|
||||
self._file_handler_transaction_log = FileHandlerTransactionLog()
|
||||
|
||||
@property
|
||||
def file_handler_transaction_log(self) -> FileHandlerTransactionLog:
|
||||
"""
|
||||
Returns
|
||||
-------
|
||||
Transaction logs of this file handler
|
||||
"""
|
||||
return self._file_handler_transaction_log
|
||||
|
||||
@classmethod
|
||||
def copy(cls, src_file_path: Union[str, Path], dst_file_path: Union[str, Path]):
|
||||
"""
|
||||
Parameters
|
||||
----------
|
||||
src_file_path
|
||||
Source file
|
||||
dst_file_path
|
||||
Destination file
|
||||
"""
|
||||
copyfile(src=src_file_path, dst=dst_file_path)
|
||||
|
||||
@classmethod
|
||||
def delete(cls, file_path: Union[str, Path]):
|
||||
"""
|
||||
Parameters
|
||||
----------
|
||||
file_path
|
||||
File to delete
|
||||
"""
|
||||
if os.path.isfile(file_path):
|
||||
os.remove(file_path)
|
||||
|
||||
def copy_file_to_output_directory(
|
||||
self, file_name: str, output_file_name: str, file_metadata: Optional[FileMetadata] = None
|
||||
):
|
||||
"""
|
||||
Copies a file from the working directory to the output directory.
|
||||
All file copies from working to output directory should use this function for tracking and
|
||||
handling dry-run logic.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
file_name
|
||||
File in the working directory
|
||||
output_file_name
|
||||
Desired output file name in the output_directory
|
||||
file_metadata
|
||||
Optional. Metadata to record to the transaction log for this file
|
||||
"""
|
||||
self._file_handler_transaction_log.log_created_file(
|
||||
file_name=output_file_name, file_metadata=file_metadata
|
||||
)
|
||||
|
||||
if not self.dry_run:
|
||||
output_file_path = Path(self.output_directory) / output_file_name
|
||||
os.makedirs(os.path.dirname(output_file_path), exist_ok=True)
|
||||
self.copy(
|
||||
src_file_path=Path(self.working_directory) / file_name,
|
||||
dst_file_path=output_file_path,
|
||||
)
|
||||
|
||||
def delete_file_from_output_directory(self, file_name: str):
|
||||
"""
|
||||
Deletes a file from the output directory. All file deletions should use this function
|
||||
for tracking and handling dry-run logic.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
file_name
|
||||
File in the output directory to delete
|
||||
"""
|
||||
file_path = Path(self.output_directory) / file_name
|
||||
exists = os.path.isfile(file_path)
|
||||
|
||||
if exists:
|
||||
self._file_handler_transaction_log.log_removed_file(file_name)
|
||||
if not self.dry_run:
|
||||
self.delete(file_path=file_path)
|
||||
|
|
@ -1,13 +1,14 @@
|
|||
import contextlib
|
||||
import io
|
||||
import logging
|
||||
import os
|
||||
import sys
|
||||
import tempfile
|
||||
from dataclasses import dataclass
|
||||
from typing import List
|
||||
from typing import Optional
|
||||
|
||||
from ytdl_sub.utils.file_handler import FileHandler
|
||||
|
||||
|
||||
@dataclass
|
||||
class LoggerLevel:
|
||||
|
|
@ -201,5 +202,5 @@ class Logger:
|
|||
"""
|
||||
cls._DEBUG_LOGGER_FILE.close()
|
||||
|
||||
if delete_debug_file and os.path.isfile(cls.debug_log_filename()):
|
||||
os.remove(cls.debug_log_filename())
|
||||
if delete_debug_file:
|
||||
FileHandler.delete(cls.debug_log_filename())
|
||||
|
|
|
|||
|
|
@ -13,6 +13,9 @@ from typing import Set
|
|||
from yt_dlp import DateRange
|
||||
|
||||
from ytdl_sub.entries.entry import Entry
|
||||
from ytdl_sub.utils.file_handler import FileHandler
|
||||
from ytdl_sub.utils.file_handler import FileHandlerTransactionLog
|
||||
from ytdl_sub.utils.file_handler import FileMetadata
|
||||
from ytdl_sub.utils.logger import Logger
|
||||
|
||||
|
||||
|
|
@ -210,7 +213,7 @@ class DownloadMappings:
|
|||
entry
|
||||
Entry that this file belongs to
|
||||
entry_file_path
|
||||
Relative path to the file that belongs to the entry
|
||||
Relative path to the file that lives in the output directory
|
||||
|
||||
Returns
|
||||
-------
|
||||
|
|
@ -334,16 +337,52 @@ class EnhancedDownloadArchive:
|
|||
6. ( Delete the working directory )
|
||||
"""
|
||||
|
||||
def __init__(self, subscription_name: str, working_directory: str, output_directory: str):
|
||||
def __init__(
|
||||
self,
|
||||
subscription_name: str,
|
||||
working_directory: str,
|
||||
output_directory: str,
|
||||
dry_run: bool = False,
|
||||
):
|
||||
self.subscription_name = subscription_name
|
||||
self.working_directory = working_directory
|
||||
self.output_directory = output_directory
|
||||
|
||||
self._download_archive: Optional[DownloadArchive] = None
|
||||
self._download_mapping: Optional[DownloadMappings] = None
|
||||
self._file_handler = FileHandler(
|
||||
working_directory=working_directory, output_directory=output_directory, dry_run=dry_run
|
||||
)
|
||||
self._download_mapping = DownloadMappings()
|
||||
|
||||
self._logger = Logger.get(name=subscription_name)
|
||||
|
||||
def reinitialize(self, dry_run: bool) -> "EnhancedDownloadArchive":
|
||||
"""
|
||||
Re-initialize the enhanced download archive for successive downloads w/the same
|
||||
subscription.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
dry_run
|
||||
Whether to actually move files to the output directory
|
||||
|
||||
Returns
|
||||
-------
|
||||
self
|
||||
"""
|
||||
self._file_handler = FileHandler(
|
||||
working_directory=self.working_directory,
|
||||
output_directory=self.output_directory,
|
||||
dry_run=dry_run,
|
||||
)
|
||||
self._download_mapping = DownloadMappings()
|
||||
return self
|
||||
|
||||
@property
|
||||
def is_dry_run(self) -> bool:
|
||||
"""
|
||||
Returns
|
||||
-------
|
||||
True if this session is a dry-run. False otherwise.
|
||||
"""
|
||||
return self._file_handler.dry_run
|
||||
|
||||
@property
|
||||
def archive_file_name(self) -> str:
|
||||
"""
|
||||
|
|
@ -353,6 +392,24 @@ class EnhancedDownloadArchive:
|
|||
"""
|
||||
return f".ytdl-subscribe-{self.subscription_name}-download-archive.txt"
|
||||
|
||||
@property
|
||||
def working_directory(self) -> str:
|
||||
"""
|
||||
Returns
|
||||
-------
|
||||
Path to the working directory
|
||||
"""
|
||||
return self._file_handler.working_directory
|
||||
|
||||
@property
|
||||
def output_directory(self) -> str:
|
||||
"""
|
||||
Returns
|
||||
-------
|
||||
Path to the output directory
|
||||
"""
|
||||
return self._file_handler.output_directory
|
||||
|
||||
@property
|
||||
def _mapping_file_name(self) -> str:
|
||||
"""
|
||||
|
|
@ -363,7 +420,7 @@ class EnhancedDownloadArchive:
|
|||
return f".ytdl-sub-{self.subscription_name}-download-archive.json"
|
||||
|
||||
@property
|
||||
def _mapping_output_file_path(self):
|
||||
def _mapping_output_file_path(self) -> str:
|
||||
"""
|
||||
Returns
|
||||
-------
|
||||
|
|
@ -371,6 +428,15 @@ class EnhancedDownloadArchive:
|
|||
"""
|
||||
return str(Path(self.output_directory) / self._mapping_file_name)
|
||||
|
||||
@property
|
||||
def _mapping_working_file_path(self) -> str:
|
||||
"""
|
||||
Returns
|
||||
-------
|
||||
The download mapping's file path in the working directory.
|
||||
"""
|
||||
return str(Path(self.working_directory) / self._mapping_file_name)
|
||||
|
||||
@property
|
||||
def _archive_working_file_path(self) -> str:
|
||||
"""
|
||||
|
|
@ -399,7 +465,6 @@ class EnhancedDownloadArchive:
|
|||
def _load(self) -> "EnhancedDownloadArchive":
|
||||
"""
|
||||
Tries to load download mappings if they are present in the output directory.
|
||||
If they are not, initialize an empty mapping.
|
||||
|
||||
Returns
|
||||
-------
|
||||
|
|
@ -410,14 +475,9 @@ class EnhancedDownloadArchive:
|
|||
self._download_mapping = DownloadMappings.from_file(
|
||||
json_file_path=self._mapping_output_file_path
|
||||
)
|
||||
# Otherwise, init an empty download mappings object. Keep _download_archive as None to
|
||||
# indicate it was not loaded
|
||||
else:
|
||||
self._download_mapping = DownloadMappings()
|
||||
|
||||
return self
|
||||
|
||||
def _copy_to_working_directory(self) -> "EnhancedDownloadArchive":
|
||||
def _copy_mapping_to_working_directory(self) -> "EnhancedDownloadArchive":
|
||||
"""
|
||||
If the mapping is not empty, create a download archive from it and save it into the
|
||||
working directory. This will tell YTDL to not redownload already downloaded entries.
|
||||
|
|
@ -446,7 +506,7 @@ class EnhancedDownloadArchive:
|
|||
self
|
||||
"""
|
||||
self._load()
|
||||
self._copy_to_working_directory()
|
||||
self._copy_mapping_to_working_directory()
|
||||
return self
|
||||
|
||||
def remove_stale_files(self, date_range: DateRange) -> "EnhancedDownloadArchive":
|
||||
|
|
@ -468,12 +528,8 @@ class EnhancedDownloadArchive:
|
|||
)
|
||||
|
||||
for uid, mapping in stale_mappings.items():
|
||||
self._logger.info("[%s] Removing the following stale file(s):", uid)
|
||||
for file_name in mapping.file_names:
|
||||
file_path = Path(self.output_directory) / Path(file_name)
|
||||
self._logger.info(" - %s", file_path)
|
||||
if os.path.exists(file_path):
|
||||
os.remove(file_path)
|
||||
self._file_handler.delete_file_from_output_directory(file_name=file_name)
|
||||
|
||||
self.mapping.remove_entry(entry_id=uid)
|
||||
|
||||
|
|
@ -487,17 +543,96 @@ class EnhancedDownloadArchive:
|
|||
-------
|
||||
self
|
||||
"""
|
||||
# TODO: Make sure this logic is actually right...
|
||||
# Load the download archive from the working directory, which should contain any past
|
||||
# and new entries downloaded in this session
|
||||
self._download_archive = DownloadArchive.from_file(self._archive_working_file_path)
|
||||
|
||||
# Keep the download archive in sync with the mapping
|
||||
for entry_id in self.mapping.entry_ids:
|
||||
if not self._download_archive.contains(entry_id):
|
||||
self._download_archive.remove_entry(entry_id)
|
||||
|
||||
# Save the updated mapping file to the output directory
|
||||
self._download_mapping.to_file(output_json_file=self._mapping_output_file_path)
|
||||
|
||||
self._download_mapping.to_file(output_json_file=self._mapping_working_file_path)
|
||||
self.save_file_to_output_directory(file_name=self._mapping_file_name)
|
||||
return self
|
||||
|
||||
def save_file_to_output_directory(
|
||||
self,
|
||||
file_name: str,
|
||||
file_metadata: Optional[FileMetadata] = None,
|
||||
output_file_name: Optional[str] = None,
|
||||
entry: Optional[Entry] = None,
|
||||
):
|
||||
"""
|
||||
Saves a file from the working directory to the output directory and record it in the
|
||||
transaction log.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
file_name
|
||||
Name of the file to move (does not include working directory path)
|
||||
file_metadata
|
||||
Optional. Metadata to record to the transaction log for this file
|
||||
output_file_name
|
||||
Optional. Final name of the file in the output directory (does not include output
|
||||
directory path). If None, use the same working_directory file_name
|
||||
entry
|
||||
Optional. Entry that this file belongs to
|
||||
"""
|
||||
if output_file_name is None:
|
||||
output_file_name = file_name
|
||||
|
||||
if entry:
|
||||
self.mapping.add_entry(entry=entry, entry_file_path=output_file_name)
|
||||
|
||||
self._file_handler.copy_file_to_output_directory(
|
||||
file_name=file_name, file_metadata=file_metadata, output_file_name=output_file_name
|
||||
)
|
||||
|
||||
def get_file_handler_transaction_log(self) -> FileHandlerTransactionLog:
|
||||
"""
|
||||
Returns
|
||||
-------
|
||||
File handler transaction log for this session
|
||||
"""
|
||||
return self._file_handler.file_handler_transaction_log
|
||||
|
||||
|
||||
class DownloadArchiver:
|
||||
"""
|
||||
Used for any class that saves files. Does not allow direct access to output_directory,
|
||||
forcing the user of the class to use ``save_file`` so it gets archived and avoids any writes
|
||||
during dry-run.
|
||||
"""
|
||||
|
||||
def __init__(self, enhanced_download_archive: EnhancedDownloadArchive):
|
||||
self.__enhanced_download_archive = enhanced_download_archive
|
||||
|
||||
@property
|
||||
def working_directory(self) -> str:
|
||||
"""
|
||||
Returns
|
||||
-------
|
||||
Path to the working directory
|
||||
"""
|
||||
return self.__enhanced_download_archive.working_directory
|
||||
|
||||
@property
|
||||
def is_dry_run(self) -> bool:
|
||||
"""
|
||||
Returns
|
||||
-------
|
||||
True if this session is a dry-run. False otherwise.
|
||||
"""
|
||||
return self.__enhanced_download_archive.is_dry_run
|
||||
|
||||
def save_file(
|
||||
self, file_name: str, output_file_name: Optional[str] = None, entry: Optional[Entry] = None
|
||||
) -> None:
|
||||
"""
|
||||
Saves a file in the working directory to the output directory.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
file_name
|
||||
Name of the file relative to the working directory
|
||||
output_file_name
|
||||
Optional. Final name of the file in the output directory (does not include output
|
||||
directory path). If None, use the same working_directory file_name
|
||||
entry
|
||||
Optional. Entry that the file belongs to
|
||||
"""
|
||||
self.__enhanced_download_archive.save_file_to_output_directory(
|
||||
file_name=file_name, output_file_name=output_file_name, entry=entry
|
||||
)
|
||||
|
|
|
|||
|
|
@ -6,6 +6,8 @@ from typing import List
|
|||
from typing import Optional
|
||||
from typing import Union
|
||||
|
||||
from ytdl_sub.utils.file_handler import FileHandlerTransactionLog
|
||||
|
||||
|
||||
class ExpectedDownload:
|
||||
"""
|
||||
|
|
@ -58,3 +60,13 @@ class ExpectedDownload:
|
|||
f"MD5 hash for {str(relative_path)} does not match: "
|
||||
f"{md5_hash} != {expected_md5_hash}"
|
||||
)
|
||||
|
||||
def assert_dry_run_files_logged(self, transaction_log: FileHandlerTransactionLog):
|
||||
assert (
|
||||
len(transaction_log.files_created) == self.file_count
|
||||
), "Mismatch in number of created files"
|
||||
|
||||
for relative_path in self.expected_md5_file_hashes.keys():
|
||||
assert (
|
||||
str(relative_path) in transaction_log.files_created
|
||||
), f"Expected {str(relative_path)} to be a file but it is not"
|
||||
|
|
|
|||
|
|
@ -29,7 +29,7 @@ def subscription_dict(output_directory, subscription_name):
|
|||
"preset": "sc_discography",
|
||||
"soundcloud": {"url": "https://soundcloud.com/jessebannon"},
|
||||
# override the output directory with our fixture-generated dir
|
||||
"output_options": {"output_directory": output_directory + "/{artist_sanitized}"},
|
||||
"output_options": {"output_directory": output_directory},
|
||||
# download the worst format so it is fast
|
||||
"ytdl_options": {
|
||||
"format": "worst[ext=mp3]",
|
||||
|
|
@ -59,7 +59,7 @@ def expected_discography_download():
|
|||
return ExpectedDownload(
|
||||
expected_md5_file_hashes={
|
||||
# Download mapping
|
||||
Path("j_b/.ytdl-sub-jb-download-archive.json"): "ae55de93b71267b5712c9a3d06c07c26",
|
||||
Path(".ytdl-sub-jb-download-archive.json"): "1a99156e9ece62539fb2608416a07200",
|
||||
|
||||
# Entry files (singles)
|
||||
Path("j_b/[2021] Baby Santana's Dorian Groove/01 - Baby Santana's Dorian Groove.mp3"): "bffbd558e12c6a9e029dc136a88342c4",
|
||||
|
|
@ -99,3 +99,9 @@ class TestSoundcloudDiscography:
|
|||
):
|
||||
discography_subscription.download()
|
||||
expected_discography_download.assert_files_exist(relative_directory=output_directory)
|
||||
|
||||
def test_discography_dry_run(
|
||||
self, discography_subscription, expected_discography_download, output_directory
|
||||
):
|
||||
transaction_log = discography_subscription.download(dry_run=True)
|
||||
expected_discography_download.assert_dry_run_files_logged(transaction_log=transaction_log)
|
||||
|
|
|
|||
|
|
@ -27,12 +27,12 @@ def config(config_path):
|
|||
|
||||
|
||||
@pytest.fixture
|
||||
def subscription_dict(output_directory, subscription_name):
|
||||
def subscription_dict(output_directory):
|
||||
return {
|
||||
"preset": "yt_channel_as_tv",
|
||||
"youtube": {"channel_url": "https://youtube.com/channel/UCcRSMoQqXc_JrBZRHDFGbqA"},
|
||||
# override the output directory with our fixture-generated dir
|
||||
"output_options": {"output_directory": str(Path(output_directory) / subscription_name)},
|
||||
"output_options": {"output_directory": output_directory},
|
||||
# download the worst format so it is fast
|
||||
"ytdl_options": {
|
||||
"format": "worst[ext=mp4]",
|
||||
|
|
@ -68,61 +68,61 @@ def expected_full_channel_download():
|
|||
return ExpectedDownload(
|
||||
expected_md5_file_hashes={
|
||||
# Download mapping
|
||||
Path("pz/.ytdl-sub-pz-download-archive.json"): "b7e7c19d2cf0277e4e42453a64fbaa90",
|
||||
Path(".ytdl-sub-pz-download-archive.json"): "b7e7c19d2cf0277e4e42453a64fbaa90",
|
||||
|
||||
# Output directory files
|
||||
Path("pz/fanart.jpg"): "e6e323373c8902568e96e374817179cf",
|
||||
Path("pz/poster.jpg"): "a14c593bcc75bb8d2c7145de4767ad01",
|
||||
Path("pz/tvshow.nfo"): "83c7db96081ac5bdf289fcf396bec157",
|
||||
Path("fanart.jpg"): "e6e323373c8902568e96e374817179cf",
|
||||
Path("poster.jpg"): "a14c593bcc75bb8d2c7145de4767ad01",
|
||||
Path("tvshow.nfo"): "83c7db96081ac5bdf289fcf396bec157",
|
||||
|
||||
# Entry files
|
||||
Path("pz/Season 2010/s2010.e0813 - Oblivion Mod 'Falcor' p.1.jpg"): "b58377dfe7c39527e1990a24b36bbd77",
|
||||
Path("pz/Season 2010/s2010.e0813 - Oblivion Mod 'Falcor' p.1.mp4"): "931a705864c57d21d6fedebed4af6bbc",
|
||||
Path("pz/Season 2010/s2010.e0813 - Oblivion Mod 'Falcor' p.1.nfo"): "67d8d71d048039080acbba3bce4febaa",
|
||||
Path("Season 2010/s2010.e0813 - Oblivion Mod 'Falcor' p.1.jpg"): "b58377dfe7c39527e1990a24b36bbd77",
|
||||
Path("Season 2010/s2010.e0813 - Oblivion Mod 'Falcor' p.1.mp4"): "931a705864c57d21d6fedebed4af6bbc",
|
||||
Path("Season 2010/s2010.e0813 - Oblivion Mod 'Falcor' p.1.nfo"): "67d8d71d048039080acbba3bce4febaa",
|
||||
|
||||
Path("pz/Season 2010/s2010.e1202 - Oblivion Mod 'Falcor' p.2.jpg"): "a5ee6247c8dce255aec79c9a51d49da4",
|
||||
Path("pz/Season 2010/s2010.e1202 - Oblivion Mod 'Falcor' p.2.mp4"): "d3469b4dca7139cb3dbc38712b6796bf",
|
||||
Path("pz/Season 2010/s2010.e1202 - Oblivion Mod 'Falcor' p.2.nfo"): "d81f49cedbd7edaee987521e89b37904",
|
||||
Path("Season 2010/s2010.e1202 - Oblivion Mod 'Falcor' p.2.jpg"): "a5ee6247c8dce255aec79c9a51d49da4",
|
||||
Path("Season 2010/s2010.e1202 - Oblivion Mod 'Falcor' p.2.mp4"): "d3469b4dca7139cb3dbc38712b6796bf",
|
||||
Path("Season 2010/s2010.e1202 - Oblivion Mod 'Falcor' p.2.nfo"): "d81f49cedbd7edaee987521e89b37904",
|
||||
|
||||
Path("pz/Season 2011/s2011.e0201 - Jesse's Minecraft Server [Trailer - Feb.1].jpg"): "048a19cf0f674437351872c3f312ebf1",
|
||||
Path("pz/Season 2011/s2011.e0201 - Jesse's Minecraft Server [Trailer - Feb.1].mp4"): "e66287b9832277b6a4d1554e29d9fdcc",
|
||||
Path("pz/Season 2011/s2011.e0201 - Jesse's Minecraft Server [Trailer - Feb.1].nfo"): "f7c0de89038f8c491bded8a3968720a2",
|
||||
Path("Season 2011/s2011.e0201 - Jesse's Minecraft Server [Trailer - Feb.1].jpg"): "048a19cf0f674437351872c3f312ebf1",
|
||||
Path("Season 2011/s2011.e0201 - Jesse's Minecraft Server [Trailer - Feb.1].mp4"): "e66287b9832277b6a4d1554e29d9fdcc",
|
||||
Path("Season 2011/s2011.e0201 - Jesse's Minecraft Server [Trailer - Feb.1].nfo"): "f7c0de89038f8c491bded8a3968720a2",
|
||||
|
||||
Path("pz/Season 2011/s2011.e0227 - Jesse's Minecraft Server [Trailer - Feb.27].jpg"): None,
|
||||
Path("pz/Season 2011/s2011.e0227 - Jesse's Minecraft Server [Trailer - Feb.27].mp4"): "04ab5cb3cc12325d0c96a7cd04a8b91d",
|
||||
Path("pz/Season 2011/s2011.e0227 - Jesse's Minecraft Server [Trailer - Feb.27].nfo"): "ee1eda78fa0980bc703e602b5012dd1f",
|
||||
Path("Season 2011/s2011.e0227 - Jesse's Minecraft Server [Trailer - Feb.27].jpg"): None,
|
||||
Path("Season 2011/s2011.e0227 - Jesse's Minecraft Server [Trailer - Feb.27].mp4"): "04ab5cb3cc12325d0c96a7cd04a8b91d",
|
||||
Path("Season 2011/s2011.e0227 - Jesse's Minecraft Server [Trailer - Feb.27].nfo"): "ee1eda78fa0980bc703e602b5012dd1f",
|
||||
|
||||
Path("pz/Season 2011/s2011.e0321 - Jesse's Minecraft Server [Trailer - Mar.21].jpg"): "9baaddc6b62f5b9ae3781eb4eef0e3b3",
|
||||
Path("pz/Season 2011/s2011.e0321 - Jesse's Minecraft Server [Trailer - Mar.21].mp4"): "025de6099a5c98e6397153c7a62d517d",
|
||||
Path("pz/Season 2011/s2011.e0321 - Jesse's Minecraft Server [Trailer - Mar.21].nfo"): "61eb6369430da0ab6134d78829a7621b",
|
||||
Path("Season 2011/s2011.e0321 - Jesse's Minecraft Server [Trailer - Mar.21].jpg"): "9baaddc6b62f5b9ae3781eb4eef0e3b3",
|
||||
Path("Season 2011/s2011.e0321 - Jesse's Minecraft Server [Trailer - Mar.21].mp4"): "025de6099a5c98e6397153c7a62d517d",
|
||||
Path("Season 2011/s2011.e0321 - Jesse's Minecraft Server [Trailer - Mar.21].nfo"): "61eb6369430da0ab6134d78829a7621b",
|
||||
|
||||
Path("pz/Season 2011/s2011.e0529 - Project Zombie _Official Trailer_ (IP - mc.projectzombie.beastnode.net).jpg"): "ce1df7f623fffaefe04606ecbafcfec6",
|
||||
Path("pz/Season 2011/s2011.e0529 - Project Zombie _Official Trailer_ (IP - mc.projectzombie.beastnode.net).mp4"): "3d9c19835b03355d6fd5d00cd59dbe5b",
|
||||
Path("pz/Season 2011/s2011.e0529 - Project Zombie _Official Trailer_ (IP - mc.projectzombie.beastnode.net).nfo"): "60f72b99f5c69f9e03a071a12160928f",
|
||||
Path("Season 2011/s2011.e0529 - Project Zombie _Official Trailer_ (IP - mc.projectzombie.beastnode.net).jpg"): "ce1df7f623fffaefe04606ecbafcfec6",
|
||||
Path("Season 2011/s2011.e0529 - Project Zombie _Official Trailer_ (IP - mc.projectzombie.beastnode.net).mp4"): "3d9c19835b03355d6fd5d00cd59dbe5b",
|
||||
Path("Season 2011/s2011.e0529 - Project Zombie _Official Trailer_ (IP - mc.projectzombie.beastnode.net).nfo"): "60f72b99f5c69f9e03a071a12160928f",
|
||||
|
||||
Path("pz/Season 2011/s2011.e0630 - Project Zombie _Fin.jpg"): "bc3f511915869720c37617a7de706b2b",
|
||||
Path("pz/Season 2011/s2011.e0630 - Project Zombie _Fin.mp4"): "4971cb2d4fa29460361031f3fa8e1ea9",
|
||||
Path("pz/Season 2011/s2011.e0630 - Project Zombie _Fin.nfo"): "a7b5d9e57d20852f5daf360a1373bb7a",
|
||||
Path("Season 2011/s2011.e0630 - Project Zombie _Fin.jpg"): "bc3f511915869720c37617a7de706b2b",
|
||||
Path("Season 2011/s2011.e0630 - Project Zombie _Fin.mp4"): "4971cb2d4fa29460361031f3fa8e1ea9",
|
||||
Path("Season 2011/s2011.e0630 - Project Zombie _Fin.nfo"): "a7b5d9e57d20852f5daf360a1373bb7a",
|
||||
|
||||
Path("pz/Season 2011/s2011.e1121 - Skyrim 'Ultra HD w_Mods' [PC].jpg"): "12babdb3b86cd868b90b60d013295f66",
|
||||
Path("pz/Season 2011/s2011.e1121 - Skyrim 'Ultra HD w_Mods' [PC].mp4"): "55e9b0add08c48c9c66105da0def2426",
|
||||
Path("pz/Season 2011/s2011.e1121 - Skyrim 'Ultra HD w_Mods' [PC].nfo"): "fe60e2b6b564f9316b6c7c183e1cf300",
|
||||
Path("Season 2011/s2011.e1121 - Skyrim 'Ultra HD w_Mods' [PC].jpg"): "12babdb3b86cd868b90b60d013295f66",
|
||||
Path("Season 2011/s2011.e1121 - Skyrim 'Ultra HD w_Mods' [PC].mp4"): "55e9b0add08c48c9c66105da0def2426",
|
||||
Path("Season 2011/s2011.e1121 - Skyrim 'Ultra HD w_Mods' [PC].nfo"): "fe60e2b6b564f9316b6c7c183e1cf300",
|
||||
|
||||
Path("pz/Season 2012/s2012.e0123 - Project Zombie _Map Trailer.jpg"): "82d303e16aba75acdde30b15c4154231",
|
||||
Path("pz/Season 2012/s2012.e0123 - Project Zombie _Map Trailer.mp4"): "65e4ce53ed5ec4139995469f99477a50",
|
||||
Path("pz/Season 2012/s2012.e0123 - Project Zombie _Map Trailer.nfo"): "c8900adcca83c473c79a4afbc7ad2de1",
|
||||
Path("Season 2012/s2012.e0123 - Project Zombie _Map Trailer.jpg"): "82d303e16aba75acdde30b15c4154231",
|
||||
Path("Season 2012/s2012.e0123 - Project Zombie _Map Trailer.mp4"): "65e4ce53ed5ec4139995469f99477a50",
|
||||
Path("Season 2012/s2012.e0123 - Project Zombie _Map Trailer.nfo"): "c8900adcca83c473c79a4afbc7ad2de1",
|
||||
|
||||
Path("pz/Season 2013/s2013.e0719 - Project Zombie Rewind _Trailer.jpg"): "83b1af4c3614d262b2ad419586fff730",
|
||||
Path("pz/Season 2013/s2013.e0719 - Project Zombie Rewind _Trailer.mp4"): "18620a8257a686beda65e54add4d4cd1",
|
||||
Path("pz/Season 2013/s2013.e0719 - Project Zombie Rewind _Trailer.nfo"): "1c993c41d4308a6049333154d0adee16",
|
||||
Path("Season 2013/s2013.e0719 - Project Zombie Rewind _Trailer.jpg"): "83b1af4c3614d262b2ad419586fff730",
|
||||
Path("Season 2013/s2013.e0719 - Project Zombie Rewind _Trailer.mp4"): "18620a8257a686beda65e54add4d4cd1",
|
||||
Path("Season 2013/s2013.e0719 - Project Zombie Rewind _Trailer.nfo"): "1c993c41d4308a6049333154d0adee16",
|
||||
|
||||
Path("pz/Season 2018/s2018.e1029 - Jesse's Minecraft Server _ Teaser Trailer.jpg"): "2a24de903059f48c7d0df0476046c975",
|
||||
Path("pz/Season 2018/s2018.e1029 - Jesse's Minecraft Server _ Teaser Trailer.mp4"): "82f6ee7253e1dbb83ae7215af08ffacc",
|
||||
Path("pz/Season 2018/s2018.e1029 - Jesse's Minecraft Server _ Teaser Trailer.nfo"): "cc7886aae3af6b7b0facd82f95390242",
|
||||
Path("Season 2018/s2018.e1029 - Jesse's Minecraft Server _ Teaser Trailer.jpg"): "2a24de903059f48c7d0df0476046c975",
|
||||
Path("Season 2018/s2018.e1029 - Jesse's Minecraft Server _ Teaser Trailer.mp4"): "82f6ee7253e1dbb83ae7215af08ffacc",
|
||||
Path("Season 2018/s2018.e1029 - Jesse's Minecraft Server _ Teaser Trailer.nfo"): "cc7886aae3af6b7b0facd82f95390242",
|
||||
|
||||
Path("pz/Season 2018/s2018.e1102 - Jesse's Minecraft Server _ IP mc.jesse.id.jpg"): "c8baea83b9edeb081657f1130a1031f7",
|
||||
Path("pz/Season 2018/s2018.e1102 - Jesse's Minecraft Server _ IP mc.jesse.id.mp4"): "e733b4cc385b953b08c8eb0f47e03c1e",
|
||||
Path("pz/Season 2018/s2018.e1102 - Jesse's Minecraft Server _ IP mc.jesse.id.nfo"): "2b3ccb3f1ef81ee49fe1afb88f275a09",
|
||||
Path("Season 2018/s2018.e1102 - Jesse's Minecraft Server _ IP mc.jesse.id.jpg"): "c8baea83b9edeb081657f1130a1031f7",
|
||||
Path("Season 2018/s2018.e1102 - Jesse's Minecraft Server _ IP mc.jesse.id.mp4"): "e733b4cc385b953b08c8eb0f47e03c1e",
|
||||
Path("Season 2018/s2018.e1102 - Jesse's Minecraft Server _ IP mc.jesse.id.nfo"): "2b3ccb3f1ef81ee49fe1afb88f275a09",
|
||||
}
|
||||
)
|
||||
# fmt: on
|
||||
|
|
@ -164,21 +164,21 @@ def expected_recent_channel_download():
|
|||
return ExpectedDownload(
|
||||
expected_md5_file_hashes={
|
||||
# Download mapping
|
||||
Path("pz/.ytdl-sub-pz-download-archive.json"): "b1675ca4d9f0d4b9c2102b6749e4cdfd",
|
||||
Path(".ytdl-sub-pz-download-archive.json"): "b1675ca4d9f0d4b9c2102b6749e4cdfd",
|
||||
|
||||
# Output directory files
|
||||
Path("pz/fanart.jpg"): "e6e323373c8902568e96e374817179cf",
|
||||
Path("pz/poster.jpg"): "a14c593bcc75bb8d2c7145de4767ad01",
|
||||
Path("pz/tvshow.nfo"): "83c7db96081ac5bdf289fcf396bec157",
|
||||
Path("fanart.jpg"): "e6e323373c8902568e96e374817179cf",
|
||||
Path("poster.jpg"): "a14c593bcc75bb8d2c7145de4767ad01",
|
||||
Path("tvshow.nfo"): "83c7db96081ac5bdf289fcf396bec157",
|
||||
|
||||
# Recent Entry files
|
||||
Path("pz/Season 2018/s2018.e1029 - Jesse's Minecraft Server _ Teaser Trailer.jpg"): "2a24de903059f48c7d0df0476046c975",
|
||||
Path("pz/Season 2018/s2018.e1029 - Jesse's Minecraft Server _ Teaser Trailer.mp4"): "82f6ee7253e1dbb83ae7215af08ffacc",
|
||||
Path("pz/Season 2018/s2018.e1029 - Jesse's Minecraft Server _ Teaser Trailer.nfo"): "cc7886aae3af6b7b0facd82f95390242",
|
||||
Path("Season 2018/s2018.e1029 - Jesse's Minecraft Server _ Teaser Trailer.jpg"): "2a24de903059f48c7d0df0476046c975",
|
||||
Path("Season 2018/s2018.e1029 - Jesse's Minecraft Server _ Teaser Trailer.mp4"): "82f6ee7253e1dbb83ae7215af08ffacc",
|
||||
Path("Season 2018/s2018.e1029 - Jesse's Minecraft Server _ Teaser Trailer.nfo"): "cc7886aae3af6b7b0facd82f95390242",
|
||||
|
||||
Path("pz/Season 2018/s2018.e1102 - Jesse's Minecraft Server _ IP mc.jesse.id.jpg"): "c8baea83b9edeb081657f1130a1031f7",
|
||||
Path("pz/Season 2018/s2018.e1102 - Jesse's Minecraft Server _ IP mc.jesse.id.mp4"): "e733b4cc385b953b08c8eb0f47e03c1e",
|
||||
Path("pz/Season 2018/s2018.e1102 - Jesse's Minecraft Server _ IP mc.jesse.id.nfo"): "2b3ccb3f1ef81ee49fe1afb88f275a09",
|
||||
Path("Season 2018/s2018.e1102 - Jesse's Minecraft Server _ IP mc.jesse.id.jpg"): "c8baea83b9edeb081657f1130a1031f7",
|
||||
Path("Season 2018/s2018.e1102 - Jesse's Minecraft Server _ IP mc.jesse.id.mp4"): "e733b4cc385b953b08c8eb0f47e03c1e",
|
||||
Path("Season 2018/s2018.e1102 - Jesse's Minecraft Server _ IP mc.jesse.id.nfo"): "2b3ccb3f1ef81ee49fe1afb88f275a09",
|
||||
}
|
||||
)
|
||||
# fmt: on
|
||||
|
|
@ -222,12 +222,12 @@ def expected_recent_channel_no_vids_in_range_download():
|
|||
return ExpectedDownload(
|
||||
expected_md5_file_hashes={
|
||||
# Download mapping
|
||||
Path("pz/.ytdl-sub-pz-download-archive.json"): "99914b932bd37a50b983c5e7c90ae93b",
|
||||
Path(".ytdl-sub-pz-download-archive.json"): "99914b932bd37a50b983c5e7c90ae93b",
|
||||
|
||||
# Output directory files
|
||||
Path("pz/fanart.jpg"): "e6e323373c8902568e96e374817179cf",
|
||||
Path("pz/poster.jpg"): "a14c593bcc75bb8d2c7145de4767ad01",
|
||||
Path("pz/tvshow.nfo"): "83c7db96081ac5bdf289fcf396bec157",
|
||||
Path("fanart.jpg"): "e6e323373c8902568e96e374817179cf",
|
||||
Path("poster.jpg"): "a14c593bcc75bb8d2c7145de4767ad01",
|
||||
Path("tvshow.nfo"): "83c7db96081ac5bdf289fcf396bec157",
|
||||
}
|
||||
)
|
||||
# fmt: on
|
||||
|
|
@ -269,17 +269,17 @@ def expected_rolling_recent_channel_download():
|
|||
return ExpectedDownload(
|
||||
expected_md5_file_hashes={
|
||||
# Download mapping
|
||||
Path("pz/.ytdl-sub-pz-download-archive.json"): "9ae3463bd2dc39830003aba68a276df4",
|
||||
Path(".ytdl-sub-pz-download-archive.json"): "9ae3463bd2dc39830003aba68a276df4",
|
||||
|
||||
# Output directory files
|
||||
Path("pz/fanart.jpg"): "e6e323373c8902568e96e374817179cf",
|
||||
Path("pz/poster.jpg"): "a14c593bcc75bb8d2c7145de4767ad01",
|
||||
Path("pz/tvshow.nfo"): "83c7db96081ac5bdf289fcf396bec157",
|
||||
Path("fanart.jpg"): "e6e323373c8902568e96e374817179cf",
|
||||
Path("poster.jpg"): "a14c593bcc75bb8d2c7145de4767ad01",
|
||||
Path("tvshow.nfo"): "83c7db96081ac5bdf289fcf396bec157",
|
||||
|
||||
# Rolling Recent Entry files
|
||||
Path("pz/Season 2018/s2018.e1102 - Jesse's Minecraft Server _ IP mc.jesse.id.jpg"): "c8baea83b9edeb081657f1130a1031f7",
|
||||
Path("pz/Season 2018/s2018.e1102 - Jesse's Minecraft Server _ IP mc.jesse.id.mp4"): "e733b4cc385b953b08c8eb0f47e03c1e",
|
||||
Path("pz/Season 2018/s2018.e1102 - Jesse's Minecraft Server _ IP mc.jesse.id.nfo"): "2b3ccb3f1ef81ee49fe1afb88f275a09",
|
||||
Path("Season 2018/s2018.e1102 - Jesse's Minecraft Server _ IP mc.jesse.id.jpg"): "c8baea83b9edeb081657f1130a1031f7",
|
||||
Path("Season 2018/s2018.e1102 - Jesse's Minecraft Server _ IP mc.jesse.id.mp4"): "e733b4cc385b953b08c8eb0f47e03c1e",
|
||||
Path("Season 2018/s2018.e1102 - Jesse's Minecraft Server _ IP mc.jesse.id.nfo"): "2b3ccb3f1ef81ee49fe1afb88f275a09",
|
||||
}
|
||||
)
|
||||
# fmt: on
|
||||
|
|
@ -297,6 +297,12 @@ class TestChannelAsKodiTvShow:
|
|||
full_channel_subscription.download()
|
||||
expected_full_channel_download.assert_files_exist(relative_directory=output_directory)
|
||||
|
||||
def test_full_channel_dry_run(
|
||||
self, full_channel_subscription, expected_full_channel_download, output_directory
|
||||
):
|
||||
transaction_log = full_channel_subscription.download(dry_run=True)
|
||||
expected_full_channel_download.assert_dry_run_files_logged(transaction_log=transaction_log)
|
||||
|
||||
def test_recent_channel_download(
|
||||
self, recent_channel_subscription, expected_recent_channel_download, output_directory
|
||||
):
|
||||
|
|
@ -311,6 +317,14 @@ class TestChannelAsKodiTvShow:
|
|||
recent_channel_subscription.download()
|
||||
expected_recent_channel_download.assert_files_exist(relative_directory=output_directory)
|
||||
|
||||
def test_recent_channel_dry_run(
|
||||
self, recent_channel_subscription, expected_recent_channel_download, output_directory
|
||||
):
|
||||
transaction_log = recent_channel_subscription.download(dry_run=True)
|
||||
expected_recent_channel_download.assert_dry_run_files_logged(
|
||||
transaction_log=transaction_log
|
||||
)
|
||||
|
||||
def test_recent_channel_download__no_vids_in_range(
|
||||
self,
|
||||
recent_channel_no_vids_in_range_subscription,
|
||||
|
|
@ -328,6 +342,17 @@ class TestChannelAsKodiTvShow:
|
|||
relative_directory=output_directory
|
||||
)
|
||||
|
||||
def test_recent_channel_dry_run__no_vids_in_range(
|
||||
self,
|
||||
recent_channel_no_vids_in_range_subscription,
|
||||
expected_recent_channel_no_vids_in_range_download,
|
||||
output_directory,
|
||||
):
|
||||
transaction_log = recent_channel_no_vids_in_range_subscription.download(dry_run=True)
|
||||
expected_recent_channel_no_vids_in_range_download.assert_dry_run_files_logged(
|
||||
transaction_log=transaction_log
|
||||
)
|
||||
|
||||
def test_rolling_recent_channel_download(
|
||||
self,
|
||||
recent_channel_subscription,
|
||||
|
|
|
|||
|
|
@ -81,8 +81,14 @@ class TestYoutubeMergePlaylist:
|
|||
files exist and have the expected md5 file hashes.
|
||||
"""
|
||||
|
||||
def test_playlist_download(
|
||||
def test_merge_playlist_download(
|
||||
self, playlist_subscription, expected_playlist_download, output_directory
|
||||
):
|
||||
playlist_subscription.download()
|
||||
expected_playlist_download.assert_files_exist(relative_directory=output_directory)
|
||||
|
||||
def test_merge_playlist_dry_run(
|
||||
self, playlist_subscription, expected_playlist_download, output_directory
|
||||
):
|
||||
transaction_log = playlist_subscription.download(dry_run=True)
|
||||
expected_playlist_download.assert_dry_run_files_logged(transaction_log=transaction_log)
|
||||
|
|
|
|||
|
|
@ -148,8 +148,22 @@ class TestPlaylistAsKodiMusicVideo:
|
|||
playlist_subscription.download()
|
||||
expected_playlist_download.assert_files_exist(relative_directory=output_directory)
|
||||
|
||||
def test_playlist_dry_run(
|
||||
self, playlist_subscription, expected_playlist_download, output_directory
|
||||
):
|
||||
file_transaction_log = playlist_subscription.download(dry_run=True)
|
||||
expected_playlist_download.assert_dry_run_files_logged(transaction_log=file_transaction_log)
|
||||
|
||||
def test_single_video_download(
|
||||
self, single_video_subscription, expected_single_video_download, output_directory
|
||||
):
|
||||
single_video_subscription.download()
|
||||
expected_single_video_download.assert_files_exist(relative_directory=output_directory)
|
||||
|
||||
def test_single_video_dry_run(
|
||||
self, single_video_subscription, expected_single_video_download, output_directory
|
||||
):
|
||||
file_transaction_log = single_video_subscription.download(dry_run=True)
|
||||
expected_single_video_download.assert_dry_run_files_logged(
|
||||
transaction_log=file_transaction_log
|
||||
)
|
||||
|
|
|
|||
|
|
@ -119,8 +119,12 @@ class TestPlaylistAsKodiMusicVideo:
|
|||
files exist and have the expected md5 file hashes.
|
||||
"""
|
||||
|
||||
def test_single_video_download(
|
||||
def test_split_video_download(
|
||||
self, single_video_subscription, expected_single_video_download, output_directory
|
||||
):
|
||||
single_video_subscription.download()
|
||||
expected_single_video_download.assert_files_exist(relative_directory=output_directory)
|
||||
|
||||
def test_split_video_dry_run(self, single_video_subscription, expected_single_video_download):
|
||||
transaction_log = single_video_subscription.download(dry_run=True)
|
||||
expected_single_video_download.assert_dry_run_files_logged(transaction_log=transaction_log)
|
||||
|
|
|
|||
Loading…
Reference in a new issue