danvics/ytdl-sub
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

all tests passing, lint 10, should test dry-run arg

Browse source
This commit is contained in:
jbannon 2022-07-02 18:11:29 +00:00
parent 179540093c
commit 685ea60607
7 changed files with 92 additions and 46 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

21
src/ytdl_sub/downloaders/youtube/merge_playlist.py
View file

@ -100,8 +100,7 @@ class YoutubeMergePlaylistDownloader(
},
)
@classmethod
def _add_chapters(cls, merged_video: YoutubeVideo) -> None:
def _add_chapters(self, merged_video: YoutubeVideo) -> None:
titles: List[str] = []
timestamps: List[Timestamp] = []
@ -112,11 +111,13 @@ class YoutubeMergePlaylistDownloader(
current_timestamp_sec += video_entry["duration"]
add_ffmpeg_metadata(
file_path=merged_video.get_download_file_path(),
chapters=Chapters(timestamps=timestamps, titles=titles),
file_duration_sec=merged_video.kwargs("duration"),
)
# TODO: return chapter metadata here
if not self.is_dry_run:
add_ffmpeg_metadata(
file_path=merged_video.get_download_file_path(),
chapters=Chapters(timestamps=timestamps, titles=titles),
file_duration_sec=merged_video.kwargs("duration"),
)
def _to_merged_video(self, entry_dict: Dict) -> YoutubeVideo:
"""
@ -130,7 +131,11 @@ class YoutubeMergePlaylistDownloader(
entry_dict["duration"] = sum(
playlist_entry["duration"] for playlist_entry in entry_dict["entries"]
)
entry_dict["ext"] = entry_dict["requested_downloads"][0]["ext"]
entry_dict["ext"] = (
entry_dict["requested_downloads"][0]["ext"]
if "requested_downloads" in entry_dict
else "mkv"
)
return YoutubeVideo(entry_dict=entry_dict, working_directory=self.working_directory)
def download(self) -> List[YoutubeVideo]:

32
src/ytdl_sub/downloaders/youtube/split_video.py
View file

@ -155,26 +155,26 @@ class YoutubeSplitVideoDownloader(
for idx, title in enumerate(chapters.titles):
new_uid = _split_video_uid(source_uid=entry.uid, idx=idx)
# Get the input/output file paths
input_file = entry.get_download_file_path()
output_file = str(Path(self.working_directory) / f"{new_uid}.{entry.ext}")
output_thumbnail_file = str(
Path(self.working_directory) / f"{new_uid}.{entry.thumbnail_ext}"
)
if not self.is_dry_run:
# Get the input/output file paths
input_file = entry.get_download_file_path()
output_file = str(Path(self.working_directory) / f"{new_uid}.{entry.ext}")
# Run ffmpeg to create the split the video
FFMPEG.run(
_split_video_ffmpeg_cmd(
input_file=input_file,
output_file=output_file,
timestamps=chapters.timestamps,
idx=idx,
# Run ffmpeg to create the split the video
FFMPEG.run(
_split_video_ffmpeg_cmd(
input_file=input_file,
output_file=output_file,
timestamps=chapters.timestamps,
idx=idx,
)
)
)
# Copy the thumbnail
# Copy the original vid thumbnail to the working directory with the new uid. This so
# downstream logic thinks this split video has its own thumbnail
FileHandler.copy(
src_file_path=entry.get_download_thumbnail_path(),
dst_file_path=output_thumbnail_file,
dst_file_path=Path(self.working_directory) / f"{new_uid}.{entry.thumbnail_ext}",
)
# Format the split video as a YoutubePlaylistVideo

31
src/ytdl_sub/subscriptions/subscription.py
View file

@ -4,6 +4,7 @@ import os
import shutil
from pathlib import Path
from typing import List
from typing import Optional
from typing import Tuple
from typing import Type
@ -19,6 +20,7 @@ from ytdl_sub.entries.entry import Entry
from ytdl_sub.plugins.plugin import Plugin
from ytdl_sub.plugins.plugin import PluginOptions
from ytdl_sub.utils.file_handler import FileHandlerTransactionLog
from ytdl_sub.utils.file_handler import FileMetadata
from ytdl_sub.utils.thumbnail import convert_download_thumbnail
from ytdl_sub.ytdl_additions.enhanced_download_archive import EnhancedDownloadArchive
@ -145,7 +147,9 @@ class Subscription:
and self.downloader_class.supports_download_archive
)
def _copy_entry_files_to_output_directory(self, entry: Entry):
def _copy_entry_files_to_output_directory(
self, entry: Entry, entry_metadata: Optional[FileMetadata] = None
):
"""
Helper function to move the media file and optionally thumbnail file to the output directory
for a single entry.
@ -154,13 +158,18 @@ class Subscription:
----------
entry:
The entry with files to move
entry_metadata
Optional. Metadata to record to the transaction log for this entry
"""
# Move the file after all direct file modifications are complete
output_file_name = self.overrides.apply_formatter(
formatter=self.output_options.file_name, entry=entry
)
self._enhanced_download_archive.save_file_to_output_directory(
file_name=entry.get_download_file_name(), output_file_name=output_file_name, entry=entry
file_name=entry.get_download_file_name(),
file_metadata=entry_metadata,
output_file_name=output_file_name,
entry=entry,
)
if self.output_options.thumbnail_name:
@ -257,17 +266,19 @@ class Subscription:
ytdl_options=ytdl_options,
)
entries = downloader.download()
for plugin in plugins:
for entry in entries:
plugin.post_process_entry(entry)
for entry in downloader.download():
# TODO: Add entry metadata from the downloader.download function
entry_metadata = FileMetadata()
for plugin in plugins:
entry_metadata.extend(plugin.post_process_entry(entry))
plugin.post_process_subscription()
for entry in entries:
self._copy_entry_files_to_output_directory(entry=entry)
self._copy_entry_files_to_output_directory(
entry=entry, entry_metadata=entry_metadata
)
downloader.post_download(overrides=self.overrides)
for plugin in plugins:
plugin.post_process_subscription()
return self._enhanced_download_archive.get_file_handler_transaction_log()

17
src/ytdl_sub/utils/file_handler.py
View file

@ -11,7 +11,7 @@ from typing import Union
class FileMetadata:
"""
Stores pretty-printed information about a file
Stores pretty-printed information about a file. Each line in the metadata represents a newline
"""
def __init__(self, metadata: Optional[List[str]] = None):
@ -27,14 +27,15 @@ class FileMetadata:
self.metadata.append(line)
return self
def extend(self, other: "FileMetadata") -> "FileMetadata":
def extend(self, other: Optional["FileMetadata"]) -> "FileMetadata":
"""
Parameters
----------
other
Other metadata to extend to this one in its entirety
"""
self.metadata.extend(other.metadata)
if other is not None:
self.metadata.extend(other.metadata)
return self
@classmethod
@ -146,7 +147,9 @@ class FileHandler:
if os.path.isfile(file_path):
os.remove(file_path)
def copy_file_to_output_directory(self, file_name: str, output_file_name: str):
def copy_file_to_output_directory(
self, file_name: str, output_file_name: str, file_metadata: Optional[FileMetadata] = None
):
"""
Copies a file from the working directory to the output directory.
All file copies from working to output directory should use this function for tracking and
@ -158,8 +161,12 @@ class FileHandler:
File in the working directory
output_file_name
Desired output file name in the output_directory
file_metadata
Optional. Metadata to record to the transaction log for this file
"""
self._file_handler_transaction_log.log_created_file(output_file_name)
self._file_handler_transaction_log.log_created_file(
file_name=output_file_name, file_metadata=file_metadata
)
if not self.dry_run:
output_file_path = Path(self.output_directory) / output_file_name

23
src/ytdl_sub/ytdl_additions/enhanced_download_archive.py
View file

@ -15,6 +15,7 @@ from yt_dlp import DateRange
from ytdl_sub.entries.entry import Entry
from ytdl_sub.utils.file_handler import FileHandler
from ytdl_sub.utils.file_handler import FileHandlerTransactionLog
from ytdl_sub.utils.file_handler import FileMetadata
from ytdl_sub.utils.logger import Logger
@ -547,15 +548,22 @@ class EnhancedDownloadArchive:
return self
def save_file_to_output_directory(
self, file_name: str, output_file_name: Optional[str] = None, entry: Optional[Entry] = None
self,
file_name: str,
file_metadata: Optional[FileMetadata] = None,
output_file_name: Optional[str] = None,
entry: Optional[Entry] = None,
):
"""
Saves a file from the working directory to the output directory
Saves a file from the working directory to the output directory and record it in the
transaction log.
Parameters
----------
file_name
Name of the file to move (does not include working directory path)
file_metadata
Optional. Metadata to record to the transaction log for this file
output_file_name
Optional. Final name of the file in the output directory (does not include output
directory path). If None, use the same working_directory file_name
@ -569,7 +577,7 @@ class EnhancedDownloadArchive:
self.mapping.add_entry(entry=entry, entry_file_path=output_file_name)
self._file_handler.copy_file_to_output_directory(
file_name=file_name, output_file_name=output_file_name
file_name=file_name, file_metadata=file_metadata, output_file_name=output_file_name
)
def get_file_handler_transaction_log(self) -> FileHandlerTransactionLog:
@ -609,7 +617,9 @@ class DownloadArchiver:
"""
return self.__enhanced_download_archive.is_dry_run
def save_file(self, file_name: str, entry: Optional[Entry] = None) -> None:
def save_file(
self, file_name: str, output_file_name: Optional[str] = None, entry: Optional[Entry] = None
) -> None:
"""
Saves a file in the working directory to the output directory.
@ -617,9 +627,12 @@ class DownloadArchiver:
----------
file_name
Name of the file relative to the working directory
output_file_name
Optional. Final name of the file in the output directory (does not include output
directory path). If None, use the same working_directory file_name
entry
Optional. Entry that the file belongs to
"""
self.__enhanced_download_archive.save_file_to_output_directory(
file_name=file_name, entry=entry
file_name=file_name, output_file_name=output_file_name, entry=entry
)

8
tests/e2e/youtube/test_merge_playlist.py
View file

@ -81,8 +81,14 @@ class TestYoutubeMergePlaylist:
files exist and have the expected md5 file hashes.
"""
def test_playlist_download(
def test_merge_playlist_download(
self, playlist_subscription, expected_playlist_download, output_directory
):
playlist_subscription.download()
expected_playlist_download.assert_files_exist(relative_directory=output_directory)
def test_merge_playlist_dry_run(
self, playlist_subscription, expected_playlist_download, output_directory
):
transaction_log = playlist_subscription.download(dry_run=True)
expected_playlist_download.assert_dry_run_files_logged(transaction_log=transaction_log)

6
tests/e2e/youtube/test_split_video.py
View file

@ -119,8 +119,12 @@ class TestPlaylistAsKodiMusicVideo:
files exist and have the expected md5 file hashes.
"""
def test_single_video_download(
def test_split_video_download(
self, single_video_subscription, expected_single_video_download, output_directory
):
single_video_subscription.download()
expected_single_video_download.assert_files_exist(relative_directory=output_directory)
def test_split_video_dry_run(self, single_video_subscription, expected_single_video_download):
transaction_log = single_video_subscription.download(dry_run=True)
expected_single_video_download.assert_dry_run_files_logged(transaction_log=transaction_log)