danvics/ytdl-sub
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

[BACKEND] Remove split_video download strategy (#232)

Browse source
This commit is contained in:
Jesse Bannon 2022-09-14 22:03:44 -07:00 committed by GitHub
parent c08ea64b5a
commit 3894625479
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
5 changed files with 1 additions and 343 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

13
docs/config.rst
View file

@ -83,17 +83,6 @@ _____
-------------------------------------------------------------------------------
split_video
___________
.. autoclass:: ytdl_sub.downloaders.youtube.split_video.YoutubeSplitVideoDownloaderOptions()
:members:
:member-order: bysource
:inherited-members:
.. autofunction:: ytdl_sub.downloaders.youtube.split_video.YoutubeSplitVideoDownloader.ytdl_option_defaults()
-------------------------------------------------------------------------------
merge_playlist
______________
.. autoclass:: ytdl_sub.downloaders.youtube.merge_playlist.YoutubeMergePlaylistDownloaderOptions()
@ -315,7 +304,7 @@ the ``{playlist_name}`` variable since the preset did not.
Source Variables
----------------
.. autoclass:: ytdl_sub.entries.variables.entry_variables.SourceVariables
.. autoclass:: ytdl_sub.entries.variables.entry_variables.EntryVariables
.. _youtube-variables:

2
src/ytdl_sub/config/preset_class_mappings.py
View file

@ -7,7 +7,6 @@ from ytdl_sub.downloaders.soundcloud.albums_and_singles import SoundcloudAlbumsA
from ytdl_sub.downloaders.youtube.channel import YoutubeChannelDownloader
from ytdl_sub.downloaders.youtube.merge_playlist import YoutubeMergePlaylistDownloader
from ytdl_sub.downloaders.youtube.playlist import YoutubePlaylistDownloader
from ytdl_sub.downloaders.youtube.split_video import YoutubeSplitVideoDownloader
from ytdl_sub.downloaders.youtube.video import YoutubeVideoDownloader
from ytdl_sub.plugins.audio_extract import AudioExtractPlugin
from ytdl_sub.plugins.chapters import ChaptersPlugin
@ -32,7 +31,6 @@ class DownloadStrategyMapping:
"video": YoutubeVideoDownloader,
"playlist": YoutubePlaylistDownloader,
"channel": YoutubeChannelDownloader,
"split_video": YoutubeSplitVideoDownloader,
"merge_playlist": YoutubeMergePlaylistDownloader,
},
"soundcloud": {

194
src/ytdl_sub/downloaders/youtube/split_video.py
View file

@ -1,194 +0,0 @@
import copy
import re
from pathlib import Path
from typing import Dict
from typing import List
from typing import Tuple
from ytdl_sub.downloaders.youtube.abc import YoutubeDownloader
from ytdl_sub.downloaders.youtube.video import YoutubeVideoDownloaderOptions
from ytdl_sub.entries.youtube import YoutubePlaylistVideo
from ytdl_sub.entries.youtube import YoutubeVideo
from ytdl_sub.plugins.split_by_chapters import _split_video_ffmpeg_cmd
from ytdl_sub.utils.chapters import Chapters
from ytdl_sub.utils.chapters import Timestamp
from ytdl_sub.utils.ffmpeg import FFMPEG
# Captures the following formats:
# 0:00 title
# 00:00 title
# 1:00:00 title
# 01:00:00 title
# where capture group 1 and 2 are the timestamp and title, respectively
from ytdl_sub.utils.file_handler import FileHandler
from ytdl_sub.utils.file_handler import FileMetadata
from ytdl_sub.utils.thumbnail import convert_download_thumbnail
from ytdl_sub.validators.validators import StringValidator
_SPLIT_TIMESTAMP_REGEX = re.compile(r"^((?:\d\d:)?(?:\d:)?(?:\d)?\d:\d\d) (.+)$")
def _split_video_uid(source_uid: str, idx: int) -> str:
return f"{source_uid}___{idx}"
class YoutubeSplitVideoDownloaderOptions(YoutubeVideoDownloaderOptions):
r"""
DEPRECATED: Will be removed in v0.5.0. Use the ``split_by_chapters`` plugin instead.
Downloads a single youtube video, then splits in to separate videos using a file containing
timestamps. Each separate video will be formatted as if it was downloaded from a playlist.
This download strategy is intended for CLI usage performing a one-time download of a video,
not a subscription.
Usage:
.. code-block:: yaml
presets:
example_preset:
youtube:
# required
download_strategy: "split_video"
video_url: "youtube.com/watch?v=VMAPTo7RVDo"
split_timestamps: path/to/timestamps.txt
CLI usage:
.. code-block:: bash
ytdl-sub dl \
--preset "example_preset" \
--youtube.video_url "youtube.com/watch?v=VMAPTo7RVDo" \
--youtube.split_timestamps "path/to/timestamps.txt"
``split_timestamps`` file format:
.. code-block:: markdown
0:00 Intro
0:24 Blackwater Park
10:23 Bleak
16:39 Jokes
1:02:23 Ending
The above will create 5 videos in total. The last timestamp, in this example, would create a
video starting at ``1:02:23`` and end at Youtube video's ending.
"""
_required_keys = {"video_url", "split_timestamps"}
def __init__(self, name, value):
super().__init__(name, value)
self._split_timestamps = self._validate_key("split_timestamps", StringValidator).value
@property
def split_timestamps(self) -> str:
"""
Required. The path to the file containing the split timestamps.
"""
return self._split_timestamps
class YoutubeSplitVideoDownloader(
YoutubeDownloader[YoutubeSplitVideoDownloaderOptions, YoutubePlaylistVideo]
):
downloader_options_type = YoutubeSplitVideoDownloaderOptions
downloader_entry_type = YoutubePlaylistVideo
supports_download_archive = False
supports_subtitles = False
@classmethod
def ytdl_option_defaults(cls) -> Dict:
"""
Default `ytdl_options`_ for ``split_video``
.. code-block:: yaml
ytdl_options:
ignoreerrors: True # ignore errors like hidden videos, age restriction, etc
"""
return dict(
super().ytdl_option_defaults(),
**{"break_on_existing": True},
)
def _create_split_video_entry(
self, source_entry_dict: Dict, title: str, idx: int, chapters: Chapters
) -> Tuple[YoutubePlaylistVideo, FileMetadata]:
"""
Runs ffmpeg to create the split video
"""
entry_dict = copy.deepcopy(source_entry_dict)
entry_dict["title"] = title
entry_dict["playlist_index"] = idx + 1
entry_dict["playlist_count"] = len(chapters.timestamps)
entry_dict["id"] = _split_video_uid(source_uid=entry_dict["id"], idx=idx)
# Remove track and artist since its now split
if "track" in entry_dict:
del entry_dict["track"]
if "artist" in entry_dict:
del entry_dict["artist"]
timestamp_begin = chapters.timestamps[idx].readable_str
timestamp_end = Timestamp(source_entry_dict["duration"]).readable_str
if idx + 1 < len(chapters.timestamps):
timestamp_end = chapters.timestamps[idx + 1].readable_str
metadata = FileMetadata(metadata=f"{timestamp_begin} - {timestamp_end}")
return (
YoutubePlaylistVideo(entry_dict=entry_dict, working_directory=self.working_directory),
metadata,
)
def download(self) -> List[Tuple[YoutubePlaylistVideo, FileMetadata]]:
"""Download a single Youtube video, then split it into multiple videos"""
split_videos_and_metadata: List[Tuple[YoutubePlaylistVideo, FileMetadata]] = []
chapters = Chapters.from_timestamps_file(
chapters_file_path=self.download_options.split_timestamps
)
entry_dict = self.extract_info(url=self.download_options.video_url)
entry = YoutubeVideo(entry_dict=entry_dict, working_directory=self.working_directory)
# convert the entry thumbnail early so we do not have to guess the thumbnail extension
# when copying it
if not self.is_dry_run:
convert_download_thumbnail(entry=entry)
for idx, title in enumerate(chapters.titles):
new_uid = _split_video_uid(source_uid=entry.uid, idx=idx)
if not self.is_dry_run:
# Get the input/output file paths
input_file = entry.get_download_file_path()
output_file = str(Path(self.working_directory) / f"{new_uid}.{entry.ext}")
# Run ffmpeg to create the split the video
FFMPEG.run(
_split_video_ffmpeg_cmd(
input_file=input_file,
output_file=output_file,
timestamps=chapters.timestamps,
idx=idx,
)
)
# Copy the original vid thumbnail to the working directory with the new uid. This so
# downstream logic thinks this split video has its own thumbnail
FileHandler.copy(
src_file_path=entry.get_download_thumbnail_path(),
dst_file_path=Path(self.working_directory) / f"{new_uid}.{entry.thumbnail_ext}",
)
# Format the split video as a YoutubePlaylistVideo
split_videos_and_metadata.append(
self._create_split_video_entry(
source_entry_dict=entry_dict, title=title, idx=idx, chapters=chapters
)
)
return split_videos_and_metadata

68
tests/e2e/resources/transaction_log_summaries/youtube/test_split_video.txt
View file

@ -1,68 +0,0 @@
Files created in '{output_directory}'
----------------------------------------
Project Zombie - 1-6.Intro.mp4
0:00 - 0:10
Project Zombie - 2-6.Part 1.mp4
0:10 - 0:20
Project Zombie - 3-6.Part 2.mp4
0:20 - 0:30
Project Zombie - 4-6.Part 3.mp4
0:30 - 0:40
Project Zombie - 5-6.Part 4.mp4
0:40 - 1:01
Project Zombie - 6-6.Part 5.mp4
1:01 - 1:28
Project Zombie - Intro-thumb.jpg
Project Zombie - Intro.info.json
Project Zombie - Intro.nfo
NFO tags:
musicvideo:
album: Music Videos
artist: Project Zombie
title: Intro
year: 2010
Project Zombie - Part 1-thumb.jpg
Project Zombie - Part 1.info.json
Project Zombie - Part 1.nfo
NFO tags:
musicvideo:
album: Music Videos
artist: Project Zombie
title: Part 1
year: 2010
Project Zombie - Part 2-thumb.jpg
Project Zombie - Part 2.info.json
Project Zombie - Part 2.nfo
NFO tags:
musicvideo:
album: Music Videos
artist: Project Zombie
title: Part 2
year: 2010
Project Zombie - Part 3-thumb.jpg
Project Zombie - Part 3.info.json
Project Zombie - Part 3.nfo
NFO tags:
musicvideo:
album: Music Videos
artist: Project Zombie
title: Part 3
year: 2010
Project Zombie - Part 4-thumb.jpg
Project Zombie - Part 4.info.json
Project Zombie - Part 4.nfo
NFO tags:
musicvideo:
album: Music Videos
artist: Project Zombie
title: Part 4
year: 2010
Project Zombie - Part 5-thumb.jpg
Project Zombie - Part 5.info.json
Project Zombie - Part 5.nfo
NFO tags:
musicvideo:
album: Music Videos
artist: Project Zombie
title: Part 5
year: 2010

67
tests/e2e/youtube/test_split_video.py
View file

@ -1,67 +0,0 @@
import pytest
from e2e.expected_download import assert_expected_downloads
from e2e.expected_transaction_log import assert_transaction_log_matches
from ytdl_sub.config.preset import Preset
from ytdl_sub.subscriptions.subscription import Subscription
@pytest.fixture
def subscription_dict(output_directory, timestamps_file_path):
return {
"preset": "yt_music_video",
"youtube": {
"download_strategy": "split_video",
"video_url": "https://youtube.com/watch?v=HKTNxEqsN3Q",
"split_timestamps": timestamps_file_path,
},
# override the output directory with our fixture-generated dir
"output_options": {
"output_directory": output_directory,
"file_name": "{channel_sanitized} - {playlist_index}-{playlist_size}"
".{title_sanitized}.{ext}",
},
# download the worst format so it is fast
"ytdl_options": {
"format": "worst[ext=mp4]",
},
}
####################################################################################################
# SINGLE VIDEO FIXTURES
@pytest.fixture
def single_video_subscription(music_video_config, subscription_dict):
single_video_preset = Preset.from_dict(
config=music_video_config,
preset_name="split_video_test",
preset_dict=subscription_dict,
)
return Subscription.from_preset(
preset=single_video_preset,
config=music_video_config,
)
class TestPlaylistAsKodiMusicVideo:
"""
Downloads my old minecraft youtube channel, pretends they are music videos. Ensure the above
files exist and have the expected md5 file hashes.
"""
@pytest.mark.parametrize("dry_run", [True, False])
def test_split_video_download(self, single_video_subscription, output_directory, dry_run):
transaction_log = single_video_subscription.download(dry_run=dry_run)
assert_transaction_log_matches(
output_directory=output_directory,
transaction_log=transaction_log,
transaction_log_summary_file_name="youtube/test_split_video.txt",
)
assert_expected_downloads(
output_directory=output_directory,
dry_run=dry_run,
expected_download_summary_file_name="youtube/test_split_video.json",
)