[BACKEND] Purge youtube + soundcloud download strategies (#408)
* [BACKEND] Purge youtube + soundcloud download strategies * update unit tests * fix missing for ffmpeg; * try update
This commit is contained in:
parent
c84496910f
commit
053042b4b9
13 changed files with 30 additions and 674 deletions
4
.github/workflows/ci.yaml
vendored
4
.github/workflows/ci.yaml
vendored
|
|
@ -69,6 +69,7 @@ jobs:
|
|||
|
||||
- name: Run unit tests with coverage
|
||||
run: |
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y ffmpeg
|
||||
source /opt/env/bin/activate
|
||||
coverage run -m pytest tests/unit && coverage xml -o /opt/coverage/unit/coverage.xml
|
||||
|
|
@ -96,6 +97,7 @@ jobs:
|
|||
|
||||
- name: Run e2e soundcloud tests with coverage
|
||||
run: |
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y ffmpeg
|
||||
source /opt/env/bin/activate
|
||||
coverage run -m pytest tests/e2e/soundcloud && coverage xml -o /opt/coverage/soundcloud/coverage.xml
|
||||
|
|
@ -123,6 +125,7 @@ jobs:
|
|||
|
||||
- name: Run e2e youtube tests with coverage
|
||||
run: |
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y ffmpeg
|
||||
source /opt/env/bin/activate
|
||||
coverage run -m pytest tests/e2e/youtube && coverage xml -o /opt/coverage/youtube/coverage.xml
|
||||
|
|
@ -150,6 +153,7 @@ jobs:
|
|||
|
||||
- name: Run e2e plugin tests with coverage
|
||||
run: |
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y ffmpeg
|
||||
source /opt/env/bin/activate
|
||||
coverage run -m pytest tests/e2e/plugins && coverage xml -o /opt/coverage/plugins/coverage.xml
|
||||
|
|
|
|||
|
|
@ -5,11 +5,6 @@ from typing import Type
|
|||
from ytdl_sub.downloaders.downloader import Downloader
|
||||
from ytdl_sub.downloaders.generic.multi_url import MultiUrlDownloader
|
||||
from ytdl_sub.downloaders.generic.url import UrlDownloader
|
||||
from ytdl_sub.downloaders.soundcloud.albums_and_singles import SoundcloudAlbumsAndSinglesDownloader
|
||||
from ytdl_sub.downloaders.youtube.channel import YoutubeChannelDownloader
|
||||
from ytdl_sub.downloaders.youtube.merge_playlist import YoutubeMergePlaylistDownloader
|
||||
from ytdl_sub.downloaders.youtube.playlist import YoutubePlaylistDownloader
|
||||
from ytdl_sub.downloaders.youtube.video import YoutubeVideoDownloader
|
||||
from ytdl_sub.plugins.audio_extract import AudioExtractPlugin
|
||||
from ytdl_sub.plugins.chapters import ChaptersPlugin
|
||||
from ytdl_sub.plugins.date_range import DateRangePlugin
|
||||
|
|
@ -32,15 +27,6 @@ class DownloadStrategyMapping:
|
|||
"""
|
||||
|
||||
_MAPPING: Dict[str, Dict[str, Type[Downloader]]] = {
|
||||
"youtube": {
|
||||
"video": YoutubeVideoDownloader,
|
||||
"playlist": YoutubePlaylistDownloader,
|
||||
"channel": YoutubeChannelDownloader,
|
||||
"merge_playlist": YoutubeMergePlaylistDownloader,
|
||||
},
|
||||
"soundcloud": {
|
||||
"albums_and_singles": SoundcloudAlbumsAndSinglesDownloader,
|
||||
},
|
||||
"download": {
|
||||
"multi_url": MultiUrlDownloader,
|
||||
"url": UrlDownloader,
|
||||
|
|
|
|||
|
|
@ -1,145 +0,0 @@
|
|||
from typing import Any
|
||||
from typing import Dict
|
||||
from typing import Generator
|
||||
|
||||
from ytdl_sub.downloaders.downloader import Downloader
|
||||
from ytdl_sub.downloaders.downloader import DownloaderValidator
|
||||
from ytdl_sub.downloaders.generic.validators import MultiUrlValidator
|
||||
from ytdl_sub.entries.entry import Entry
|
||||
from ytdl_sub.validators.url_validator import SoundcloudUsernameUrlValidator
|
||||
from ytdl_sub.validators.validators import BoolValidator
|
||||
|
||||
|
||||
class SoundcloudAlbumsAndSinglesDownloadOptions(DownloaderValidator):
|
||||
"""
|
||||
Downloads a soundcloud user's entire discography. Groups together album tracks and considers
|
||||
any track not in an album as a single. Also includes any collaboration tracks.
|
||||
|
||||
Usage:
|
||||
|
||||
.. code-block:: yaml
|
||||
|
||||
presets:
|
||||
my_example_preset:
|
||||
soundcloud:
|
||||
# required
|
||||
download_strategy: "albums_and_singles"
|
||||
url: "soundcloud.com/username"
|
||||
# optional
|
||||
skip_premiere_tracks: True
|
||||
|
||||
"""
|
||||
|
||||
_required_keys = {"url"}
|
||||
_optional_keys = {"skip_premiere_tracks"}
|
||||
|
||||
@classmethod
|
||||
def partial_validate(cls, name: str, value: Any) -> None:
|
||||
"""
|
||||
Partially validate a Soundcloud source
|
||||
"""
|
||||
if isinstance(value, dict):
|
||||
value["url"] = value.get("url", "https://soundcloud.com/jessebannon")
|
||||
_ = cls(name, value)
|
||||
|
||||
def __init__(self, name, value):
|
||||
super().__init__(name, value)
|
||||
self._url = self._validate_key(
|
||||
key="url", validator=SoundcloudUsernameUrlValidator
|
||||
).username_url
|
||||
self._skip_premiere_tracks = self._validate_key(
|
||||
"skip_premiere_tracks", BoolValidator, default=True
|
||||
)
|
||||
|
||||
@property
|
||||
def collection_validator(self) -> MultiUrlValidator:
|
||||
"""Downloads the album tracks first, then the tracks"""
|
||||
return MultiUrlValidator(
|
||||
name=self._name,
|
||||
value={
|
||||
"urls": [
|
||||
{
|
||||
"url": f"{self._url}/tracks",
|
||||
"variables": {
|
||||
"track_number": "1",
|
||||
"track_number_padded": "01",
|
||||
"track_count": "1",
|
||||
"album": "{title}",
|
||||
"album_sanitized": "{title_sanitized}",
|
||||
"album_year": "{upload_year}",
|
||||
},
|
||||
},
|
||||
{
|
||||
"url": f"{self._url}/albums",
|
||||
"variables": {
|
||||
"track_number": "{playlist_index}",
|
||||
"track_number_padded": "{playlist_index_padded}",
|
||||
"track_count": "{playlist_count}",
|
||||
"album": "{playlist_title}",
|
||||
"album_sanitized": "{playlist_title_sanitized}",
|
||||
"album_year": "{playlist_max_upload_year}",
|
||||
},
|
||||
},
|
||||
]
|
||||
},
|
||||
)
|
||||
|
||||
@property
|
||||
def skip_premiere_tracks(self) -> bool:
|
||||
"""
|
||||
Optional. True to skip tracks that require purchasing. False otherwise. Defaults to True.
|
||||
"""
|
||||
return self._skip_premiere_tracks.value
|
||||
|
||||
@property
|
||||
def url(self) -> str:
|
||||
"""
|
||||
Required. The Soundcloud user's url, i.e. ``soundcloud.com/the_username``
|
||||
"""
|
||||
return self._url
|
||||
|
||||
|
||||
class SoundcloudAlbumsAndSinglesDownloader(Downloader[SoundcloudAlbumsAndSinglesDownloadOptions]):
|
||||
downloader_options_type = SoundcloudAlbumsAndSinglesDownloadOptions
|
||||
downloader_entry_type = Entry
|
||||
|
||||
supports_subtitles = False
|
||||
supports_chapters = False
|
||||
|
||||
@classmethod
|
||||
def ytdl_option_defaults(cls) -> Dict:
|
||||
"""
|
||||
Default `ytdl_options`_ for ``albums_and_singles``
|
||||
|
||||
.. code-block:: yaml
|
||||
|
||||
ytdl_options:
|
||||
ignoreerrors: True # ignore errors like hidden videos, age restriction, etc
|
||||
format: "bestaudio[ext=mp3]" # download format the best possible mp3
|
||||
"""
|
||||
return dict(
|
||||
super().ytdl_option_defaults(),
|
||||
**{
|
||||
"format": "bestaudio[ext=mp3]",
|
||||
},
|
||||
)
|
||||
|
||||
def _should_skip(self, entry: Entry) -> bool:
|
||||
if not self.download_options.skip_premiere_tracks:
|
||||
return False
|
||||
|
||||
for url in [entry.kwargs_get("url", ""), entry.webpage_url]:
|
||||
if "/preview/" in url:
|
||||
return True
|
||||
|
||||
return False
|
||||
|
||||
def download(self) -> Generator[Entry, None, None]:
|
||||
"""
|
||||
Soundcloud subscription to download albums and tracks as singles.
|
||||
"""
|
||||
for entry in super().download():
|
||||
if self._should_skip(entry):
|
||||
continue
|
||||
|
||||
yield entry
|
||||
|
|
@ -1,139 +0,0 @@
|
|||
from typing import Any
|
||||
from typing import Dict
|
||||
from typing import List
|
||||
from typing import Optional
|
||||
|
||||
from ytdl_sub.downloaders.downloader import Downloader
|
||||
from ytdl_sub.downloaders.downloader import DownloaderValidator
|
||||
from ytdl_sub.downloaders.generic.validators import MultiUrlValidator
|
||||
from ytdl_sub.validators.string_formatter_validators import OverridesStringFormatterValidator
|
||||
from ytdl_sub.validators.url_validator import YoutubeChannelUrlValidator
|
||||
|
||||
|
||||
class YoutubeChannelDownloaderOptions(DownloaderValidator):
|
||||
"""
|
||||
Downloads all videos from a youtube channel.
|
||||
|
||||
Usage:
|
||||
|
||||
.. code-block:: yaml
|
||||
|
||||
presets:
|
||||
my_example_preset:
|
||||
youtube:
|
||||
# required
|
||||
download_strategy: "channel"
|
||||
channel_url: "UCsvn_Po0SmunchJYtttWpOxMg"
|
||||
# optional
|
||||
channel_avatar_path: "poster.jpg"
|
||||
channel_banner_path: "fanart.jpg"
|
||||
"""
|
||||
|
||||
_required_keys = {"channel_url"}
|
||||
_optional_keys = {
|
||||
"channel_avatar_path",
|
||||
"channel_banner_path",
|
||||
}
|
||||
|
||||
@classmethod
|
||||
def partial_validate(cls, name: str, value: Any) -> None:
|
||||
"""
|
||||
Partially validate a YouTube channel source
|
||||
"""
|
||||
if isinstance(value, dict):
|
||||
value["channel_url"] = value.get(
|
||||
"channel_url", "https://www.youtube.com/c/ProjectZombie603"
|
||||
)
|
||||
_ = cls(name, value)
|
||||
|
||||
def __init__(self, name, value):
|
||||
super().__init__(name, value)
|
||||
self._channel_url = self._validate_key(
|
||||
"channel_url", YoutubeChannelUrlValidator
|
||||
).channel_url
|
||||
self._channel_avatar_path = self._validate_key_if_present(
|
||||
"channel_avatar_path", OverridesStringFormatterValidator
|
||||
)
|
||||
self._channel_banner_path = self._validate_key_if_present(
|
||||
"channel_banner_path", OverridesStringFormatterValidator
|
||||
)
|
||||
|
||||
@property
|
||||
def collection_validator(self) -> MultiUrlValidator:
|
||||
"""Download from the channel url"""
|
||||
playlist_thumbnails: List[Dict] = []
|
||||
if self._channel_avatar_path:
|
||||
playlist_thumbnails.append(
|
||||
{
|
||||
"name": self._channel_avatar_path.format_string,
|
||||
"uid": "avatar_uncropped",
|
||||
}
|
||||
)
|
||||
if self._channel_banner_path:
|
||||
playlist_thumbnails.append(
|
||||
{
|
||||
"name": self._channel_banner_path.format_string,
|
||||
"uid": "banner_uncropped",
|
||||
}
|
||||
)
|
||||
|
||||
return MultiUrlValidator(
|
||||
name=self._name,
|
||||
value={
|
||||
"urls": [
|
||||
{
|
||||
"url": self.channel_url,
|
||||
"playlist_thumbnails": playlist_thumbnails,
|
||||
"variables": {"playlist_size": "{playlist_count}"},
|
||||
}
|
||||
]
|
||||
},
|
||||
)
|
||||
|
||||
@property
|
||||
def channel_url(self) -> str:
|
||||
"""
|
||||
Required. The channel's url, i.e.
|
||||
``https://www.youtube.com/channel/UCsvn_Po0SmunchJYOWpOxMg``. URLs with ``/username`` or
|
||||
``/c`` are valid to use.
|
||||
"""
|
||||
return self._channel_url
|
||||
|
||||
@property
|
||||
def channel_avatar_path(self) -> Optional[OverridesStringFormatterValidator]:
|
||||
"""
|
||||
Optional. Path to store the channel's avatar thumbnail image to.
|
||||
"""
|
||||
return self._channel_avatar_path
|
||||
|
||||
@property
|
||||
def channel_banner_path(self) -> Optional[OverridesStringFormatterValidator]:
|
||||
"""
|
||||
Optional. Path to store the channel's banner image to.
|
||||
"""
|
||||
return self._channel_banner_path
|
||||
|
||||
|
||||
class YoutubeChannelDownloader(Downloader[YoutubeChannelDownloaderOptions]):
|
||||
downloader_options_type = YoutubeChannelDownloaderOptions
|
||||
|
||||
# pylint: disable=line-too-long
|
||||
@classmethod
|
||||
def ytdl_option_defaults(cls) -> Dict:
|
||||
"""
|
||||
Default `ytdl_options`_ for ``channel``
|
||||
|
||||
.. code-block:: yaml
|
||||
|
||||
ytdl_options:
|
||||
ignoreerrors: True # ignore errors like hidden videos, age restriction, etc
|
||||
break_on_existing: True # stop downloads (newest to oldest) if a video is already downloaded
|
||||
"""
|
||||
return dict(
|
||||
super().ytdl_option_defaults(),
|
||||
**{
|
||||
"break_on_existing": True,
|
||||
},
|
||||
)
|
||||
|
||||
# pylint: enable=line-too-long
|
||||
|
|
@ -1,155 +0,0 @@
|
|||
from typing import Dict
|
||||
from typing import List
|
||||
from typing import Optional
|
||||
from typing import Tuple
|
||||
|
||||
from ytdl_sub.downloaders.downloader import Downloader
|
||||
from ytdl_sub.downloaders.youtube.playlist import YoutubePlaylistDownloaderOptions
|
||||
from ytdl_sub.entries.entry import Entry
|
||||
from ytdl_sub.utils.chapters import Chapters
|
||||
from ytdl_sub.utils.chapters import Timestamp
|
||||
from ytdl_sub.utils.ffmpeg import set_ffmpeg_metadata_chapters
|
||||
from ytdl_sub.utils.file_handler import FileMetadata
|
||||
from ytdl_sub.validators.validators import BoolValidator
|
||||
|
||||
|
||||
class YoutubeMergePlaylistDownloaderOptions(YoutubePlaylistDownloaderOptions):
|
||||
r"""
|
||||
Downloads all videos in a playlist and merges them into a single video.
|
||||
|
||||
Usage:
|
||||
|
||||
.. code-block:: yaml
|
||||
|
||||
presets:
|
||||
example_preset:
|
||||
youtube:
|
||||
# required
|
||||
download_strategy: "merge_playlist"
|
||||
playlist_url: "https://www.youtube.com/playlist?list=UCsvn_Po0SmunchJYtttWpOxMg"
|
||||
# optional
|
||||
add_chapters: False
|
||||
|
||||
CLI usage:
|
||||
|
||||
.. code-block:: bash
|
||||
|
||||
ytdl-sub dl \
|
||||
--preset "example_preset" \
|
||||
--youtube.playlist_url "https://www.youtube.com/playlist?list=UCsvn_Po0SmunchJYtttWpOxMg" \
|
||||
--youtube.add_chapters True
|
||||
"""
|
||||
|
||||
_required_keys = {"playlist_url"}
|
||||
_optional_keys = {"add_chapters"}
|
||||
|
||||
def __init__(self, name, value):
|
||||
super().__init__(name, value)
|
||||
self._add_chapters = self._validate_key_if_present(
|
||||
"add_chapters", validator=BoolValidator, default=False
|
||||
).value
|
||||
|
||||
@property
|
||||
def add_chapters(self) -> Optional[bool]:
|
||||
"""
|
||||
Optional. Whether to add chapters using each video's title in the merged playlist.
|
||||
Defaults to False.
|
||||
"""
|
||||
return self._add_chapters
|
||||
|
||||
|
||||
class YoutubeMergePlaylistDownloader(Downloader[YoutubeMergePlaylistDownloaderOptions]):
|
||||
downloader_options_type = YoutubeMergePlaylistDownloaderOptions
|
||||
supports_download_archive = False
|
||||
supports_subtitles = False
|
||||
supports_chapters = False
|
||||
|
||||
@classmethod
|
||||
def ytdl_option_defaults(cls) -> Dict:
|
||||
"""
|
||||
Default `ytdl_options`_ for ``merge_playlist``
|
||||
|
||||
.. code-block:: yaml
|
||||
|
||||
ytdl_options:
|
||||
ignoreerrors: True # ignore errors like hidden videos, age restriction, etc
|
||||
postprocessors:
|
||||
# Convert the videos to mkv format
|
||||
- key: "FFmpegVideoConvertor"
|
||||
when: "post_process"
|
||||
preferedformat: "mkv"
|
||||
# Concatenate all the playlist videos into a single file
|
||||
- key: "FFmpegConcat"
|
||||
when: "playlist"
|
||||
"""
|
||||
return dict(
|
||||
super().ytdl_option_defaults(),
|
||||
**{
|
||||
"postprocessors": [
|
||||
{
|
||||
"key": "FFmpegVideoConvertor",
|
||||
"when": "post_process",
|
||||
"preferedformat": "mkv",
|
||||
},
|
||||
{
|
||||
"key": "FFmpegConcat",
|
||||
"when": "playlist",
|
||||
},
|
||||
],
|
||||
},
|
||||
)
|
||||
|
||||
def _get_chapters(self, merged_video: Entry, add_chapters: bool) -> FileMetadata:
|
||||
titles: List[str] = []
|
||||
timestamps: List[Timestamp] = []
|
||||
|
||||
current_timestamp_sec = 0
|
||||
for video_entry in merged_video.kwargs("entries"):
|
||||
timestamps.append(Timestamp(current_timestamp_sec))
|
||||
titles.append(video_entry["title"])
|
||||
|
||||
current_timestamp_sec += video_entry["duration"]
|
||||
|
||||
chapters = Chapters(timestamps=timestamps, titles=titles)
|
||||
|
||||
if not self.is_dry_run and add_chapters:
|
||||
set_ffmpeg_metadata_chapters(
|
||||
file_path=merged_video.get_download_file_path(),
|
||||
chapters=chapters,
|
||||
file_duration_sec=merged_video.kwargs("duration"),
|
||||
)
|
||||
|
||||
return chapters.to_file_metadata(title="Timestamps of playlist videos in the merged file")
|
||||
|
||||
def _to_merged_video(self, entry_dict: Dict) -> Entry:
|
||||
"""
|
||||
Adds a few entries not included in a playlist entry to make it look like a merged video
|
||||
entry_dict
|
||||
"""
|
||||
# Set the upload date to be the latest playlist video date
|
||||
entry_dict["upload_date"] = max(
|
||||
playlist_entry["upload_date"] for playlist_entry in entry_dict["entries"]
|
||||
)
|
||||
entry_dict["duration"] = sum(
|
||||
playlist_entry["duration"] for playlist_entry in entry_dict["entries"]
|
||||
)
|
||||
entry_dict["ext"] = (
|
||||
entry_dict["requested_downloads"][0]["ext"]
|
||||
if "requested_downloads" in entry_dict
|
||||
else "mkv"
|
||||
)
|
||||
entry_dict["webpage_url"] = self.download_options.playlist_url
|
||||
|
||||
return Entry(entry_dict=entry_dict, working_directory=self.working_directory)
|
||||
|
||||
def download(self) -> List[Tuple[Entry, FileMetadata]]:
|
||||
"""Download a single Youtube video, then split it into multiple videos"""
|
||||
url = self.overrides.apply_formatter(self.collection.urls.list[0].url)
|
||||
|
||||
entry_dict = self.extract_info(url=url, ytdl_options_overrides=self.download_ytdl_options)
|
||||
merged_video = self._to_merged_video(entry_dict=entry_dict)
|
||||
|
||||
merged_video_metadata = self._get_chapters(
|
||||
merged_video=merged_video, add_chapters=self.download_options.add_chapters
|
||||
)
|
||||
return [(merged_video, merged_video_metadata)]
|
||||
|
|
@ -1,116 +0,0 @@
|
|||
from typing import Any
|
||||
from typing import Dict
|
||||
from typing import List
|
||||
from typing import Optional
|
||||
|
||||
from ytdl_sub.downloaders.downloader import Downloader
|
||||
from ytdl_sub.downloaders.downloader import DownloaderValidator
|
||||
from ytdl_sub.downloaders.generic.validators import MultiUrlValidator
|
||||
from ytdl_sub.utils.thumbnail import ThumbnailTypes
|
||||
from ytdl_sub.validators.string_formatter_validators import OverridesStringFormatterValidator
|
||||
from ytdl_sub.validators.url_validator import YoutubePlaylistUrlValidator
|
||||
|
||||
|
||||
class YoutubePlaylistDownloaderOptions(DownloaderValidator):
|
||||
"""
|
||||
Downloads all videos from a youtube playlist.
|
||||
|
||||
Usage:
|
||||
|
||||
.. code-block:: yaml
|
||||
|
||||
presets:
|
||||
my_example_preset:
|
||||
youtube:
|
||||
# required
|
||||
download_strategy: "playlist"
|
||||
playlist_url: "https://www.youtube.com/playlist?list=UCsvn_Po0SmunchJYtttWpOxMg"
|
||||
# optional
|
||||
playlist_thumbnail_name: "poster.jpg"
|
||||
"""
|
||||
|
||||
_required_keys = {"playlist_url"}
|
||||
_optional_keys = {"playlist_thumbnail_name"}
|
||||
|
||||
@classmethod
|
||||
def partial_validate(cls, name: str, value: Any) -> None:
|
||||
"""
|
||||
Partially validate a YouTube playlist source
|
||||
"""
|
||||
if isinstance(value, dict):
|
||||
value["playlist_url"] = value.get(
|
||||
"playlist_url", "https://www.youtube.com/playlist?list=UCsvn_Po0SmunchJYtttWpOxMg"
|
||||
)
|
||||
_ = cls(name, value)
|
||||
|
||||
def __init__(self, name, value):
|
||||
super().__init__(name, value)
|
||||
self._playlist_url = self._validate_key(
|
||||
"playlist_url", YoutubePlaylistUrlValidator
|
||||
).playlist_url
|
||||
self._playlist_thumbnail_name = self._validate_key_if_present(
|
||||
"playlist_thumbnail_name", OverridesStringFormatterValidator
|
||||
)
|
||||
|
||||
@property
|
||||
def collection_validator(self) -> MultiUrlValidator:
|
||||
"""Downloads the playlist url"""
|
||||
playlist_thumbnails: List[Dict] = []
|
||||
if self.playlist_thumbnail_name:
|
||||
playlist_thumbnails.append(
|
||||
{
|
||||
"name": self.playlist_thumbnail_name.format_string,
|
||||
"uid": ThumbnailTypes.LATEST_ENTRY,
|
||||
}
|
||||
)
|
||||
|
||||
return MultiUrlValidator(
|
||||
name=self._name,
|
||||
value={
|
||||
"urls": [
|
||||
{
|
||||
"url": self.playlist_url,
|
||||
"playlist_thumbnails": playlist_thumbnails,
|
||||
"variables": {"playlist_size": "{playlist_count}"},
|
||||
}
|
||||
]
|
||||
},
|
||||
)
|
||||
|
||||
@property
|
||||
def playlist_url(self) -> str:
|
||||
"""
|
||||
Required. The playlist's url, i.e.
|
||||
``https://www.youtube.com/playlist?list=UCsvn_Po0SmunchJYtttWpOxMg``.
|
||||
"""
|
||||
return self._playlist_url
|
||||
|
||||
@property
|
||||
def playlist_thumbnail_name(self) -> Optional[OverridesStringFormatterValidator]:
|
||||
"""
|
||||
Optional. Path to store the playlist's thumbnail
|
||||
"""
|
||||
return self._playlist_thumbnail_name
|
||||
|
||||
|
||||
class YoutubePlaylistDownloader(Downloader[YoutubePlaylistDownloaderOptions]):
|
||||
downloader_options_type = YoutubePlaylistDownloaderOptions
|
||||
|
||||
# pylint: disable=line-too-long
|
||||
@classmethod
|
||||
def ytdl_option_defaults(cls) -> Dict:
|
||||
"""
|
||||
Default `ytdl_options`_ for ``playlist``
|
||||
|
||||
.. code-block:: yaml
|
||||
|
||||
ytdl_options:
|
||||
ignoreerrors: True # ignore errors like hidden videos, age restriction, etc
|
||||
break_on_existing: True # stop downloads (newest to oldest) if a video is already downloaded
|
||||
"""
|
||||
return dict(
|
||||
super().ytdl_option_defaults(),
|
||||
**{"break_on_existing": True},
|
||||
)
|
||||
|
||||
# pylint: enable=line-too-long
|
||||
|
|
@ -1,80 +0,0 @@
|
|||
from typing import Any
|
||||
from typing import Dict
|
||||
|
||||
from ytdl_sub.downloaders.downloader import Downloader
|
||||
from ytdl_sub.downloaders.downloader import DownloaderValidator
|
||||
from ytdl_sub.downloaders.generic.url import UrlDownloadOptions
|
||||
from ytdl_sub.downloaders.generic.validators import MultiUrlValidator
|
||||
from ytdl_sub.validators.url_validator import YoutubeVideoUrlValidator
|
||||
|
||||
|
||||
class YoutubeVideoDownloaderOptions(DownloaderValidator):
|
||||
"""
|
||||
Downloads a single youtube video. This download strategy is intended for CLI usage performing
|
||||
a one-time download of a video, not a subscription.
|
||||
|
||||
Usage:
|
||||
|
||||
.. code-block:: yaml
|
||||
|
||||
presets:
|
||||
example_preset:
|
||||
youtube:
|
||||
# required
|
||||
download_strategy: "video"
|
||||
video_url: "youtube.com/watch?v=VMAPTo7RVDo"
|
||||
|
||||
CLI usage:
|
||||
|
||||
.. code-block:: bash
|
||||
|
||||
ytdl-sub dl --preset "example_preset" --youtube.video_url "youtube.com/watch?v=VMAPTo7RVDo"
|
||||
"""
|
||||
|
||||
_required_keys = {"video_url"}
|
||||
|
||||
@classmethod
|
||||
def partial_validate(cls, name: str, value: Any) -> None:
|
||||
"""
|
||||
Partially validate a YouTube video source
|
||||
"""
|
||||
if isinstance(value, dict):
|
||||
value["video_url"] = value.get("video_url", "youtube.com/watch?v=VMAPTo7RVDo")
|
||||
_ = cls(name, value)
|
||||
|
||||
def __init__(self, name, value):
|
||||
super().__init__(name, value)
|
||||
self._video_url = self._validate_key("video_url", YoutubeVideoUrlValidator).video_url
|
||||
|
||||
@property
|
||||
def collection_validator(self) -> MultiUrlValidator:
|
||||
"""Downloads the video url"""
|
||||
return UrlDownloadOptions(
|
||||
name=self._name, value={"url": self.video_url}
|
||||
).collection_validator
|
||||
|
||||
@property
|
||||
def video_url(self) -> str:
|
||||
"""
|
||||
Required. The url of the video, i.e. ``youtube.com/watch?v=VMAPTo7RVDo``.
|
||||
"""
|
||||
return self._video_url
|
||||
|
||||
|
||||
class YoutubeVideoDownloader(Downloader[YoutubeVideoDownloaderOptions]):
|
||||
downloader_options_type = YoutubeVideoDownloaderOptions
|
||||
|
||||
@classmethod
|
||||
def ytdl_option_defaults(cls) -> Dict:
|
||||
"""
|
||||
Default `ytdl_options`_ for ``video``
|
||||
|
||||
.. code-block:: yaml
|
||||
|
||||
ytdl_options:
|
||||
ignoreerrors: True # ignore errors like hidden videos, age restriction, etc
|
||||
"""
|
||||
return dict(
|
||||
super().ytdl_option_defaults(),
|
||||
**{"break_on_existing": True},
|
||||
)
|
||||
|
|
@ -48,6 +48,6 @@ def output_options() -> Dict:
|
|||
@pytest.fixture
|
||||
def youtube_video() -> Dict:
|
||||
return {
|
||||
"download_strategy": "video",
|
||||
"video_url": "youtube.com/watch?v=123abc",
|
||||
"download_strategy": "url",
|
||||
"url": "youtube.com/watch?v=123abc",
|
||||
}
|
||||
|
|
|
|||
|
|
@ -66,12 +66,13 @@ class TestConfigFilePartiallyValidatesPresets:
|
|||
f"Allowed fields: {', '.join(sorted(PRESET_KEYS))}",
|
||||
)
|
||||
|
||||
def test_error__multiple_sources(self):
|
||||
self._partial_validate(
|
||||
preset_dict={"youtube": {}, "download": {}},
|
||||
expected_error_message="Validation error in partial_preset: "
|
||||
"Contains the sources download, youtube but can only have one",
|
||||
)
|
||||
# TODO: Update with future source
|
||||
# def test_error__multiple_sources(self):
|
||||
# self._partial_validate(
|
||||
# preset_dict={"youtube": {}, "download": {}},
|
||||
# expected_error_message="Validation error in partial_preset: "
|
||||
# "Contains the sources download, youtube but can only have one",
|
||||
# )
|
||||
|
||||
def test_error__no_download_strategy(self):
|
||||
self._partial_validate(
|
||||
|
|
|
|||
|
|
@ -10,18 +10,18 @@ class TestPreset:
|
|||
@pytest.mark.parametrize(
|
||||
"source, download_strategy",
|
||||
[
|
||||
("youtube", {"download_strategy": "video", "video_url": "youtube.com/watch?v=123abc"}),
|
||||
("download", {"download_strategy": "url", "url": "youtube.com/watch?v=123abc"}),
|
||||
(
|
||||
"youtube",
|
||||
"download",
|
||||
{
|
||||
"download_strategy": "playlist",
|
||||
"playlist_url": "youtube.com/playlist?list=123abc",
|
||||
"download_strategy": "url",
|
||||
"url": "youtube.com/playlist?list=123abc",
|
||||
},
|
||||
),
|
||||
("youtube", {"download_strategy": "channel", "channel_url": "youtube.com/c/123abc"}),
|
||||
("download", {"download_strategy": "url", "url": "youtube.com/c/123abc"}),
|
||||
(
|
||||
"soundcloud",
|
||||
{"download_strategy": "albums_and_singles", "url": "soundcloud.com/123abc"},
|
||||
"download",
|
||||
{"download_strategy": "url", "url": "soundcloud.com/123abc"},
|
||||
),
|
||||
],
|
||||
)
|
||||
|
|
@ -37,7 +37,7 @@ class TestPreset:
|
|||
config=config_file,
|
||||
name="test",
|
||||
value={
|
||||
"youtube": youtube_video,
|
||||
"download": youtube_video,
|
||||
"output_options": {"output_directory": "dir", "file_name": "{dne_var}"},
|
||||
"overrides": {"dne_var": "not dne"},
|
||||
},
|
||||
|
|
@ -49,7 +49,7 @@ class TestPreset:
|
|||
name="test",
|
||||
value={
|
||||
"preset": "parent_preset_1",
|
||||
"youtube": youtube_video,
|
||||
"download": youtube_video,
|
||||
"output_options": output_options,
|
||||
"nfo_tags": {"tags": {"key-2": "this-preset"}},
|
||||
},
|
||||
|
|
@ -74,7 +74,7 @@ class TestPreset:
|
|||
name="test",
|
||||
value={
|
||||
"preset": preset_value,
|
||||
"youtube": youtube_video,
|
||||
"download": youtube_video,
|
||||
"output_options": output_options,
|
||||
"nfo_tags": {"tags": {"key-3": "this-preset"}},
|
||||
},
|
||||
|
|
@ -97,7 +97,7 @@ class TestPreset:
|
|||
config=config_file,
|
||||
name="test",
|
||||
value={
|
||||
"youtube": youtube_video,
|
||||
"download": youtube_video,
|
||||
"output_options": dict(
|
||||
output_options,
|
||||
**{"maintain_download_archive": True, "keep_files_after": "today-{ttl}"},
|
||||
|
|
@ -122,7 +122,7 @@ class TestPreset:
|
|||
name="test",
|
||||
value={
|
||||
"preset": parent_preset,
|
||||
"youtube": youtube_video,
|
||||
"download": youtube_video,
|
||||
"output_options": output_options,
|
||||
},
|
||||
)
|
||||
|
|
@ -138,7 +138,7 @@ class TestPreset:
|
|||
config=config_file,
|
||||
name="test",
|
||||
value={
|
||||
"youtube": youtube_video,
|
||||
"download": youtube_video,
|
||||
"output_options": {"output_directory": "dir", "file_name": "{dne_var}"},
|
||||
},
|
||||
)
|
||||
|
|
@ -154,7 +154,7 @@ class TestPreset:
|
|||
config=config_file,
|
||||
name="test",
|
||||
value={
|
||||
"youtube": youtube_video,
|
||||
"download": youtube_video,
|
||||
"output_options": {"output_directory": "{dne_var}", "file_name": "file"},
|
||||
},
|
||||
)
|
||||
|
|
@ -170,7 +170,7 @@ class TestPreset:
|
|||
config=config_file,
|
||||
name="test",
|
||||
value={
|
||||
"youtube": youtube_video,
|
||||
"download": youtube_video,
|
||||
"output_options": {"output_directory": "dir", "file_name": "file"},
|
||||
"nfo_tags": {
|
||||
"nfo_name": "the nfo name",
|
||||
|
|
@ -191,7 +191,7 @@ class TestPreset:
|
|||
config=config_file,
|
||||
name="test",
|
||||
value={
|
||||
"youtube": youtube_video,
|
||||
"download": youtube_video,
|
||||
"output_options": {"output_directory": "dir", "file_name": "file"},
|
||||
"output_directory_nfo_tags": {
|
||||
"nfo_name": "the nfo name",
|
||||
|
|
|
|||
|
|
@ -12,7 +12,7 @@ from ytdl_sub.subscriptions.subscription import Subscription
|
|||
def preset_file(youtube_video: Dict, output_options: Dict) -> Dict:
|
||||
return {
|
||||
"__preset__": {
|
||||
"youtube": youtube_video,
|
||||
"download": youtube_video,
|
||||
"output_options": output_options,
|
||||
"nfo_tags": {
|
||||
"tags": {"key-3": "file_preset"},
|
||||
|
|
|
|||
Loading…
Reference in a new issue