danvics/ytdl-sub
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

ytdl_options defaults for each download strategy (#72)

Browse source 
* `ytdl_options` defaults for each download strategy

* fix docker, fix defaults getting added, add debugger test

* test playlist as well

* fix with hack

* fix full channel test

* existing
This commit is contained in:
Jesse Bannon 2022-06-14 00:03:01 -07:00 committed by GitHub
parent 3d7595cda9
commit ee6706334f
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
12 changed files with 199 additions and 86 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
docker/Dockerfile
View file

@ -14,8 +14,8 @@ RUN apk update --no-cache && \
python3 \
py3-pip && \
apk add --repository=http://dl-3.alpinelinux.org/alpine/edge/main/ \
python3=3.10.4-r0 \
py3-setuptools=59.4.0-r0 && \
python3=~3.10 \
py3-setuptools && \
apk add --repository=http://dl-3.alpinelinux.org/alpine/edge/community/ \
py3-pip && \
mkdir -p /config && \

6
docker/root/defaults/config.yaml
View file

@ -13,9 +13,6 @@ presets:
youtube:
download_strategy: "video"
ytdl_options:
ignoreerrors: True
output_options:
output_directory: "{music_video_directory}"
file_name: "{music_video_name}.{ext}"
@ -45,6 +42,3 @@ presets:
output_options:
maintain_download_archive: True
ytdl_options:
break_on_existing: True

8
docs/config.rst
View file

@ -51,6 +51,8 @@ _______
:inherited-members:
:exclude-members: get_date_range
.. autofunction:: ytdl_sub.downloaders.youtube_downloader.YoutubeChannelDownloader.ytdl_option_defaults()
-------------------------------------------------------------------------------
.. _YouTube Playlist:
@ -62,6 +64,8 @@ ________
:member-order: bysource
:inherited-members:
.. autofunction:: ytdl_sub.downloaders.youtube_downloader.YoutubePlaylistDownloader.ytdl_option_defaults()
-------------------------------------------------------------------------------
video
@ -71,6 +75,8 @@ _____
:member-order: bysource
:inherited-members:
.. autofunction:: ytdl_sub.downloaders.youtube_downloader.YoutubeVideoDownloader.ytdl_option_defaults()
-------------------------------------------------------------------------------
soundcloud
@ -86,6 +92,8 @@ __________________
:member-order: bysource
:inherited-members:
.. autofunction:: ytdl_sub.downloaders.soundcloud_downloader.SoundcloudAlbumsAndSinglesDownloader.ytdl_option_defaults()
-------------------------------------------------------------------------------
output_options

17
examples/kodi_music_videos_config.yaml
View file

@ -23,11 +23,13 @@ presets:
youtube:
download_strategy: "video"
# For advanced YTDL users only.
# It is wise to leave ignoreerrors=True to avoid errors for things
# like age-restricted videos in case you have not set your cookie.
ytdl_options:
ignoreerrors: True
# For advanced YTDL users only; any YTDL parameter can be set here.
# To download age-restricted videos, you will need to set your cookie
# file here as a ytdl parameter. For more info, see
# https://ytdl-sub.readthedocs.io/en/latest/faq.html#download-age-restricted-youtube-videos
#
# ytdl_options:
# cookiefile: "path/to/cookie_file.txt
# For each video downloaded, set the file and thumbnail name here.
# We set both with {music_video_name}, which is a variable we define in
@ -70,8 +72,3 @@ presets:
output_options:
maintain_download_archive: True
# Setting break_on_existing=True is also a good thing anytime you are using
# a download archive, because it will tell the downloader to stop trying
# to download videos that have already been downloaded.
ytdl_options:
break_on_existing: True

52
examples/kodi_tv_shows_config.yaml
View file

@ -22,10 +22,8 @@ configuration:
presets:
###############################################################################
# LEVEL 1 - FULL ARCHIVE
#
# We will call this preset `yt_channel_as_tv`, and it will download every single video in
# a YouTube channel.
# FULL ARCHIVE
# This preset will download every single video in a YouTube channel.
yt_channel_as_tv:
# YouTube channels are our source/download strategy
# Use the channel avatar and banner images for Kodi
@ -34,11 +32,13 @@ presets:
channel_avatar_path: "poster.jpg"
channel_banner_path: "fanart.jpg"
# For advanced YTDL users only.
# It is wise to leave ignoreerrors=True to avoid errors for things
# like age-restricted videos in case you have not set your cookie.
ytdl_options:
ignoreerrors: True
# For advanced YTDL users only; any YTDL parameter can be set here.
# To download age-restricted videos, you will need to set your cookie
# file here as a ytdl parameter. For more info, see
# https://ytdl-sub.readthedocs.io/en/latest/faq.html#download-age-restricted-youtube-videos
#
# ytdl_options:
# cookiefile: "path/to/cookie_file.txt
# For each video downloaded, set the file and thumbnail name here.
# We set both with {episode_name}, which is a variable we define in
@ -85,11 +85,9 @@ presets:
episode_name: "Season {upload_year}/s{upload_year}.e{upload_month_padded}{upload_day_padded} - {title_sanitized}"
###############################################################################
# LEVEL 2 - RECENT ARCHIVE
#
# It is not always ideal to go full data-hoarder on a channel's videos.
# This example shows how you can only download the last 14 days-worth
# of videos on each download invocation.
# RECENT ARCHIVE
# This preset shows how you can download just the last 14 days-worth of videos
# on each download invocation.
yt_channel_as_tv__recent:
# `preset` can be set to any other preset in the config, and will inherit all its defined fields.
# This helps reduce copy-paste in the config.yaml
@ -100,30 +98,10 @@ presets:
youtube:
after: "today-2weeks"
# This is getting into YTDL voodoo. By default, YTDL will try to
# download all channel videos beginning with the most recent one.
# By setting break_on_reject, we will break this full-download on
# the first video that gets rejected. Since we have youtube.after
# defined, all videos after today-14days will be rejected. Therefore,
# the first video that is out of range that it tries to download, it
# will stop there, and save a significant amount of time.
#
# Similar to break_on_reject, but instead, breaks if a video has
# already been downloaded. If you were to perform a download on this
# subscription one-after-the-other, the second invocation would stop
# after looking at the first (most recent) video since it would exist
# in the download archive.
ytdl_options:
break_on_reject: True
break_on_existing: True
###############################################################################
# LEVEL 3 - ROLLING ARCHIVE
# If you put the recent archive example in a cron job, then in a year or so
# you will basically be datahoarding that channel unless you manually delete
# old videos. This example shows how to only keep the last 14-days worth of videos,
# and delete the rest.
# ROLLING ARCHIVE
# This example shows how to only keep the last 14-days worth of videos, and
# delete the rest.
yt_channel_as_tv__only_recent:
# Reuse `yt_channel_as_tv__recent` to only download the last 2 weeks' of videos
preset: "yt_channel_as_tv__recent"

10
examples/soundcloud_discography_config.yaml
View file

@ -26,11 +26,11 @@ presets:
download_strategy: "albums_and_singles"
skip_premiere_tracks: True
# For advanced YTDL users only.
# This sets the download format to grab the best mp3 available.
# Other formats should work but have not been tested.
ytdl_options:
format: 'bestaudio[ext=mp3]'
# For advanced YTDL users only; any YTDL parameter can be set here.
# You can set the type of format you prefer YTDL to download here.
# By default, soundcloud will download the best mp3.
# ytdl_options:
# format: 'bestaudio[ext=mp3]'
# For each song downloaded, set the file and thumbnail name here.
# The output directory stores all artists in a shared {music_directory}.

19
src/ytdl_sub/downloaders/downloader.py
View file

@ -21,6 +21,8 @@ from ytdl_sub.entries.entry import Entry
from ytdl_sub.utils.logger import Logger
from ytdl_sub.validators.strict_dict_validator import StrictDictValidator
logger = Logger.get(name="downloader")
class DownloaderValidator(StrictDictValidator, ABC):
"""
@ -49,8 +51,13 @@ class Downloader(Generic[DownloaderOptionsT, DownloaderEntryT], ABC):
@classmethod
def ytdl_option_defaults(cls) -> Dict:
"""Downloader defaults that can be overwritten from user input"""
return {}
"""
.. code-block:: yaml
ytdl_options:
ignoreerrors: True # ignore errors like hidden videos, age restriction, etc
"""
return {"ignoreerrors": True}
@classmethod
def _configure_ytdl_options(
@ -101,7 +108,7 @@ class Downloader(Generic[DownloaderOptionsT, DownloaderEntryT], ABC):
"""
self.working_directory = working_directory
self.download_options = download_options
self.ytdl_options = Downloader._configure_ytdl_options(
self.ytdl_options = self._configure_ytdl_options(
ytdl_options=ytdl_options,
working_directory=self.working_directory,
download_archive_file_name=download_archive_file_name,
@ -180,8 +187,10 @@ class Downloader(Generic[DownloaderOptionsT, DownloaderEntryT], ABC):
try:
_ = self.extract_info(ytdl_options_overrides=ytdl_options_overrides, **kwargs)
except (RejectedVideoReached, ExistingVideoReached):
pass
except RejectedVideoReached:
logger.debug("RejectedVideoReached, stopping additional downloads")
except ExistingVideoReached:
logger.debug("ExistingVideoReached, stopping additional downloads")
return self._get_entry_dicts_from_info_json_files()

25
src/ytdl_sub/downloaders/soundcloud_downloader.py
View file

@ -53,13 +53,6 @@ class SoundcloudDownloader(
downloader_entry_type = SoundcloudTrack
@classmethod
def ytdl_option_defaults(cls) -> Dict:
"""Returns default format to be best mp3"""
return {
"format": "bestaudio[ext=mp3]",
}
@classmethod
def artist_albums_url(cls, artist_url: str) -> str:
"""
@ -124,6 +117,24 @@ class SoundcloudAlbumsAndSinglesDownloader(
):
downloader_options_type = SoundcloudAlbumsAndSinglesDownloadOptions
@classmethod
def ytdl_option_defaults(cls) -> Dict:
"""
Default `ytdl_options`_ for ``albums_and_singles``
.. code-block:: yaml
ytdl_options:
ignoreerrors: True # ignore errors like hidden videos, age restriction, etc
format: "bestaudio[ext=mp3]" # download format the best possible mp3
"""
return dict(
super().ytdl_option_defaults(),
**{
"format": "bestaudio[ext=mp3]",
},
)
def _get_albums(self, entry_dicts: List[Dict]) -> List[SoundcloudAlbum]:
"""
Parameters

57
src/ytdl_sub/downloaders/youtube_downloader.py
View file

@ -95,6 +95,21 @@ class YoutubeVideoDownloader(YoutubeDownloader[YoutubeVideoDownloaderOptions, Yo
downloader_options_type = YoutubeVideoDownloaderOptions
downloader_entry_type = YoutubeVideo
@classmethod
def ytdl_option_defaults(cls) -> Dict:
"""
Default `ytdl_options`_ for ``video``
.. code-block:: yaml
ytdl_options:
ignoreerrors: True # ignore errors like hidden videos, age restriction, etc
"""
return dict(
super().ytdl_option_defaults(),
**{"break_on_existing": True},
)
def download(self) -> List[YoutubeVideo]:
"""Download a single Youtube video"""
entry_dict = self.extract_info(url=self.download_options.video_url)
@ -144,6 +159,25 @@ class YoutubePlaylistDownloader(
downloader_options_type = YoutubePlaylistDownloaderOptions
downloader_entry_type = YoutubePlaylistVideo
# pylint: disable=line-too-long
@classmethod
def ytdl_option_defaults(cls) -> Dict:
"""
Default `ytdl_options`_ for ``playlist``
.. code-block:: yaml
ytdl_options:
ignoreerrors: True # ignore errors like hidden videos, age restriction, etc
break_on_existing: True # stop downloads (newest to oldest) if a video is already downloaded
"""
return dict(
super().ytdl_option_defaults(),
**{"break_on_existing": True},
)
# pylint: enable=line-too-long
def download(self) -> List[YoutubePlaylistVideo]:
"""
Downloads all videos in a Youtube playlist
@ -231,6 +265,29 @@ class YoutubeChannelDownloader(YoutubeDownloader[YoutubeChannelDownloaderOptions
downloader_options_type = YoutubeChannelDownloaderOptions
downloader_entry_type = YoutubeVideo
# pylint: disable=line-too-long
@classmethod
def ytdl_option_defaults(cls) -> Dict:
"""
Default `ytdl_options`_ for ``channel``
.. code-block:: yaml
ytdl_options:
ignoreerrors: True # ignore errors like hidden videos, age restriction, etc
break_on_existing: True # stop downloads (newest to oldest) if a video is already downloaded
break_on_reject: True # stops downloads if the video's upload date is out of the specified 'before'/'after' range
"""
return dict(
super().ytdl_option_defaults(),
**{
"break_on_existing": True,
"break_on_reject": True,
},
)
# pylint: enable=line-too-long
def __init__(
self,
working_directory: str,

28
tests/conftest.py Normal file
View file

@ -0,0 +1,28 @@
import contextlib
import logging
from typing import Callable
from unittest.mock import MagicMock
from unittest.mock import patch
from ytdl_sub.utils.logger import Logger
@contextlib.contextmanager
def assert_debug_log(logger: logging.Logger, expected_message: str):
"""
Patches any function, but calls the original function.
Intended to see if the particular function is called.
"""
debug_logger = Logger.get()
def _wrapped_debug(*args, **kwargs):
debug_logger.info(*args, **kwargs)
with patch.object(logger, "debug", wraps=_wrapped_debug) as patched_debug:
yield
for call_args in patched_debug.call_args_list:
if expected_message in call_args.args[0]:
return
assert False, f"{expected_message} was not found in a logger.debug call"

49
tests/e2e/youtube/test_channel_as_kodi_tv_show.py
View file

@ -2,8 +2,10 @@ from pathlib import Path
import mergedeep
import pytest
from conftest import assert_debug_log
from e2e.expected_download import ExpectedDownload
import ytdl_sub.downloaders.downloader
from ytdl_sub.config.config_file import ConfigFile
from ytdl_sub.config.preset import Preset
from ytdl_sub.subscriptions.subscription import Subscription
@ -35,6 +37,7 @@ def subscription_dict(output_directory, subscription_name):
"ytdl_options": {
"format": "worst[ext=mp4]",
"max_views": 100000, # do not download the popular PJ concert
"break_on_reject": False, # do not break from max views
},
"overrides": {"tv_show_name": "Project / Zombie"},
}
@ -129,6 +132,8 @@ def expected_full_channel_download():
# RECENT CHANNEL FIXTURES
@pytest.fixture
def recent_channel_subscription_dict(subscription_dict):
# TODO: remove this hack by using a different channel
del subscription_dict["ytdl_options"]["break_on_reject"]
return mergedeep.merge(
subscription_dict,
{
@ -250,9 +255,12 @@ class TestChannelAsKodiTvShow:
expected_recent_channel_download.assert_files_exist(relative_directory=output_directory)
# try downloading again, ensure nothing more was downloaded
# TODO: add patch around the output of download to see what entry dicts were returned
recent_channel_subscription.download()
expected_recent_channel_download.assert_files_exist(relative_directory=output_directory)
with assert_debug_log(
logger=ytdl_sub.downloaders.downloader.logger,
expected_message="ExistingVideoReached, stopping additional downloads",
):
recent_channel_subscription.download()
expected_recent_channel_download.assert_files_exist(relative_directory=output_directory)
def test_rolling_recent_channel_download(
self,
@ -263,18 +271,31 @@ class TestChannelAsKodiTvShow:
output_directory,
):
# First, download recent vids
recent_channel_subscription.download()
expected_recent_channel_download.assert_files_exist(relative_directory=output_directory)
with assert_debug_log(
logger=ytdl_sub.downloaders.downloader.logger,
expected_message="RejectedVideoReached, stopping additional downloads",
):
recent_channel_subscription.download()
expected_recent_channel_download.assert_files_exist(relative_directory=output_directory)
# Then, download the rolling recent vids subscription. This should remove one of the
# two videos
rolling_recent_channel_subscription.download()
expected_rolling_recent_channel_download.assert_files_exist(
relative_directory=output_directory
)
with assert_debug_log(
logger=ytdl_sub.downloaders.downloader.logger,
expected_message="ExistingVideoReached, stopping additional downloads",
):
rolling_recent_channel_subscription.download()
expected_rolling_recent_channel_download.assert_files_exist(
relative_directory=output_directory
)
# Invoke the rolling download again, ensure nothing has changed
rolling_recent_channel_subscription.download()
expected_rolling_recent_channel_download.assert_files_exist(
relative_directory=output_directory
)
# Invoke the rolling download again, ensure downloading stopped early from it already
# existing
with assert_debug_log(
logger=ytdl_sub.downloaders.downloader.logger,
expected_message="ExistingVideoReached, stopping additional downloads",
):
rolling_recent_channel_subscription.download()
expected_rolling_recent_channel_download.assert_files_exist(
relative_directory=output_directory
)

10
tests/e2e/youtube/test_playlist_as_kodi_music_videos.py
View file

@ -2,8 +2,10 @@ from pathlib import Path
import mergedeep
import pytest
from conftest import assert_debug_log
from e2e.expected_download import ExpectedDownload
import ytdl_sub.downloaders.downloader
from ytdl_sub.config.config_file import ConfigFile
from ytdl_sub.config.preset import Preset
from ytdl_sub.subscriptions.subscription import Subscription
@ -138,6 +140,14 @@ class TestPlaylistAsKodiMusicVideo:
playlist_subscription.download()
expected_playlist_download.assert_files_exist(relative_directory=output_directory)
# After the playlist is downloaded, ensure another invocation will hit ExistingVideoReached
with assert_debug_log(
logger=ytdl_sub.downloaders.downloader.logger,
expected_message="ExistingVideoReached, stopping additional downloads",
):
playlist_subscription.download()
expected_playlist_download.assert_files_exist(relative_directory=output_directory)
def test_single_video_download(
self, single_video_subscription, expected_single_video_download, output_directory
):