danvics/ytdl-sub
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

[FEATURE] download_individually for Youtube channel (#101)

Browse source
This commit is contained in:
Jesse Bannon 2022-07-17 18:17:00 -07:00 committed by GitHub
parent 2b0e7e0e4f
commit 8e35c1674b
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
4 changed files with 125 additions and 107 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
src/ytdl_sub/downloaders/downloader.py
View file

@ -6,6 +6,7 @@ from contextlib import contextmanager
from pathlib import Path
from typing import Dict
from typing import Generic
from typing import Iterable
from typing import List
from typing import Optional
from typing import Tuple
@ -199,7 +200,9 @@ class Downloader(DownloadArchiver, Generic[DownloaderOptionsT, DownloaderEntryT]
return self._get_entry_dicts_from_info_json_files()
@abc.abstractmethod
def download(self) -> List[DownloaderEntryT] | List[Tuple[DownloaderEntryT, FileMetadata]]:
def download(
self,
) -> Iterable[DownloaderEntryT] | Iterable[Tuple[DownloaderEntryT, FileMetadata]]:
"""The function to perform the download of all media entries"""
def post_download(self, overrides: Overrides):

61
src/ytdl_sub/downloaders/youtube/channel.py
View file

@ -1,5 +1,6 @@
from pathlib import Path
from typing import Dict
from typing import Generator
from typing import List
from typing import Optional
@ -14,6 +15,7 @@ from ytdl_sub.utils.thumbnail import convert_url_thumbnail
from ytdl_sub.validators.date_range_validator import DateRangeValidator
from ytdl_sub.validators.string_formatter_validators import OverridesStringFormatterValidator
from ytdl_sub.validators.url_validator import YoutubeChannelUrlValidator
from ytdl_sub.validators.validators import BoolValidator
from ytdl_sub.ytdl_additions.enhanced_download_archive import EnhancedDownloadArchive
logger = Logger.get()
@ -36,12 +38,19 @@ class YoutubeChannelDownloaderOptions(YoutubeDownloaderOptions, DateRangeValidat
# optional
channel_avatar_path: "poster.jpg"
channel_banner_path: "fanart.jpg"
download_individually: True
before: "now"
after: "today-2weeks"
"""
_required_keys = {"channel_url"}
_optional_keys = {"before", "after", "channel_avatar_path", "channel_banner_path"}
_optional_keys = {
"before",
"after",
"channel_avatar_path",
"channel_banner_path",
"download_individually",
}
def __init__(self, name, value):
YoutubeDownloaderOptions.__init__(self, name, value)
@ -55,6 +64,9 @@ class YoutubeChannelDownloaderOptions(YoutubeDownloaderOptions, DateRangeValidat
self._channel_banner_path = self._validate_key_if_present(
"channel_banner_path", OverridesStringFormatterValidator
)
self._download_individually = self._validate_key_if_present(
"download_individually", BoolValidator, default=True
)
@property
def channel_url(self) -> str:
@ -79,6 +91,15 @@ class YoutubeChannelDownloaderOptions(YoutubeDownloaderOptions, DateRangeValidat
"""
return self._channel_banner_path
@property
def download_individually(self) -> Optional[bool]:
"""
Optional. Downloads files from the channel individually instead of in bulk. Setting to True
is safer when downloading large amounts of videos in case an error occurs. Downloading by
bulk (by setting to False) can increase speeds. Defaults to True.
"""
return self._download_individually.value
class YoutubeChannelDownloader(YoutubeDownloader[YoutubeChannelDownloaderOptions, YoutubeVideo]):
downloader_options_type = YoutubeChannelDownloaderOptions
@ -120,13 +141,25 @@ class YoutubeChannelDownloader(YoutubeDownloader[YoutubeChannelDownloaderOptions
)
self.channel: Optional[YoutubeChannel] = None
def download(self) -> List[YoutubeVideo]:
def _get_channel_from_entry_dicts(self, entry_dicts: List[Dict]) -> YoutubeChannel:
channel_entry_dict = [
entry_dict for entry_dict in entry_dicts if entry_dict.get("extractor") == "youtube:tab"
][0]
return YoutubeChannel(
entry_dict=channel_entry_dict, working_directory=self.working_directory
)
def download(self) -> Generator[YoutubeVideo, None, None]:
"""
Downloads all videos from a channel
"""
channel_videos: List[YoutubeVideo] = []
ytdl_options_overrides = {}
# If downloading individually, dry-run the entire channel download first, this will get the
# videos that will be downloaded. Afterwards, download each video one-by-one
if self.download_options.download_individually:
ytdl_options_overrides["skip_download"] = True
# If a date range is specified when download a YT channel, add it into the ytdl options
source_date_range = self.download_options.get_date_range()
if source_date_range:
@ -135,18 +168,22 @@ class YoutubeChannelDownloader(YoutubeDownloader[YoutubeChannelDownloaderOptions
entry_dicts = self.extract_info_via_info_json(
ytdl_options_overrides=ytdl_options_overrides, url=self.download_options.channel_url
)
self.channel = self._get_channel_from_entry_dicts(entry_dicts=entry_dicts)
# If downloading individually, remove the skip_download to actually download the video
if self.download_options.download_individually:
del ytdl_options_overrides["skip_download"]
for entry_dict in entry_dicts:
if entry_dict.get("extractor") == "youtube":
channel_videos.append(
YoutubeVideo(entry_dict=entry_dict, working_directory=self.working_directory)
)
if entry_dict.get("extractor") == "youtube:tab":
self.channel = YoutubeChannel(
entry_dict=entry_dict, working_directory=self.working_directory
)
return channel_videos
# Only do the individual download if it is not dry-run and downloading individually
if not self.is_dry_run and self.download_options.download_individually:
ytdl_options_overrides["playlist_items"] = str(entry_dict.get("playlist_index"))
_ = self.extract_info(
ytdl_options_overrides=ytdl_options_overrides,
url=self.download_options.channel_url,
)
yield YoutubeVideo(entry_dict=entry_dict, working_directory=self.working_directory)
def _download_thumbnail(
self,

4
src/ytdl_sub/subscriptions/subscription.py
View file

@ -292,6 +292,10 @@ class Subscription:
entry=entry, entry_metadata=entry_metadata
)
# Re-save the download archive after each entry is moved to the output directory
if self.maintain_download_archive:
self._enhanced_download_archive.save_download_mappings()
downloader.post_download(overrides=self.overrides)
for plugin in plugins:
plugin.post_process_subscription()

162
tests/e2e/youtube/test_channel_as_kodi_tv_show.py
View file

@ -1,4 +1,6 @@
import copy
from pathlib import Path
from typing import Dict
import mergedeep
import pytest
@ -8,28 +10,16 @@ from e2e.expected_download import ExpectedDownloads
from e2e.expected_transaction_log import assert_transaction_log_matches
import ytdl_sub.downloaders.downloader
from ytdl_sub.config.config_file import ConfigFile
from ytdl_sub.config.preset import Preset
from ytdl_sub.subscriptions.subscription import Subscription
@pytest.fixture
def config_path():
return "examples/kodi_tv_shows_config.yaml"
@pytest.fixture
def subscription_name():
return "pz"
@pytest.fixture
def config(config_path):
return ConfigFile.from_file_path(config_path=config_path)
@pytest.fixture
def subscription_dict(output_directory):
def channel_preset_dict(output_directory):
return {
"preset": "yt_channel_as_tv",
"youtube": {"channel_url": "https://youtube.com/channel/UCcRSMoQqXc_JrBZRHDFGbqA"},
@ -45,24 +35,20 @@ def subscription_dict(output_directory):
}
@pytest.fixture
def channel_subscription_generator(channel_as_tv_show_config, subscription_name):
def _channel_subscription_generator(preset_dict: Dict):
return Subscription.from_dict(
config=channel_as_tv_show_config, preset_name=subscription_name, preset_dict=preset_dict
)
return _channel_subscription_generator
####################################################################################################
# FULL CHANNEL FIXTURES
@pytest.fixture
def full_channel_subscription(config, subscription_name, subscription_dict):
full_channel_preset = Preset.from_dict(
config=config,
preset_name=subscription_name,
preset_dict=subscription_dict,
)
return Subscription.from_preset(
preset=full_channel_preset,
config=config,
)
@pytest.fixture
def expected_full_channel_download():
# turn off black formatter here for readability
@ -133,11 +119,12 @@ def expected_full_channel_download():
####################################################################################################
# RECENT CHANNEL FIXTURES
@pytest.fixture
def recent_channel_subscription_dict(subscription_dict):
def recent_channel_preset_dict(channel_preset_dict):
# TODO: remove this hack by using a different channel
del subscription_dict["ytdl_options"]["break_on_reject"]
channel_preset_dict = copy.deepcopy(channel_preset_dict)
del channel_preset_dict["ytdl_options"]["break_on_reject"]
return mergedeep.merge(
subscription_dict,
channel_preset_dict,
{
"preset": "yt_channel_as_tv__recent",
"youtube": {"after": "20150101"},
@ -145,20 +132,6 @@ def recent_channel_subscription_dict(subscription_dict):
)
@pytest.fixture
def recent_channel_subscription(config, subscription_name, recent_channel_subscription_dict):
recent_channel_preset = Preset.from_dict(
config=config,
preset_name=subscription_name,
preset_dict=recent_channel_subscription_dict,
)
return Subscription.from_preset(
preset=recent_channel_preset,
config=config,
)
@pytest.fixture
def expected_recent_channel_download():
# turn off black formatter here for readability
@ -188,33 +161,6 @@ def expected_recent_channel_download():
####################################################################################################
# RECENT CHANNEL FIXTURES -- NO VIDS IN RANGE
@pytest.fixture
def recent_channel_no_vids_in_range_subscription_dict(subscription_dict):
# TODO: remove this hack by using a different channel
del subscription_dict["ytdl_options"]["break_on_reject"]
return mergedeep.merge(
subscription_dict,
{
"preset": "yt_channel_as_tv__recent",
"youtube": {"after": "20880101"},
},
)
@pytest.fixture
def recent_channel_no_vids_in_range_subscription(
config, subscription_name, recent_channel_no_vids_in_range_subscription_dict
):
recent_channel_preset = Preset.from_dict(
config=config,
preset_name=subscription_name,
preset_dict=recent_channel_no_vids_in_range_subscription_dict,
)
return Subscription.from_preset(
preset=recent_channel_preset,
config=config,
)
@pytest.fixture
@ -238,9 +184,10 @@ def expected_recent_channel_no_vids_in_range_download():
####################################################################################################
# ROLLING RECENT CHANNEL FIXTURES
@pytest.fixture
def rolling_recent_channel_subscription_dict(recent_channel_subscription_dict):
def rolling_recent_channel_preset_dict(recent_channel_preset_dict):
recent_channel_preset_dict = copy.deepcopy(recent_channel_preset_dict)
return mergedeep.merge(
recent_channel_subscription_dict,
recent_channel_preset_dict,
{
"preset": "yt_channel_as_tv__only_recent",
"output_options": {"keep_files_after": "20181101"},
@ -248,22 +195,6 @@ def rolling_recent_channel_subscription_dict(recent_channel_subscription_dict):
)
@pytest.fixture
def rolling_recent_channel_subscription(
config, subscription_name, rolling_recent_channel_subscription_dict
):
rolling_recent_channel_preset = Preset.from_dict(
config=config,
preset_name=subscription_name,
preset_dict=rolling_recent_channel_subscription_dict,
)
return Subscription.from_preset(
preset=rolling_recent_channel_preset,
config=config,
)
@pytest.fixture
def expected_rolling_recent_channel_download():
# turn off black formatter here for readability
@ -294,9 +225,19 @@ class TestChannelAsKodiTvShow:
"""
@pytest.mark.parametrize("dry_run", [True, False])
@pytest.mark.parametrize("download_individually", [True, False])
def test_full_channel_download(
self, full_channel_subscription, expected_full_channel_download, output_directory, dry_run
self,
channel_subscription_generator,
channel_preset_dict,
expected_full_channel_download,
output_directory,
dry_run,
download_individually,
):
channel_preset_dict["youtube"]["download_individually"] = download_individually
full_channel_subscription = channel_subscription_generator(preset_dict=channel_preset_dict)
transaction_log = full_channel_subscription.download(dry_run=dry_run)
assert_transaction_log_matches(
output_directory=output_directory,
@ -307,13 +248,22 @@ class TestChannelAsKodiTvShow:
expected_full_channel_download.assert_files_exist(relative_directory=output_directory)
@pytest.mark.parametrize("dry_run", [True, False])
@pytest.mark.parametrize("download_individually", [True, False])
def test_recent_channel_download(
self,
recent_channel_subscription,
channel_subscription_generator,
recent_channel_preset_dict,
expected_recent_channel_download,
output_directory,
dry_run,
download_individually,
):
recent_channel_preset_dict["youtube"]["download_individually"] = download_individually
recent_channel_subscription = channel_subscription_generator(
preset_dict=recent_channel_preset_dict
)
transaction_log = recent_channel_subscription.download(dry_run=dry_run)
assert_transaction_log_matches(
output_directory=output_directory,
@ -341,13 +291,22 @@ class TestChannelAsKodiTvShow:
)
@pytest.mark.parametrize("dry_run", [True, False])
@pytest.mark.parametrize("download_individually", [True, False])
def test_recent_channel_download__no_vids_in_range(
self,
recent_channel_no_vids_in_range_subscription,
channel_subscription_generator,
recent_channel_preset_dict,
expected_recent_channel_no_vids_in_range_download,
output_directory,
dry_run,
download_individually,
):
recent_channel_preset_dict["youtube"]["download_individually"] = download_individually
recent_channel_preset_dict["youtube"]["after"] = "21000101"
recent_channel_no_vids_in_range_subscription = channel_subscription_generator(
preset_dict=recent_channel_preset_dict
)
# Run twice, ensure nothing changes between runs
for _ in range(2):
transaction_log = recent_channel_no_vids_in_range_subscription.download(dry_run=dry_run)
@ -362,15 +321,30 @@ class TestChannelAsKodiTvShow:
)
@pytest.mark.parametrize("dry_run", [True, False])
@pytest.mark.parametrize("download_individually", [True, False])
def test_rolling_recent_channel_download(
self,
recent_channel_subscription,
rolling_recent_channel_subscription,
channel_subscription_generator,
recent_channel_preset_dict,
rolling_recent_channel_preset_dict,
expected_recent_channel_download,
expected_rolling_recent_channel_download,
output_directory,
dry_run,
download_individually,
):
recent_channel_preset_dict["youtube"]["download_individually"] = download_individually
rolling_recent_channel_preset_dict["youtube"][
"download_individually"
] = download_individually
recent_channel_subscription = channel_subscription_generator(
preset_dict=recent_channel_preset_dict
)
rolling_recent_channel_subscription = channel_subscription_generator(
preset_dict=rolling_recent_channel_preset_dict
)
# First, download recent vids. Always download since we want to test dry-run
# on the rolling recent portion.
with assert_debug_log(