danvics/ytdl-sub
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

moved tests

Browse source
This commit is contained in:
Jesse Bannon 2022-08-28 10:52:26 -07:00
parent 21767e21ad
commit ef085d87ff
10 changed files with 226 additions and 212 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
src/ytdl_sub/config/preset_class_mappings.py
View file

@ -11,6 +11,7 @@ from ytdl_sub.downloaders.youtube.split_video import YoutubeSplitVideoDownloader
from ytdl_sub.downloaders.youtube.video import YoutubeVideoDownloader
from ytdl_sub.plugins.audio_extract import AudioExtractPlugin
from ytdl_sub.plugins.chapters import ChaptersPlugin
from ytdl_sub.plugins.date_range import DateRangePlugin
from ytdl_sub.plugins.music_tags import MusicTagsPlugin
from ytdl_sub.plugins.nfo_tags import NfoTagsPlugin
from ytdl_sub.plugins.output_directory_nfo_tags import OutputDirectoryNfoTagsPlugin
@ -111,6 +112,7 @@ class PluginMapping:
_MAPPING: Dict[str, Type[Plugin]] = {
"audio_extract": AudioExtractPlugin,
"date_range": DateRangePlugin,
"music_tags": MusicTagsPlugin,
"video_tags": VideoTagsPlugin,
"nfo_tags": NfoTagsPlugin,

16
src/ytdl_sub/subscriptions/subscription_ytdl_options.py
View file

@ -10,6 +10,7 @@ from ytdl_sub.downloaders.downloader import Downloader
from ytdl_sub.downloaders.ytdl_options_builder import YTDLOptionsBuilder
from ytdl_sub.plugins.audio_extract import AudioExtractPlugin
from ytdl_sub.plugins.chapters import ChaptersPlugin
from ytdl_sub.plugins.date_range import DateRangePlugin
from ytdl_sub.plugins.plugin import Plugin
from ytdl_sub.plugins.subtitles import SubtitlesPlugin
from ytdl_sub.ytdl_additions.enhanced_download_archive import EnhancedDownloadArchive
@ -111,6 +112,13 @@ class SubscriptionYTDLOptions:
return chapters_plugin.ytdl_options()
@property
def _date_range_options(self) -> Dict:
if not (date_range_plugin := self._get_plugin(DateRangePlugin)):
return {}
return date_range_plugin.ytdl_options()
@property
def _user_ytdl_options(self) -> Dict:
return self._preset.ytdl_options.dict
@ -125,18 +133,20 @@ class SubscriptionYTDLOptions:
ytdl_options_builder = YTDLOptionsBuilder().add(self._global_options)
if self._dry_run:
ytdl_options_builder.add(
self._date_range_options,
self._subtitle_options,
self._chapter_options,
self._user_ytdl_options,
self._dry_run_options,
self._user_ytdl_options, # user ytdl options...
self._dry_run_options, # then dry-run
)
else:
ytdl_options_builder.add(
self._output_options,
self._date_range_options,
self._subtitle_options,
self._chapter_options,
self._audio_extract_options,
self._user_ytdl_options,
self._user_ytdl_options, # user ytdl options last
)
return ytdl_options_builder

201
tests/e2e/plugins/test_date_range.py Normal file
View file

@ -0,0 +1,201 @@
import copy
import mergedeep
import pytest
from conftest import assert_debug_log
from e2e.expected_download import assert_expected_downloads
from e2e.expected_transaction_log import assert_transaction_log_matches
import ytdl_sub.downloaders.downloader
from ytdl_sub.subscriptions.subscription import Subscription
@pytest.fixture
def recent_preset_dict(output_directory):
return {
"preset": "yt_channel_as_tv",
"youtube": {"channel_url": "https://youtube.com/channel/UCcRSMoQqXc_JrBZRHDFGbqA"},
"date_range": {"after": "20150101"},
# override the output directory with our fixture-generated dir
"output_options": {"output_directory": output_directory},
# download the worst format so it is fast
"ytdl_options": {
"format": "worst[ext=mp4]",
"max_views": 100000, # do not download the popular PJ concert
},
"subtitles": {
"subtitles_name": "{episode_name}.{lang}.{subtitles_ext}",
"allow_auto_generated_subtitles": True,
},
"overrides": {"tv_show_name": "Project / Zombie"},
}
@pytest.fixture
def rolling_recent_channel_preset_dict(recent_preset_dict):
preset_dict = copy.deepcopy(recent_preset_dict)
return mergedeep.merge(
preset_dict,
{
"output_options": {"keep_files_after": "20181101"},
},
)
class TestDateRange:
@pytest.mark.parametrize("dry_run", [True, False])
def test_recent_channel_download(
self,
recent_preset_dict,
channel_as_tv_show_config,
output_directory,
dry_run,
):
recent_channel_subscription = Subscription.from_dict(
config=channel_as_tv_show_config,
preset_name="recent",
preset_dict=recent_preset_dict,
)
transaction_log = recent_channel_subscription.download(dry_run=dry_run)
assert_transaction_log_matches(
output_directory=output_directory,
transaction_log=transaction_log,
transaction_log_summary_file_name="plugins/date_range/test_channel_recent.txt",
)
assert_expected_downloads(
output_directory=output_directory,
dry_run=dry_run,
expected_download_summary_file_name="plugins/date_range/test_channel_recent.json",
)
if not dry_run:
# try downloading again, ensure nothing more was downloaded
with assert_debug_log(
logger=ytdl_sub.downloaders.downloader.download_logger,
expected_message="ExistingVideoReached, stopping additional downloads",
):
transaction_log = recent_channel_subscription.download()
assert_transaction_log_matches(
output_directory=output_directory,
transaction_log=transaction_log,
transaction_log_summary_file_name=("plugins/date_range/no_downloads.txt"),
)
assert_expected_downloads(
output_directory=output_directory,
dry_run=dry_run,
expected_download_summary_file_name="plugins/date_range/test_channel_recent.json",
)
@pytest.mark.parametrize("dry_run", [True, False])
def test_recent_channel_download__no_vids_in_range(
self,
channel_as_tv_show_config,
recent_preset_dict,
output_directory,
dry_run,
):
recent_preset_dict["date_range"]["after"] = "21000101"
recent_channel_no_vids_in_range_subscription = Subscription.from_dict(
config=channel_as_tv_show_config,
preset_name="recent",
preset_dict=recent_preset_dict,
)
# Run twice, ensure nothing changes between runsyoutube
for _ in range(2):
transaction_log = recent_channel_no_vids_in_range_subscription.download(dry_run=dry_run)
assert_transaction_log_matches(
output_directory=output_directory,
transaction_log=transaction_log,
transaction_log_summary_file_name="plugins/date_range/no_downloads.txt",
)
assert_expected_downloads(
output_directory=output_directory,
dry_run=dry_run,
expected_download_summary_file_name="plugins/date_range/no_downloads.json",
)
@pytest.mark.parametrize("dry_run", [True, False])
def test_rolling_recent_channel_download(
self,
channel_as_tv_show_config,
recent_preset_dict,
rolling_recent_channel_preset_dict,
output_directory,
dry_run,
):
recent_channel_subscription = Subscription.from_dict(
config=channel_as_tv_show_config,
preset_name="recent",
preset_dict=recent_preset_dict,
)
rolling_recent_channel_subscription = Subscription.from_dict(
config=channel_as_tv_show_config,
preset_name="recent",
preset_dict=rolling_recent_channel_preset_dict,
)
# First, download recent vids. Always download since we want to test dry-run
# on the rolling recent portion.
with assert_debug_log(
logger=ytdl_sub.downloaders.downloader.download_logger,
expected_message="RejectedVideoReached, stopping additional downloads",
):
transaction_log = recent_channel_subscription.download(dry_run=False)
assert_transaction_log_matches(
output_directory=output_directory,
transaction_log=transaction_log,
transaction_log_summary_file_name="plugins/date_range/test_channel_recent.txt",
)
assert_expected_downloads(
output_directory=output_directory,
dry_run=False,
expected_download_summary_file_name="plugins/date_range/test_channel_recent.json",
)
# Then, download the rolling recent vids subscription. This should remove one of the
# two videos
with assert_debug_log(
logger=ytdl_sub.downloaders.downloader.download_logger,
expected_message="ExistingVideoReached, stopping additional downloads",
):
transaction_log = rolling_recent_channel_subscription.download(dry_run=dry_run)
expected_downloads_summary = (
"plugins/date_range/test_channel_recent.json"
if dry_run
else "plugins/date_range/test_channel_rolling_recent.json"
)
assert_transaction_log_matches(
output_directory=output_directory,
transaction_log=transaction_log,
transaction_log_summary_file_name="plugins/date_range/test_channel_rolling_recent.txt",
)
assert_expected_downloads(
output_directory=output_directory,
dry_run=False,
expected_download_summary_file_name=expected_downloads_summary,
)
# Invoke the rolling download again, ensure downloading stopped early from it already
# existing
if not dry_run:
with assert_debug_log(
logger=ytdl_sub.downloaders.downloader.download_logger,
expected_message="ExistingVideoReached, stopping additional downloads",
):
transaction_log = rolling_recent_channel_subscription.download()
assert_transaction_log_matches(
output_directory=output_directory,
transaction_log=transaction_log,
transaction_log_summary_file_name="plugins/date_range/no_downloads.txt",
)
assert_expected_downloads(
output_directory=output_directory,
dry_run=False,
expected_download_summary_file_name=expected_downloads_summary,
)

2
tests/e2e/resources/expected_downloads_summaries/youtube/test_channel_no_additional_downloads.json → tests/e2e/resources/expected_downloads_summaries/plugins/date_range/no_downloads.json
View file

@ -1,5 +1,5 @@
{
".ytdl-sub-pz-download-archive.json": "99914b932bd37a50b983c5e7c90ae93b",
".ytdl-sub-recent-download-archive.json": "99914b932bd37a50b983c5e7c90ae93b",
"fanart.jpg": "129c6639b47299bc48062f0365e670ee",
"poster.jpg": "5de28eea5a921a041452ab3ce1041f73",
"tvshow.nfo": "85cef9db54e9afb7af8e4912b5262d3f"

2
tests/e2e/resources/expected_downloads_summaries/youtube/test_channel_recent.json → tests/e2e/resources/expected_downloads_summaries/plugins/date_range/test_channel_recent.json
View file

@ -1,5 +1,5 @@
{
".ytdl-sub-pz-download-archive.json": "756b60d7a6c47d8163e3283404493a8d",
".ytdl-sub-recent-download-archive.json": "756b60d7a6c47d8163e3283404493a8d",
"Season 2018/s2018.e1029 - Jesse's Minecraft Server ｜ Teaser Trailer-thumb.jpg": "705ca4e0d99b37e9ecdf6bfe4b90c59b",
"Season 2018/s2018.e1029 - Jesse's Minecraft Server ｜ Teaser Trailer.mp4": "82f6ee7253e1dbb83ae7215af08ffacc",
"Season 2018/s2018.e1029 - Jesse's Minecraft Server ｜ Teaser Trailer.nfo": "368d68db0cbe9eb4f43ece0517445e82",

2
tests/e2e/resources/expected_downloads_summaries/youtube/test_channel_rolling_recent.json → tests/e2e/resources/expected_downloads_summaries/plugins/date_range/test_channel_rolling_recent.json
View file

@ -1,5 +1,5 @@
{
".ytdl-sub-pz-download-archive.json": "68e164a35d1541ce761a6d7fef19ee08",
".ytdl-sub-recent-download-archive.json": "68e164a35d1541ce761a6d7fef19ee08",
"Season 2018/s2018.e1102 - Jesse's Minecraft Server ｜ IP mc.jesse.id-thumb.jpg": "28d852ede73b879b9ebf9a061cfc7d46",
"Season 2018/s2018.e1102 - Jesse's Minecraft Server ｜ IP mc.jesse.id.en.srt": "3d2c4e7f65d2ca5e96da38ce7eecfc4e",
"Season 2018/s2018.e1102 - Jesse's Minecraft Server ｜ IP mc.jesse.id.mp4": "e733b4cc385b953b08c8eb0f47e03c1e",

2
tests/e2e/resources/transaction_log_summaries/youtube/test_channel_no_additional_downloads.txt → tests/e2e/resources/transaction_log_summaries/plugins/date_range/no_downloads.txt
View file

@ -1,6 +1,6 @@
Files created in '{output_directory}'
----------------------------------------
.ytdl-sub-pz-download-archive.json
.ytdl-sub-recent-download-archive.json
fanart.jpg
poster.jpg
tvshow.nfo

2
tests/e2e/resources/transaction_log_summaries/youtube/test_channel_recent.txt → tests/e2e/resources/transaction_log_summaries/plugins/date_range/test_channel_recent.txt
View file

@ -1,6 +1,6 @@
Files created in '{output_directory}'
----------------------------------------
.ytdl-sub-pz-download-archive.json
.ytdl-sub-recent-download-archive.json
Season 2018/s2018.e1029 - Jesse's Minecraft Server ｜ Teaser Trailer-thumb.jpg
Season 2018/s2018.e1029 - Jesse's Minecraft Server ｜ Teaser Trailer.mp4
Season 2018/s2018.e1029 - Jesse's Minecraft Server ｜ Teaser Trailer.nfo

2
tests/e2e/resources/transaction_log_summaries/youtube/test_channel_rolling_recent.txt → tests/e2e/resources/transaction_log_summaries/plugins/date_range/test_channel_rolling_recent.txt
View file

@ -1,6 +1,6 @@
Files created in '{output_directory}'
----------------------------------------
.ytdl-sub-pz-download-archive.json
.ytdl-sub-recent-download-archive.json
fanart.jpg
poster.jpg
tvshow.nfo

207
tests/e2e/youtube/test_channel.py
View file

@ -1,21 +1,10 @@
import copy
from typing import Dict
import mergedeep
import pytest
from conftest import assert_debug_log
from e2e.expected_download import assert_expected_downloads
from e2e.expected_transaction_log import assert_transaction_log_matches
import ytdl_sub.downloaders.downloader
from ytdl_sub.subscriptions.subscription import Subscription
@pytest.fixture
def subscription_name():
return "pz"
@pytest.fixture
def channel_preset_dict(output_directory):
return {
@ -37,46 +26,6 @@ def channel_preset_dict(output_directory):
}
@pytest.fixture
def channel_subscription_generator(channel_as_tv_show_config, subscription_name):
def _channel_subscription_generator(preset_dict: Dict):
return Subscription.from_dict(
config=channel_as_tv_show_config, preset_name=subscription_name, preset_dict=preset_dict
)
return _channel_subscription_generator
####################################################################################################
# RECENT CHANNEL FIXTURES
@pytest.fixture
def recent_channel_preset_dict(channel_preset_dict):
# TODO: remove this hack by using a different channel
channel_preset_dict = copy.deepcopy(channel_preset_dict)
del channel_preset_dict["ytdl_options"]["break_on_reject"]
return mergedeep.merge(
channel_preset_dict,
{
"preset": "yt_channel_as_tv__recent",
"youtube": {"after": "20150101"},
},
)
####################################################################################################
# ROLLING RECENT CHANNEL FIXTURES
@pytest.fixture
def rolling_recent_channel_preset_dict(recent_channel_preset_dict):
recent_channel_preset_dict = copy.deepcopy(recent_channel_preset_dict)
return mergedeep.merge(
recent_channel_preset_dict,
{
"preset": "yt_channel_as_tv__only_recent",
"output_options": {"keep_files_after": "20181101"},
},
)
class TestChannelAsKodiTvShow:
"""
Downloads my old minecraft youtube channel. Ensure the above files exist and have the
@ -86,12 +35,14 @@ class TestChannelAsKodiTvShow:
@pytest.mark.parametrize("dry_run", [True, False])
def test_full_channel_download(
self,
channel_subscription_generator,
channel_as_tv_show_config,
channel_preset_dict,
output_directory,
dry_run,
):
full_channel_subscription = channel_subscription_generator(preset_dict=channel_preset_dict)
full_channel_subscription = Subscription.from_dict(
config=channel_as_tv_show_config, preset_name="pz", preset_dict=channel_preset_dict
)
transaction_log = full_channel_subscription.download(dry_run=dry_run)
assert_transaction_log_matches(
output_directory=output_directory,
@ -103,153 +54,3 @@ class TestChannelAsKodiTvShow:
dry_run=dry_run,
expected_download_summary_file_name="youtube/test_channel_full.json",
)
@pytest.mark.parametrize("dry_run", [True, False])
def test_recent_channel_download(
self,
channel_subscription_generator,
recent_channel_preset_dict,
output_directory,
dry_run,
):
recent_channel_subscription = channel_subscription_generator(
preset_dict=recent_channel_preset_dict
)
transaction_log = recent_channel_subscription.download(dry_run=dry_run)
assert_transaction_log_matches(
output_directory=output_directory,
transaction_log=transaction_log,
transaction_log_summary_file_name="youtube/test_channel_recent.txt",
)
assert_expected_downloads(
output_directory=output_directory,
dry_run=dry_run,
expected_download_summary_file_name="youtube/test_channel_recent.json",
)
if not dry_run:
# try downloading again, ensure nothing more was downloaded
with assert_debug_log(
logger=ytdl_sub.downloaders.downloader.download_logger,
expected_message="ExistingVideoReached, stopping additional downloads",
):
transaction_log = recent_channel_subscription.download()
assert_transaction_log_matches(
output_directory=output_directory,
transaction_log=transaction_log,
transaction_log_summary_file_name=(
"youtube/test_channel_no_additional_downloads.txt"
),
)
assert_expected_downloads(
output_directory=output_directory,
dry_run=dry_run,
expected_download_summary_file_name="youtube/test_channel_recent.json",
)
@pytest.mark.parametrize("dry_run", [True, False])
def test_recent_channel_download__no_vids_in_range(
self,
channel_subscription_generator,
recent_channel_preset_dict,
output_directory,
dry_run,
):
recent_channel_preset_dict["youtube"]["after"] = "21000101"
recent_channel_no_vids_in_range_subscription = channel_subscription_generator(
preset_dict=recent_channel_preset_dict
)
# Run twice, ensure nothing changes between runs
for _ in range(2):
transaction_log = recent_channel_no_vids_in_range_subscription.download(dry_run=dry_run)
assert_transaction_log_matches(
output_directory=output_directory,
transaction_log=transaction_log,
transaction_log_summary_file_name="youtube/test_channel_no_additional_downloads.txt",
)
assert_expected_downloads(
output_directory=output_directory,
dry_run=dry_run,
expected_download_summary_file_name="youtube/test_channel_no_additional_downloads.json",
)
@pytest.mark.parametrize("dry_run", [True, False])
def test_rolling_recent_channel_download(
self,
channel_subscription_generator,
recent_channel_preset_dict,
rolling_recent_channel_preset_dict,
output_directory,
dry_run,
):
recent_channel_subscription = channel_subscription_generator(
preset_dict=recent_channel_preset_dict
)
rolling_recent_channel_subscription = channel_subscription_generator(
preset_dict=rolling_recent_channel_preset_dict
)
# First, download recent vids. Always download since we want to test dry-run
# on the rolling recent portion.
with assert_debug_log(
logger=ytdl_sub.downloaders.downloader.download_logger,
expected_message="RejectedVideoReached, stopping additional downloads",
):
transaction_log = recent_channel_subscription.download(dry_run=False)
assert_transaction_log_matches(
output_directory=output_directory,
transaction_log=transaction_log,
transaction_log_summary_file_name="youtube/test_channel_recent.txt",
)
assert_expected_downloads(
output_directory=output_directory,
dry_run=False,
expected_download_summary_file_name="youtube/test_channel_recent.json",
)
# Then, download the rolling recent vids subscription. This should remove one of the
# two videos
with assert_debug_log(
logger=ytdl_sub.downloaders.downloader.download_logger,
expected_message="ExistingVideoReached, stopping additional downloads",
):
transaction_log = rolling_recent_channel_subscription.download(dry_run=dry_run)
expected_downloads_summary = (
"youtube/test_channel_recent.json"
if dry_run
else "youtube/test_channel_rolling_recent.json"
)
assert_transaction_log_matches(
output_directory=output_directory,
transaction_log=transaction_log,
transaction_log_summary_file_name="youtube/test_channel_rolling_recent.txt",
)
assert_expected_downloads(
output_directory=output_directory,
dry_run=False,
expected_download_summary_file_name=expected_downloads_summary,
)
# Invoke the rolling download again, ensure downloading stopped early from it already
# existing
if not dry_run:
with assert_debug_log(
logger=ytdl_sub.downloaders.downloader.download_logger,
expected_message="ExistingVideoReached, stopping additional downloads",
):
transaction_log = rolling_recent_channel_subscription.download()
assert_transaction_log_matches(
output_directory=output_directory,
transaction_log=transaction_log,
transaction_log_summary_file_name="youtube/test_channel_no_additional_downloads.txt",
)
assert_expected_downloads(
output_directory=output_directory,
dry_run=False,
expected_download_summary_file_name=expected_downloads_summary,
)