[REFACTOR] Support simple download

This commit is contained in:
Jesse Bannon 2023-07-25 23:23:36 -07:00
parent 598b1da8a4
commit 5b190a718f
20 changed files with 383 additions and 418 deletions

View file

@ -13,6 +13,5 @@
# #
rammstein_music_videos: rammstein_music_videos:
preset: "video" preset: "video"
download: download: "https://youtube.com/playlist?list=PLVTLbc6i-h_iuhdwUfuPDLFLXG2QQnz-x"
url: "https://youtube.com/playlist?list=PLVTLbc6i-h_iuhdwUfuPDLFLXG2QQnz-x"

View file

@ -165,7 +165,6 @@ presets:
# Download using the multi_url strategy # Download using the multi_url strategy
download: download:
urls:
# The first URL will be all the artist's tracks. # The first URL will be all the artist's tracks.
# Treat these as singles - an album with a single track # Treat these as singles - an album with a single track
- url: "{sc_artist_url}/tracks" - url: "{sc_artist_url}/tracks"

View file

@ -18,11 +18,10 @@ configuration:
presets: presets:
music_video: music_video:
# Set the download details
download:
# We will only use a single URL to download music video(s). # We will only use a single URL to download music video(s).
# Make {url} an override variable to set later. # Make {url} an override variable to set later.
url: "{url}" download:
- "{url}"
# For advanced YTDL users only; any YTDL parameter can be set here. # For advanced YTDL users only; any YTDL parameter can be set here.
# To download age-restricted videos, you will need to set your cookie # To download age-restricted videos, you will need to set your cookie

View file

@ -146,8 +146,27 @@ class UrlValidator(StrictDictValidator):
return self._download_reverse.value return self._download_reverse.value
class UrlListValidator(ListValidator[UrlValidator]): class UrlStringOrDictValidator(UrlValidator):
_inner_list_type = UrlValidator """
URL validator that supports a single string like:
download:
- "https://"
or
download:
- url: "https://"
"""
_expected_value_type = (dict, str)
def __init__(self, name, value):
super().__init__(name, {"url": value} if isinstance(value, str) else value)
class UrlListValidator(ListValidator[UrlStringOrDictValidator]):
_inner_list_type = UrlStringOrDictValidator
_expected_value_type_name = "collection url list" _expected_value_type_name = "collection url list"
def __init__(self, name, value): def __init__(self, name, value):
@ -197,17 +216,11 @@ class MultiUrlValidator(OptionsValidator):
# Pop old required field in case it's still there # Pop old required field in case it's still there
value_copy.pop("download_strategy", None) value_copy.pop("download_strategy", None)
if "urls" in value_copy: # Deal with old multi-url download strategy
if isinstance(value, dict) and "urls" in value_copy:
self._urls = UrlListValidator(name=name, value=value_copy["urls"]) self._urls = UrlListValidator(name=name, value=value_copy["urls"])
else: else:
# Validate using a single URL validator first self._urls = UrlListValidator(name=name, value=value_copy)
_ = UrlValidator(name=name, value=value_copy)
self._urls = UrlListValidator(name=name, value=[value_copy])
else:
# Should error here. TODO: Add simplifications download here (string, list)
self._urls = UrlListValidator(
name=name, value=UrlValidator(name=name, value=value_copy)
)
@property @property
def urls(self) -> UrlListValidator: def urls(self) -> UrlListValidator:

View file

@ -1,8 +1,7 @@
presets: presets:
_view: _view:
download: download: "{url}"
url: "{url}"
output_options: output_options:
output_directory: "/tmp/ytdl-sub-view" output_directory: "/tmp/ytdl-sub-view"
file_name: "{uid}.{ext}" file_name: "{uid}.{ext}"

View file

@ -2,7 +2,7 @@ presets:
# TODO: Update this # TODO: Update this
kodi_music_video: kodi_music_video:
download: download:
url: "{music_video_url}" - "{music_video_url}"
output_options: output_options:
output_directory: "{music_video_directory}" output_directory: "{music_video_directory}"

View file

@ -34,7 +34,6 @@ presets:
# TV show from one or more sources. Uses {url}'s avatar and banner as poster and fanart # TV show from one or more sources. Uses {url}'s avatar and banner as poster and fanart
_tv_show_by_date: _tv_show_by_date:
download: download:
urls:
- url: "{url}" - url: "{url}"
playlist_thumbnails: playlist_thumbnails:
- name: "{tv_show_poster_file_name}" - name: "{tv_show_poster_file_name}"

View file

@ -21,7 +21,6 @@ presets:
collection_season_1: collection_season_1:
download: download:
urls:
- url: "{collection_season_1_url}" - url: "{collection_season_1_url}"
variables: variables:
collection_season_number: "1" collection_season_number: "1"
@ -45,7 +44,6 @@ presets:
collection_season_2: collection_season_2:
download: download:
urls:
- url: "{collection_season_2_url}" - url: "{collection_season_2_url}"
variables: variables:
collection_season_number: "2" collection_season_number: "2"
@ -63,7 +61,6 @@ presets:
collection_season_3: collection_season_3:
download: download:
urls:
- url: "{collection_season_3_url}" - url: "{collection_season_3_url}"
variables: variables:
collection_season_number: "3" collection_season_number: "3"
@ -81,7 +78,6 @@ presets:
collection_season_4: collection_season_4:
download: download:
urls:
- url: "{collection_season_4_url}" - url: "{collection_season_4_url}"
variables: variables:
collection_season_number: "4" collection_season_number: "4"
@ -99,7 +95,6 @@ presets:
collection_season_5: collection_season_5:
download: download:
urls:
- url: "{collection_season_5_url}" - url: "{collection_season_5_url}"
variables: variables:
collection_season_number: "5" collection_season_number: "5"
@ -117,7 +112,6 @@ presets:
collection_season_6: collection_season_6:
download: download:
urls:
- url: "{collection_season_6_url}" - url: "{collection_season_6_url}"
variables: variables:
collection_season_number: "6" collection_season_number: "6"
@ -135,7 +129,6 @@ presets:
collection_season_7: collection_season_7:
download: download:
urls:
- url: "{collection_season_7_url}" - url: "{collection_season_7_url}"
variables: variables:
collection_season_number: "7" collection_season_number: "7"
@ -153,7 +146,6 @@ presets:
collection_season_8: collection_season_8:
download: download:
urls:
- url: "{collection_season_8_url}" - url: "{collection_season_8_url}"
variables: variables:
collection_season_number: "8" collection_season_number: "8"
@ -171,7 +163,6 @@ presets:
collection_season_9: collection_season_9:
download: download:
urls:
- url: "{collection_season_9_url}" - url: "{collection_season_9_url}"
variables: variables:
collection_season_number: "9" collection_season_number: "9"
@ -189,7 +180,6 @@ presets:
collection_season_10: collection_season_10:
download: download:
urls:
- url: "{collection_season_10_url}" - url: "{collection_season_10_url}"
variables: variables:
collection_season_number: "10" collection_season_number: "10"
@ -207,7 +197,6 @@ presets:
collection_season_11: collection_season_11:
download: download:
urls:
- url: "{collection_season_11_url}" - url: "{collection_season_11_url}"
variables: variables:
collection_season_number: "11" collection_season_number: "11"
@ -225,7 +214,6 @@ presets:
collection_season_12: collection_season_12:
download: download:
urls:
- url: "{collection_season_12_url}" - url: "{collection_season_12_url}"
variables: variables:
collection_season_number: "12" collection_season_number: "12"
@ -243,7 +231,6 @@ presets:
collection_season_13: collection_season_13:
download: download:
urls:
- url: "{collection_season_13_url}" - url: "{collection_season_13_url}"
variables: variables:
collection_season_number: "13" collection_season_number: "13"
@ -261,7 +248,6 @@ presets:
collection_season_14: collection_season_14:
download: download:
urls:
- url: "{collection_season_14_url}" - url: "{collection_season_14_url}"
variables: variables:
collection_season_number: "14" collection_season_number: "14"
@ -279,7 +265,6 @@ presets:
collection_season_15: collection_season_15:
download: download:
urls:
- url: "{collection_season_15_url}" - url: "{collection_season_15_url}"
variables: variables:
collection_season_number: "15" collection_season_number: "15"
@ -297,7 +282,6 @@ presets:
collection_season_16: collection_season_16:
download: download:
urls:
- url: "{collection_season_16_url}" - url: "{collection_season_16_url}"
variables: variables:
collection_season_number: "16" collection_season_number: "16"
@ -315,7 +299,6 @@ presets:
collection_season_17: collection_season_17:
download: download:
urls:
- url: "{collection_season_17_url}" - url: "{collection_season_17_url}"
variables: variables:
collection_season_number: "17" collection_season_number: "17"
@ -333,7 +316,6 @@ presets:
collection_season_18: collection_season_18:
download: download:
urls:
- url: "{collection_season_18_url}" - url: "{collection_season_18_url}"
variables: variables:
collection_season_number: "18" collection_season_number: "18"
@ -351,7 +333,6 @@ presets:
collection_season_19: collection_season_19:
download: download:
urls:
- url: "{collection_season_19_url}" - url: "{collection_season_19_url}"
variables: variables:
collection_season_number: "19" collection_season_number: "19"
@ -369,7 +350,6 @@ presets:
collection_season_20: collection_season_20:
download: download:
urls:
- url: "{collection_season_20_url}" - url: "{collection_season_20_url}"
variables: variables:
collection_season_number: "20" collection_season_number: "20"
@ -387,7 +367,6 @@ presets:
collection_season_21: collection_season_21:
download: download:
urls:
- url: "{collection_season_21_url}" - url: "{collection_season_21_url}"
variables: variables:
collection_season_number: "21" collection_season_number: "21"
@ -405,7 +384,6 @@ presets:
collection_season_22: collection_season_22:
download: download:
urls:
- url: "{collection_season_22_url}" - url: "{collection_season_22_url}"
variables: variables:
collection_season_number: "22" collection_season_number: "22"
@ -423,7 +401,6 @@ presets:
collection_season_23: collection_season_23:
download: download:
urls:
- url: "{collection_season_23_url}" - url: "{collection_season_23_url}"
variables: variables:
collection_season_number: "23" collection_season_number: "23"
@ -441,7 +418,6 @@ presets:
collection_season_24: collection_season_24:
download: download:
urls:
- url: "{collection_season_24_url}" - url: "{collection_season_24_url}"
variables: variables:
collection_season_number: "24" collection_season_number: "24"
@ -459,7 +435,6 @@ presets:
collection_season_25: collection_season_25:
download: download:
urls:
- url: "{collection_season_25_url}" - url: "{collection_season_25_url}"
variables: variables:
collection_season_number: "25" collection_season_number: "25"
@ -477,7 +452,6 @@ presets:
collection_season_26: collection_season_26:
download: download:
urls:
- url: "{collection_season_26_url}" - url: "{collection_season_26_url}"
variables: variables:
collection_season_number: "26" collection_season_number: "26"
@ -495,7 +469,6 @@ presets:
collection_season_27: collection_season_27:
download: download:
urls:
- url: "{collection_season_27_url}" - url: "{collection_season_27_url}"
variables: variables:
collection_season_number: "27" collection_season_number: "27"
@ -513,7 +486,6 @@ presets:
collection_season_28: collection_season_28:
download: download:
urls:
- url: "{collection_season_28_url}" - url: "{collection_season_28_url}"
variables: variables:
collection_season_number: "28" collection_season_number: "28"
@ -531,7 +503,6 @@ presets:
collection_season_29: collection_season_29:
download: download:
urls:
- url: "{collection_season_29_url}" - url: "{collection_season_29_url}"
variables: variables:
collection_season_number: "29" collection_season_number: "29"
@ -549,7 +520,6 @@ presets:
collection_season_30: collection_season_30:
download: download:
urls:
- url: "{collection_season_30_url}" - url: "{collection_season_30_url}"
variables: variables:
collection_season_number: "30" collection_season_number: "30"
@ -567,7 +537,6 @@ presets:
collection_season_31: collection_season_31:
download: download:
urls:
- url: "{collection_season_31_url}" - url: "{collection_season_31_url}"
variables: variables:
collection_season_number: "31" collection_season_number: "31"
@ -585,7 +554,6 @@ presets:
collection_season_32: collection_season_32:
download: download:
urls:
- url: "{collection_season_32_url}" - url: "{collection_season_32_url}"
variables: variables:
collection_season_number: "32" collection_season_number: "32"
@ -603,7 +571,6 @@ presets:
collection_season_33: collection_season_33:
download: download:
urls:
- url: "{collection_season_33_url}" - url: "{collection_season_33_url}"
variables: variables:
collection_season_number: "33" collection_season_number: "33"
@ -621,7 +588,6 @@ presets:
collection_season_34: collection_season_34:
download: download:
urls:
- url: "{collection_season_34_url}" - url: "{collection_season_34_url}"
variables: variables:
collection_season_number: "34" collection_season_number: "34"
@ -639,7 +605,6 @@ presets:
collection_season_35: collection_season_35:
download: download:
urls:
- url: "{collection_season_35_url}" - url: "{collection_season_35_url}"
variables: variables:
collection_season_number: "35" collection_season_number: "35"
@ -657,7 +622,6 @@ presets:
collection_season_36: collection_season_36:
download: download:
urls:
- url: "{collection_season_36_url}" - url: "{collection_season_36_url}"
variables: variables:
collection_season_number: "36" collection_season_number: "36"
@ -675,7 +639,6 @@ presets:
collection_season_37: collection_season_37:
download: download:
urls:
- url: "{collection_season_37_url}" - url: "{collection_season_37_url}"
variables: variables:
collection_season_number: "37" collection_season_number: "37"
@ -693,7 +656,6 @@ presets:
collection_season_38: collection_season_38:
download: download:
urls:
- url: "{collection_season_38_url}" - url: "{collection_season_38_url}"
variables: variables:
collection_season_number: "38" collection_season_number: "38"
@ -711,7 +673,6 @@ presets:
collection_season_39: collection_season_39:
download: download:
urls:
- url: "{collection_season_39_url}" - url: "{collection_season_39_url}"
variables: variables:
collection_season_number: "39" collection_season_number: "39"
@ -729,7 +690,6 @@ presets:
collection_season_40: collection_season_40:
download: download:
urls:
- url: "{collection_season_40_url}" - url: "{collection_season_40_url}"
variables: variables:
collection_season_number: "40" collection_season_number: "40"

View file

@ -11,7 +11,7 @@ from ytdl_sub.subscriptions.subscription import Subscription
def sponsorblock_and_subs_preset_dict(output_directory) -> Dict: def sponsorblock_and_subs_preset_dict(output_directory) -> Dict:
return { return {
"preset": "music_video", "preset": "music_video",
"download": {"url": "https://www.youtube.com/watch?v=-wJOUAuKZm8"}, "download": "https://www.youtube.com/watch?v=-wJOUAuKZm8",
# override the output directory with our fixture-generated dir # override the output directory with our fixture-generated dir
"output_options": {"output_directory": output_directory}, "output_options": {"output_directory": output_directory},
"subtitles": { "subtitles": {

View file

@ -10,7 +10,7 @@ from ytdl_sub.subscriptions.subscription import Subscription
def preset_dict(output_directory): def preset_dict(output_directory):
return { return {
"preset": "music_video", "preset": "music_video",
"download": {"url": "https://www.youtube.com/watch?v=2zYF9JLHDmA"}, "download": "https://www.youtube.com/watch?v=2zYF9JLHDmA",
"output_options": {"output_directory": output_directory}, "output_options": {"output_directory": output_directory},
# download the worst format so it is fast # download the worst format so it is fast
"ytdl_options": { "ytdl_options": {

View file

@ -9,7 +9,7 @@ from ytdl_sub.subscriptions.subscription import Subscription
def preset_dict(output_directory): def preset_dict(output_directory):
return { return {
"preset": "music_video", "preset": "music_video",
"download": {"url": "https://www.youtube.com/watch?v=2zYF9JLHDmA"}, "download": "https://www.youtube.com/watch?v=2zYF9JLHDmA",
"output_options": {"output_directory": output_directory}, "output_options": {"output_directory": output_directory},
# download the worst format so it is fast # download the worst format so it is fast
"ytdl_options": { "ytdl_options": {
@ -23,7 +23,7 @@ def preset_dict(output_directory):
def playlist_preset_dict(output_directory): def playlist_preset_dict(output_directory):
return { return {
"preset": "music_video", "preset": "music_video",
"download": {"url": "https://www.youtube.com/playlist?list=PL5BC0FC26BECA5A35"}, "download": "https://www.youtube.com/playlist?list=PL5BC0FC26BECA5A35",
"output_options": {"output_directory": output_directory}, "output_options": {"output_directory": output_directory},
# download the worst format so it is fast # download the worst format so it is fast
"ytdl_options": { "ytdl_options": {

View file

@ -9,9 +9,7 @@ from ytdl_sub.utils.exceptions import ValidationException
@pytest.fixture @pytest.fixture
def single_song_video_dict(output_directory): def single_song_video_dict(output_directory):
return { return {
"download": { "download": "https://www.youtube.com/watch?v=2lAe1cqCOXo",
"url": "https://www.youtube.com/watch?v=2lAe1cqCOXo",
},
"output_options": {"output_directory": output_directory, "file_name": "will_error.mp4"}, "output_options": {"output_directory": output_directory, "file_name": "will_error.mp4"},
# test multi-tags # test multi-tags
"music_tags": {"genres": ["multi_tag_1", "multi_tag_2"]}, "music_tags": {"genres": ["multi_tag_1", "multi_tag_2"]},

View file

@ -8,7 +8,7 @@ from ytdl_sub.subscriptions.subscription import Subscription
def subscription_dict(output_directory): def subscription_dict(output_directory):
return { return {
"preset": "music_video", "preset": "music_video",
"download": {"url": "https://www.youtube.com/shorts/ucYmEqmlhFw"}, "download": "https://www.youtube.com/shorts/ucYmEqmlhFw",
# override the output directory with our fixture-generated dir # override the output directory with our fixture-generated dir
"output_options": {"output_directory": output_directory}, "output_options": {"output_directory": output_directory},
# download the worst format so it is fast # download the worst format so it is fast

View file

@ -17,7 +17,7 @@ from ytdl_sub.utils.exceptions import ValidationException
def regex_subscription_dict_base(output_directory): def regex_subscription_dict_base(output_directory):
return { return {
"preset": "music_video", "preset": "music_video",
"download": {"url": "https://youtube.com/playlist?list=PL5BC0FC26BECA5A35"}, "download": "https://youtube.com/playlist?list=PL5BC0FC26BECA5A35",
# override the output directory with our fixture-generated dir # override the output directory with our fixture-generated dir
"output_options": {"output_directory": output_directory}, "output_options": {"output_directory": output_directory},
# download the worst format so it is fast # download the worst format so it is fast

View file

@ -136,7 +136,7 @@ class TestSplitByChapters:
mergedeep.merge( mergedeep.merge(
yt_album_as_chapters_with_regex_preset_dict, yt_album_as_chapters_with_regex_preset_dict,
{ {
"download": {"url": "https://youtube.com/watch?v=HKTNxEqsN3Q"}, "download": "https://youtube.com/watch?v=HKTNxEqsN3Q",
"split_by_chapters": {"when_no_chapters": when_no_chapters}, "split_by_chapters": {"when_no_chapters": when_no_chapters},
}, },
) )

View file

@ -10,7 +10,7 @@ from ytdl_sub.subscriptions.subscription import Subscription
def single_video_subs_embed_preset_dict(output_directory): def single_video_subs_embed_preset_dict(output_directory):
return { return {
"preset": "music_video", "preset": "music_video",
"download": {"url": "https://www.youtube.com/watch?v=2lAe1cqCOXo"}, "download": "https://www.youtube.com/watch?v=2lAe1cqCOXo",
# override the output directory with our fixture-generated dir # override the output directory with our fixture-generated dir
"output_options": {"output_directory": output_directory}, "output_options": {"output_directory": output_directory},
"subtitles": { "subtitles": {

View file

@ -11,7 +11,7 @@ from ytdl_sub.subscriptions.subscription import Subscription
def single_video_preset_dict_old_video_tags_format(output_directory): def single_video_preset_dict_old_video_tags_format(output_directory):
return { return {
"preset": "music_video", "preset": "music_video",
"download": {"url": "https://youtube.com/watch?v=HKTNxEqsN3Q"}, "download": "https://youtube.com/watch?v=HKTNxEqsN3Q",
# override the output directory with our fixture-generated dir # override the output directory with our fixture-generated dir
"output_options": { "output_options": {
"output_directory": output_directory, "output_directory": output_directory,
@ -37,7 +37,7 @@ def single_video_preset_dict_old_video_tags_format(output_directory):
def single_video_preset_dict(output_directory): def single_video_preset_dict(output_directory):
return { return {
"preset": "music_video", "preset": "music_video",
"download": {"url": "https://youtube.com/watch?v=HKTNxEqsN3Q"}, "download": "https://youtube.com/watch?v=HKTNxEqsN3Q",
# override the output directory with our fixture-generated dir # override the output directory with our fixture-generated dir
"output_options": { "output_options": {
"output_directory": output_directory, "output_directory": output_directory,

View file

@ -67,8 +67,8 @@ class TestConfigFilePartiallyValidatesPresets:
def test_error__download_args(self): def test_error__download_args(self):
self._partial_validate( self._partial_validate(
preset_dict={"download": {"bad_key": "nope"}}, preset_dict={"download": {"bad_key": "nope"}},
expected_error_message="Validation error in partial_preset.download: " expected_error_message="Validation error in partial_preset.download.1: "
"'partial_preset.download' contains the field 'bad_key' which is not allowed. " "'partial_preset.download.1' contains the field 'bad_key' which is not allowed. "
"Allowed fields: download_reverse, playlist_thumbnails, source_thumbnails, url, " "Allowed fields: download_reverse, playlist_thumbnails, source_thumbnails, url, "
"variables", "variables",
) )
@ -77,7 +77,7 @@ class TestConfigFilePartiallyValidatesPresets:
"preset_dict", "preset_dict",
[ [
{"nfo_tags": {"tags": {"key-1": {"attributes": {"test": "2"}}}}}, {"nfo_tags": {"tags": {"key-1": {"attributes": {"test": "2"}}}}},
{"download": {"urls": [{"variables_to_set": {"name": "value"}}]}}, {"download": [{"variables_to_set": {"name": "value"}}]},
], ],
) )
def test_partial_validate__incomplete_list_item(self, preset_dict): def test_partial_validate__incomplete_list_item(self, preset_dict):

View file

@ -12,8 +12,11 @@ class TestPreset:
@pytest.mark.parametrize( @pytest.mark.parametrize(
"download_value", "download_value",
[ [
{"url": "youtube.com/watch?v=123abc"}, {"url": "youtube.com/watch?v=123abc"}, # url download strategy
{"urls": [{"url": "youtube.com/watch?v=123abc"}]}, {"urls": [{"url": "youtube.com/watch?v=123abc"}]}, # multi-url download strategy
"youtube.com/watch?v=123abc", # single string
["youtube.com/watch?v=123abc", "youtube.com/watch?v=123xyz"], # list of strings
[{"url": "youtube.com/watch?v=123abc"}, "youtube.com/watch?v=123abc"], # dict and str
########################################################### ###########################################################
##### OLD download_strategy format ##### OLD download_strategy format
{"download_strategy": "url", "url": "youtube.com/watch?v=123abc"}, {"download_strategy": "url", "url": "youtube.com/watch?v=123abc"},
@ -201,9 +204,7 @@ class TestPreset:
config=config_file, config=config_file,
name="test", name="test",
value={ value={
"download": { "download": [{"url": "non-empty url"}, {"url": ""}], # empty url
"urls": [{"url": "non-empty url"}, {"url": ""}], # empty url
},
"output_options": output_options, "output_options": output_options,
}, },
) )
@ -222,9 +223,7 @@ class TestPreset:
config=config_file, config=config_file,
name="test", name="test",
value={ value={
"download": { "download": [{"url": "{url}"}, {"url": "{url2}"}],
"urls": [{"url": "{url}"}, {"url": "{url2}"}],
},
"output_options": output_options, "output_options": output_options,
"overrides": {"url": "", "url2": ""}, "overrides": {"url": "", "url2": ""},
}, },

View file

@ -52,7 +52,7 @@ class TestPrebuiltTVShowPresets:
preset_name="preset_test", preset_name="preset_test",
preset_dict={ preset_dict={
"preset": parent_presets, "preset": parent_presets,
"download": {"urls": [{"url": "https://second.url"}]}, "download": "https://second.url",
"overrides": { "overrides": {
"url": "https://your.name.here", "url": "https://your.name.here",
"tv_show_name": "test-compile", "tv_show_name": "test-compile",