[BACKEND] Remove old format for video_tags, music_tags, download_strategy (#899)

Now that sufficient time has passed, we can formally deprecate the following:
- https://ytdl-sub.readthedocs.io/en/latest/deprecation_notices.html#video-tags
- https://ytdl-sub.readthedocs.io/en/latest/deprecation_notices.html#music-tags
- usage of `download_strategy` (simply remove it if you're getting an error about it)
This commit is contained in:
Jesse Bannon 2024-01-15 00:04:32 -08:00 committed by GitHub
parent e3158583ca
commit c0ca3c3945
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
15 changed files with 25 additions and 252 deletions

View file

@ -1,3 +1,3 @@
sphinx-book-theme==1.0.1
sphinx-book-theme==1.1.0
sphinx-copybutton==0.5.2
sphinx-design==0.5.0

View file

@ -50,8 +50,9 @@ lint =
isort==5.10.1
pylint==2.13.5
docs =
sphinx==4.5.0
sphinx-rtd-theme==1.0.0
sphinx==7.2.6
sphinx-rtd-theme==2.0.0
sphinx-book-theme==1.1.0
build =
build
twine

View file

@ -1,4 +1,3 @@
import copy
from typing import Any
from typing import Dict
from typing import Optional
@ -276,17 +275,11 @@ class MultiUrlValidator(OptionsValidator):
def __init__(self, name, value):
super().__init__(name, value)
# Copy since we're popping things
value_copy = copy.deepcopy(value)
if isinstance(value, dict):
# Pop old required field in case it's still there
value_copy.pop("download_strategy", None)
# Deal with old multi-url download strategy
if isinstance(value, dict) and "urls" in value_copy:
self._urls = UrlListValidator(name=name, value=value_copy["urls"])
if isinstance(value, dict) and "urls" in value:
self._urls = UrlListValidator(name=name, value=value["urls"])
else:
self._urls = UrlListValidator(name=name, value=value_copy)
self._urls = UrlListValidator(name=name, value=value)
@property
def urls(self) -> UrlListValidator:

View file

@ -1,6 +1,4 @@
import copy
from collections import defaultdict
from typing import Any
from typing import Dict
from typing import List
@ -14,10 +12,8 @@ from ytdl_sub.entries.script.variable_definitions import VariableDefinitions
from ytdl_sub.utils.file_handler import FileMetadata
from ytdl_sub.utils.logger import Logger
from ytdl_sub.validators.audo_codec_validator import AUDIO_CODEC_EXTS
from ytdl_sub.validators.strict_dict_validator import StrictDictValidator
from ytdl_sub.validators.string_formatter_validators import ListFormatterValidator
from ytdl_sub.validators.string_formatter_validators import StringFormatterValidator
from ytdl_sub.validators.validators import BoolValidator
v: VariableDefinitions = VARIABLES
@ -40,31 +36,6 @@ def _is_multi_field(tag_name: str) -> bool:
}
class MusicTagsValidator(StrictDictValidator):
"""
Validator for the music_tag's `tags` field. Treat each value as a list.
Can still specify it like a single value but under-the-hood it's a list of a single element.
"""
_optional_keys = set(list(mediafile.MediaFile.sorted_fields()))
def __init__(self, name, value):
super().__init__(name, value)
self._tags: Dict[str, List[StringFormatterValidator]] = {}
for key in self._keys:
self._tags[key] = self._validate_key(key=key, validator=ListFormatterValidator).list
@property
def as_lists(self) -> Dict[str, List[StringFormatterValidator]]:
"""
Returns
-------
Tag formatter(s) as a list
"""
return self._tags
class MusicTagsOptions(OptionsDictValidator):
"""
Adds tags to every download audio file using
@ -93,40 +64,24 @@ class MusicTagsOptions(OptionsDictValidator):
- "ytdl-sub"
"""
_optional_keys = {"tags", "embed_thumbnail"}
_allow_extra_keys = True
_optional_keys = set(list(mediafile.MediaFile.sorted_fields()))
def __init__(self, name, value):
super().__init__(name, value)
self._embed_thumbnail = self._validate_key_if_present(
key="embed_thumbnail", validator=BoolValidator
)
new_tags_dict: Dict[str, Any] = copy.deepcopy(value)
old_tags_dict = new_tags_dict.pop("tags", {})
new_tags_dict.pop("embed_thumbnail", None)
self._is_old_format = len(old_tags_dict) > 0 or self._embed_thumbnail is not None
self._tags = MusicTagsValidator(name=name, value=dict(old_tags_dict, **new_tags_dict))
self._tags: Dict[str, List[StringFormatterValidator]] = {}
for key in self._keys:
self._tags[key] = self._validate_key(key=key, validator=ListFormatterValidator).list
@property
def tags(self) -> MusicTagsValidator:
def as_lists(self) -> Dict[str, List[StringFormatterValidator]]:
"""
Key, values of tag names, tag values. Supports source and override variables.
Supports lists which will get written to MP3s as id3v2.4 multi-tags.
Returns
-------
Tag formatter(s) as a list
"""
return self._tags
@property
def embed_thumbnail(self) -> bool:
"""
Optional. Whether to embed the thumbnail into the audio file.
"""
if self._embed_thumbnail is None:
return False
return self._embed_thumbnail.value
class MusicTagsPlugin(Plugin[MusicTagsOptions]):
plugin_options_type = MusicTagsOptions
@ -142,23 +97,9 @@ class MusicTagsPlugin(Plugin[MusicTagsOptions]):
f"to audio using the audio_extract plugin."
)
# pylint: disable=protected-access
if self.plugin_options._is_old_format:
logger.warning(
"music_tags.tags is now deprecated. Place your tags directly under music_tags "
"instead. The old format will be removed in October of 2023. See "
"https://ytdl-sub.readthedocs.io/en/latest/deprecation_notices.html#music-tags "
"for more details."
)
if self.plugin_options.embed_thumbnail:
logger.warning(
"music_tags.embed_thumbnail is also deprecated. Use the dedicated "
"embed_thumbnail plugin instead. This will be removed in October of 2023."
)
# Resolve the tags into this dict
tags_to_write: Dict[str, List[str]] = defaultdict(list)
for tag_name, tag_formatters in self.plugin_options.tags.as_lists.items():
for tag_name, tag_formatters in self.plugin_options.as_lists.items():
for tag_formatter in tag_formatters:
tag_value = self.overrides.apply_formatter(formatter=tag_formatter, entry=entry)
tags_to_write[tag_name].append(tag_value)
@ -180,19 +121,10 @@ class MusicTagsPlugin(Plugin[MusicTagsOptions]):
)
setattr(audio_file, tag_name, tag_value[0])
if self.plugin_options.embed_thumbnail and entry.is_thumbnail_downloaded():
with open(entry.get_download_thumbnail_path(), "rb") as thumb:
mediafile_img = mediafile.Image(
data=thumb.read(), desc="cover", type=mediafile.ImageType.front
)
audio_file.images = [mediafile_img]
audio_file.save()
# report the tags written
title = f"{'Embedded Thumbnail, ' if self.plugin_options.embed_thumbnail else ''}Music Tags"
return FileMetadata.from_dict(
title="Music Tags",
value_dict=tags_to_write,
title=title,
)

View file

@ -1,9 +1,7 @@
import copy
from typing import Any
from typing import Dict
from ytdl_sub.config.plugin.plugin import Plugin
from ytdl_sub.config.validators.options import ToggleableOptionsDictValidator
from ytdl_sub.config.validators.options import OptionsValidator
from ytdl_sub.entries.entry import Entry
from ytdl_sub.utils.ffmpeg import add_ffmpeg_metadata_key_values
from ytdl_sub.utils.file_handler import FileMetadata
@ -13,7 +11,7 @@ from ytdl_sub.validators.string_formatter_validators import DictFormatterValidat
logger = Logger.get("video-tags")
class VideoTagsOptions(ToggleableOptionsDictValidator):
class VideoTagsOptions(DictFormatterValidator, OptionsValidator):
"""
Adds tags to every downloaded video file using ffmpeg ``-metadata key=value`` args.
@ -27,34 +25,6 @@ class VideoTagsOptions(ToggleableOptionsDictValidator):
description: "{description}"
"""
_optional_keys = {"enable", "tags"}
_allow_extra_keys = True
@classmethod
def partial_validate(cls, name: str, value: Any) -> None:
"""
Partially validate video tags
"""
if isinstance(value, dict):
value["tags"] = value.get("tags", {})
_ = cls(name, value)
def __init__(self, name, value):
super().__init__(name, value)
new_tags_dict: Dict[str, Any] = copy.deepcopy(value)
old_tags_dict = new_tags_dict.pop("tags", {})
self._is_old_format = len(old_tags_dict) > 0
self._tags = DictFormatterValidator(name=name, value=dict(old_tags_dict, **new_tags_dict))
@property
def tags(self) -> DictFormatterValidator:
"""
Key/values of tag names/values. Supports source and override variables.
"""
return self._tags
class VideoTagsPlugin(Plugin[VideoTagsOptions]):
plugin_options_type = VideoTagsOptions
@ -63,17 +33,8 @@ class VideoTagsPlugin(Plugin[VideoTagsOptions]):
"""
Tags the entry's audio file using values defined in the metadata options
"""
# pylint: disable=protected-access
if self.plugin_options._is_old_format:
logger.warning(
"video_tags.tags is now deprecated. Place your tags directly under video_tags "
"instead. The old format will be removed in October of 2023. See "
"https://ytdl-sub.readthedocs.io/en/latest/deprecation_notices.html#video-tags "
"for more details."
)
tags_to_write: Dict[str, str] = {}
for tag_name, tag_formatter in self.plugin_options.tags.dict.items():
for tag_name, tag_formatter in self.plugin_options.dict.items():
tag_value = self.overrides.apply_formatter(formatter=tag_formatter, entry=entry)
tags_to_write[tag_name] = tag_value

View file

@ -5,25 +5,6 @@ from expected_transaction_log import assert_transaction_log_matches
from ytdl_sub.subscriptions.subscription import Subscription
@pytest.fixture
def single_preset_dict_old_format(output_directory):
return {
"preset": "Single",
# test multi-tags
"music_tags": {"embed_thumbnail": True, "tags": {"genres": ["multi_tag_1", "multi_tag_2"]}},
"format": "worst[ext=mp4]",
"audio_extract": {"codec": "mp3", "quality": 320},
"ytdl_options": {
"postprocessor_args": {"ffmpeg": ["-bitexact"]}, # Must add this for reproducibility
},
"overrides": {
"track_artist": "YouTube",
"url": "https://www.youtube.com/watch?v=2lAe1cqCOXo",
"music_directory": output_directory,
},
}
@pytest.fixture
def single_preset_dict(output_directory):
return {
@ -68,32 +49,6 @@ def youtube_release_preset_dict(output_directory):
class TestAudioExtract:
@pytest.mark.parametrize("dry_run", [True, False])
def test_audio_extract_single_song_old_format(
self,
default_config,
single_preset_dict_old_format,
output_directory,
dry_run,
):
subscription = Subscription.from_dict(
config=default_config,
preset_name="single_song_test",
preset_dict=single_preset_dict_old_format,
)
transaction_log = subscription.download(dry_run=dry_run)
assert_transaction_log_matches(
output_directory=output_directory,
transaction_log=transaction_log,
transaction_log_summary_file_name="plugins/test_audio_extract_single_old_format.txt",
)
assert_expected_downloads(
output_directory=output_directory,
dry_run=dry_run,
expected_download_summary_file_name="plugins/test_audio_extract_single_old_format.json",
)
@pytest.mark.parametrize("dry_run", [False])
def test_audio_extract_single_song(
self,

View file

@ -14,30 +14,6 @@ from ytdl_sub.utils.system import IS_WINDOWS
from ytdl_sub.utils.thumbnail import try_convert_download_thumbnail
@pytest.fixture
def single_video_preset_dict_old_video_tags_format(output_directory):
return {
"preset": "Jellyfin Music Videos",
"download": "https://youtube.com/watch?v=HKTNxEqsN3Q",
# override the output directory with our fixture-generated dir
"output_options": {
"maintain_download_archive": False,
},
"embed_thumbnail": True, # embed thumb into the video
"format": "worst[ext=mp4]", # download the worst format so it is fast
# also test video tags
"video_tags": {
"tags": {
"title": "{title}",
}
},
"overrides": {
"music_video_artist": "JMC",
"music_video_directory": output_directory,
},
}
@pytest.fixture
def single_video_preset_dict(output_directory):
return {
@ -100,25 +76,6 @@ def single_video_preset_dict_dl_args(single_video_preset_dict):
class TestYoutubeVideo:
def test_single_video_old_video_tags_format_download(
self,
default_config,
single_video_preset_dict_old_video_tags_format,
output_directory,
):
single_video_subscription = Subscription.from_dict(
config=default_config,
preset_name="music_video_single_video_test",
preset_dict=single_video_preset_dict_old_video_tags_format,
)
transaction_log = single_video_subscription.download(dry_run=True)
assert_transaction_log_matches(
output_directory=output_directory,
transaction_log=transaction_log,
transaction_log_summary_file_name="youtube/test_video.txt",
)
@pytest.mark.parametrize("dry_run", [True, False])
def test_single_video_download(
self,

View file

@ -1,6 +1,6 @@
{
".ytdl-sub-match_filter_test-download-archive.json": "30f10646149d9eea4eb970749f352f7d",
"match_filter_test/Jesse's Minecraft Server [Trailer - Mar.21].jpg": "e7830aa8a64b0cde65ba3f7e5fc56530",
"match_filter_test/Jesse's Minecraft Server [Trailer - Mar.21].mp4": "70204418e9af11a696611aa19571cf3a",
"match_filter_test/Jesse's Minecraft Server [Trailer - Mar.21].mp4": "f17a540070964a199b35f981561d94e4",
"match_filter_test/Jesse's Minecraft Server [Trailer - Mar.21].nfo": "d85f4500bb5d8a2425d734a23b5a944c"
}

View file

@ -1,6 +1,6 @@
{
".ytdl-sub-multiple_songs_test-download-archive.json": "54237df5e00d1598dfd39f341ee03d75",
"Project Zombie/[2011] Jesse's Minecraft Server/01 - Jesse's Minecraft Server [Trailer - Mar.21].ogg": "5657c5b92f8980b20d8bbee0fdc7e5d8",
"Project Zombie/[2011] Jesse's Minecraft Server/01 - Jesse's Minecraft Server [Trailer - Mar.21].ogg": "93665cc4302010d9f20001f308c4978a",
"Project Zombie/[2011] Jesse's Minecraft Server/02 - Jesse's Minecraft Server [Trailer - Feb.27].ogg": "a2a3a34e02e26a6c0265530d4499473b",
"Project Zombie/[2011] Jesse's Minecraft Server/03 - Jesse's Minecraft Server [Trailer - Feb.1].ogg": "0a385da3aa06b994a69b8ab812b44975",
"Project Zombie/[2011] Jesse's Minecraft Server/folder.jpg": "e7830aa8a64b0cde65ba3f7e5fc56530"

View file

@ -1,5 +0,0 @@
{
".ytdl-sub-single_song_test-download-archive.json": "c8ff22ec3304c9f8dab18cedaed4e8b4",
"YouTube/[2019] YouTube Rewind 2019 For the Record #YouTubeRewind/01 - YouTube Rewind 2019 For the Record #YouTubeRewind.mp3": "11376667a11bb71565b520f8ce5fa303",
"YouTube/[2019] YouTube Rewind 2019 For the Record #YouTubeRewind/folder.jpg": "50ee47c80f679029f5d3503bb91b045a"
}

View file

@ -18,7 +18,7 @@
"Project Zombie/Season 2011/s2011.e022701 - Jesse's Minecraft Server [Trailer - Feb.27].nfo": "20af231e30a035fb2bc0c946f4b4026d",
"Project Zombie/Season 2011/s2011.e032101 - Jesse's Minecraft Server [Trailer - Mar.21]-thumb.jpg": "e7830aa8a64b0cde65ba3f7e5fc56530",
"Project Zombie/Season 2011/s2011.e032101 - Jesse's Minecraft Server [Trailer - Mar.21].info.json": "INFO_JSON",
"Project Zombie/Season 2011/s2011.e032101 - Jesse's Minecraft Server [Trailer - Mar.21].mp4": "5f12b36e5ce717fa9550000237e2e5b8",
"Project Zombie/Season 2011/s2011.e032101 - Jesse's Minecraft Server [Trailer - Mar.21].mp4": "eac3dfce44c2a723f2fc74e9552aa510",
"Project Zombie/Season 2011/s2011.e032101 - Jesse's Minecraft Server [Trailer - Mar.21].nfo": "f8bdc463c0cb2ffdc82aba2bcc27ad5d",
"Project Zombie/Season 2011/s2011.e052901 - Project Zombie Official Trailer (IP mc.projectzombie.beastnode.net)-thumb.jpg": "c956192a379b3661595c9920972d4819",
"Project Zombie/Season 2011/s2011.e052901 - Project Zombie Official Trailer (IP mc.projectzombie.beastnode.net).info.json": "INFO_JSON",

View file

@ -10,7 +10,7 @@
"JMC/Season 01/s01.e11022701 - Jesse's Minecraft Server [Trailer - Feb.27].nfo": "da7645e8826586388ae0d8278ef6a1c1",
"JMC/Season 01/s01.e11032101 - Jesse's Minecraft Server [Trailer - Mar.21]-thumb.jpg": "e7830aa8a64b0cde65ba3f7e5fc56530",
"JMC/Season 01/s01.e11032101 - Jesse's Minecraft Server [Trailer - Mar.21].info.json": "INFO_JSON",
"JMC/Season 01/s01.e11032101 - Jesse's Minecraft Server [Trailer - Mar.21].mp4": "7f1a1d5c1d94938a9b6f565e8159c3d6",
"JMC/Season 01/s01.e11032101 - Jesse's Minecraft Server [Trailer - Mar.21].mp4": "dc04853870bbb811f6b312e20b58253b",
"JMC/Season 01/s01.e11032101 - Jesse's Minecraft Server [Trailer - Mar.21].nfo": "c56083e2f3545fa2cafc4d67cbfdacf8",
"JMC/fanart.jpg": "129c6639b47299bc48062f0365e670ee",
"JMC/poster.jpg": "5de28eea5a921a041452ab3ce1041f73",

View file

@ -10,7 +10,7 @@
"JMC/Season 01/s01.e11022701 - Jesse's Minecraft Server [Trailer - Feb.27].nfo": "da7645e8826586388ae0d8278ef6a1c1",
"JMC/Season 01/s01.e11032101 - Jesse's Minecraft Server [Trailer - Mar.21]-thumb.jpg": "e7830aa8a64b0cde65ba3f7e5fc56530",
"JMC/Season 01/s01.e11032101 - Jesse's Minecraft Server [Trailer - Mar.21].info.json": "INFO_JSON",
"JMC/Season 01/s01.e11032101 - Jesse's Minecraft Server [Trailer - Mar.21].mp4": "7f1a1d5c1d94938a9b6f565e8159c3d6",
"JMC/Season 01/s01.e11032101 - Jesse's Minecraft Server [Trailer - Mar.21].mp4": "dc04853870bbb811f6b312e20b58253b",
"JMC/Season 01/s01.e11032101 - Jesse's Minecraft Server [Trailer - Mar.21].nfo": "c56083e2f3545fa2cafc4d67cbfdacf8",
"JMC/fanart.jpg": "129c6639b47299bc48062f0365e670ee",
"JMC/poster.jpg": "5de28eea5a921a041452ab3ce1041f73",

View file

@ -1,18 +0,0 @@
Files created:
----------------------------------------
{output_directory}
.ytdl-sub-single_song_test-download-archive.json
{output_directory}/YouTube/[2019] YouTube Rewind 2019 For the Record #YouTubeRewind
01 - YouTube Rewind 2019 For the Record #YouTubeRewind.mp3
Embedded Thumbnail, Music Tags:
album: YouTube Rewind 2019: For the Record | #YouTubeRewind
albumartist: YouTube
albumartists: YouTube
artist: YouTube
artists: YouTube
genres: Unset
title: YouTube Rewind 2019: For the Record | #YouTubeRewind
track: 1
tracktotal: 1
year: 2019
folder.jpg

View file

@ -17,9 +17,6 @@ class TestPreset:
"youtube.com/watch?v=123abc", # single string
["youtube.com/watch?v=123abc", "youtube.com/watch?v=123xyz"], # list of strings
[{"url": "youtube.com/watch?v=123abc"}, "youtube.com/watch?v=123abc"], # dict and str
# OLD download_strategy format
{"download_strategy": "url", "url": "youtube.com/watch?v=123abc"},
{"download_strategy": "multi-url", "urls": [{"url": "youtube.com/watch?v=123abc"}]},
],
)
def test_bare_minimum_preset(self, config_file, output_options, download_value):