danvics/ytdl-sub
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

[FEATURE+BUGFIX] Add kodi_safe arg for NFO plugins, support YT shorts (#157)

Browse source
This commit is contained in:
Jesse Bannon 2022-08-05 16:53:36 -07:00 committed by GitHub
parent 7dfbb043fa
commit 034855fbc4
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
7 changed files with 197 additions and 29 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

47
src/ytdl_sub/plugins/nfo_tags.py
View file

@ -1,14 +1,17 @@
import os
from pathlib import Path
import dicttoxml
from typing import Optional
from ytdl_sub.entries.entry import Entry
from ytdl_sub.plugins.plugin import Plugin
from ytdl_sub.plugins.plugin import PluginOptions
from ytdl_sub.utils.file_handler import FileMetadata
from ytdl_sub.utils.xml import to_max_3_byte_utf8_dict
from ytdl_sub.utils.xml import to_max_3_byte_utf8_string
from ytdl_sub.utils.xml import to_xml
from ytdl_sub.validators.string_formatter_validators import DictFormatterValidator
from ytdl_sub.validators.string_formatter_validators import StringFormatterValidator
from ytdl_sub.validators.validators import BoolValidator
class NfoTagsOptions(PluginOptions):
@ -23,15 +26,19 @@ class NfoTagsOptions(PluginOptions):
presets:
my_example_preset:
nfo:
nfo_name: "{title_sanitized}.nfo"
nfo_root: "episodedetails"
tags:
title: "{title}"
season: "{upload_year}"
episode: "{upload_month}{upload_day_padded}"
# required
nfo_name: "{title_sanitized}.nfo"
nfo_root: "episodedetails"
tags:
title: "{title}"
season: "{upload_year}"
episode: "{upload_month}{upload_day_padded}"
# optional
kodi_safe: False
"""
_required_keys = {"nfo_name", "nfo_root", "tags"}
_optional_keys = {"kodi_safe"}
def __init__(self, name, value):
super().__init__(name, value)
@ -39,6 +46,9 @@ class NfoTagsOptions(PluginOptions):
self._nfo_name = self._validate_key(key="nfo_name", validator=StringFormatterValidator)
self._nfo_root = self._validate_key(key="nfo_root", validator=StringFormatterValidator)
self._tags = self._validate_key(key="tags", validator=DictFormatterValidator)
self._kodi_safe = self._validate_key_if_present(
key="kodi_safe", validator=BoolValidator, default=False
).value
@property
def nfo_name(self) -> StringFormatterValidator:
@ -76,6 +86,15 @@ class NfoTagsOptions(PluginOptions):
"""
return self._tags
@property
def kodi_safe(self) -> Optional[bool]:
"""
Optional. Kodi does not support > 3-byte unicode characters, which include emojis and some
foreign language characters. Setting this to True will replace those characters with '□'.
Defaults to False.
"""
return self._kodi_safe
class NfoTagsPlugin(Plugin[NfoTagsOptions]):
plugin_options_type = NfoTagsOptions
@ -98,12 +117,12 @@ class NfoTagsPlugin(Plugin[NfoTagsOptions]):
nfo_root = self.overrides.apply_formatter(
formatter=self.plugin_options.nfo_root, entry=entry
)
xml = dicttoxml.dicttoxml(
obj=nfo,
root=True, # We assume all NFOs have a root. Maybe we should not?
custom_root=nfo_root,
attr_type=False,
)
if self.plugin_options.kodi_safe:
nfo = to_max_3_byte_utf8_dict(nfo)
nfo_root = to_max_3_byte_utf8_string(nfo_root)
xml = to_xml(nfo_dict=nfo, nfo_root=nfo_root)
nfo_file_name = self.overrides.apply_formatter(
formatter=self.plugin_options.nfo_name, entry=entry

34
src/ytdl_sub/plugins/output_directory_nfo_tags.py
View file

@ -1,13 +1,16 @@
import os
from pathlib import Path
import dicttoxml
from typing import Optional
from ytdl_sub.plugins.plugin import Plugin
from ytdl_sub.plugins.plugin import PluginOptions
from ytdl_sub.utils.file_handler import FileMetadata
from ytdl_sub.utils.xml import to_max_3_byte_utf8_dict
from ytdl_sub.utils.xml import to_max_3_byte_utf8_string
from ytdl_sub.utils.xml import to_xml
from ytdl_sub.validators.string_formatter_validators import OverridesDictFormatterValidator
from ytdl_sub.validators.string_formatter_validators import OverridesStringFormatterValidator
from ytdl_sub.validators.validators import BoolValidator
class OutputDirectoryNfoTagsOptions(PluginOptions):
@ -22,13 +25,17 @@ class OutputDirectoryNfoTagsOptions(PluginOptions):
presets:
my_example_preset:
output_directory_nfo_tags:
# required
nfo_name: "tvshow.nfo"
nfo_root: "tvshow"
tags:
title: "Sweet youtube TV show"
# optional
kodi_safe: False
"""
_required_keys = {"nfo_name", "nfo_root", "tags"}
_optional_keys = {"kodi_safe"}
def __init__(self, name, value):
super().__init__(name, value)
@ -41,6 +48,9 @@ class OutputDirectoryNfoTagsOptions(PluginOptions):
key="nfo_root", validator=OverridesStringFormatterValidator
)
self._tags = self._validate_key(key="tags", validator=OverridesDictFormatterValidator)
self._kodi_safe = self._validate_key_if_present(
key="kodi_safe", validator=BoolValidator, default=False
).value
@property
def nfo_name(self) -> OverridesStringFormatterValidator:
@ -76,6 +86,15 @@ class OutputDirectoryNfoTagsOptions(PluginOptions):
"""
return self._tags
@property
def kodi_safe(self) -> Optional[bool]:
"""
Optional. Kodi does not support > 3-byte unicode characters, which include emojis and some
foreign language characters. Setting this to True will replace those characters with '□'.
Defaults to False.
"""
return self._kodi_safe
class OutputDirectoryNfoTagsPlugin(Plugin[OutputDirectoryNfoTagsOptions]):
plugin_options_type = OutputDirectoryNfoTagsOptions
@ -91,13 +110,12 @@ class OutputDirectoryNfoTagsPlugin(Plugin[OutputDirectoryNfoTagsOptions]):
# Write the nfo tags to XML with the nfo_root
nfo_root = self.overrides.apply_formatter(formatter=self.plugin_options.nfo_root)
xml = dicttoxml.dicttoxml(
obj=nfo,
root=True, # We assume all NFOs have a root. Maybe we should not?
custom_root=nfo_root,
attr_type=False,
)
if self.plugin_options.kodi_safe:
nfo = to_max_3_byte_utf8_dict(nfo)
nfo_root = to_max_3_byte_utf8_string(nfo_root)
xml = to_xml(nfo_dict=nfo, nfo_root=nfo_root)
nfo_file_name = self.overrides.apply_formatter(formatter=self.plugin_options.nfo_name)
# Save the nfo's XML to file

61
src/ytdl_sub/utils/xml.py Normal file
View file

@ -0,0 +1,61 @@
from typing import Dict
import dicttoxml
def _to_max_3_byte_utf8_char(char: str) -> str:
return "□" if len(char.encode("utf-8")) > 3 else char
def to_max_3_byte_utf8_string(string: str) -> str:
"""
Parameters
----------
string
input string
Returns
-------
Casted unicode string
"""
return "".join(_to_max_3_byte_utf8_char(char) for char in string)
def to_max_3_byte_utf8_dict(string_dict: Dict[str, str]) -> Dict[str, str]:
"""
Parameters
----------
string_dict
Input string dict
Returns
-------
Casted dict
"""
return {
to_max_3_byte_utf8_string(key): to_max_3_byte_utf8_string(value)
for key, value in string_dict.items()
}
def to_xml(nfo_dict: Dict[str, str], nfo_root: str) -> bytes:
"""
Transforms a dict to XML
Parameters
----------
nfo_dict
XML contents
nfo_root
Root of the XML
Returns
-------
XML bytes
"""
return dicttoxml.dicttoxml(
obj=nfo_dict,
root=True,
custom_root=nfo_root,
attr_type=False,
)

18
src/ytdl_sub/validators/url_validator.py
View file

@ -14,10 +14,16 @@ class YoutubeVideoUrlValidator(StringValidator):
def _get_video_id(cls, url: str) -> Optional[str]:
"""
Examples:
- https://youtu.be/SA2iWivDJiE
- https://www.youtube.com/watch?v=_oPAwA_Udwc&feature=feedu
- https://www.youtube.com/embed/SA2iWivDJiE
- https://www.youtube.com/v/SA2iWivDJiE?version=3&hl=en_US
https://youtu.be/SA2iWivDJiE
https://www.youtube.com/watch?v=_oPAwA_Udwc&feature=feedu
https://www.youtube.com/embed/SA2iWivDJiE
https://www.youtube.com/v/SA2iWivDJiE?version=3&hl=en_US
https://www.youtube.com/shorts/ucYmEqmlhFw
Returns
-------
The video id if it is parsed correctly. None if parsing fails
"""
# If the url doesn't contain youtube or youtu.be, assume it is invalid
if "youtube.com" not in url and "youtu.be" not in url:
@ -35,9 +41,7 @@ class YoutubeVideoUrlValidator(StringValidator):
parsed_q = parse_qs(query.query)
if "v" in parsed_q:
return parsed_q["v"][0]
if query.path[:7] == "/embed/":
return query.path.split("/")[2]
if query.path[:3] == "/v/":
if query.path.startswith(("/embed/", "/v/", "/shorts/")):
return query.path.split("/")[2]
return None

48
tests/e2e/plugins/test_nfo_tags.py Normal file
View file

@ -0,0 +1,48 @@
import pytest
from e2e.expected_transaction_log import assert_transaction_log_matches
from ytdl_sub.subscriptions.subscription import Subscription
@pytest.fixture
def kodi_safe_subscription_dict(output_directory):
return {
"preset": "yt_music_video",
"youtube": {"video_url": "https://www.youtube.com/shorts/ucYmEqmlhFw"},
# override the output directory with our fixture-generated dir
"output_options": {"output_directory": output_directory},
# download the worst format so it is fast
"ytdl_options": {
"format": "best[height<=480]",
},
"nfo_tags": {
"tags": {
"kodi_safe_title 🎸": "{title}",
},
"kodi_safe": True,
},
"output_directory_nfo_tags": {
"nfo_name": "test.nfo",
"nfo_root": "kodi_safe_root 🎸",
"tags": {"kodi_safe_title 🎸": "kodi_safe_value 🎸"},
"kodi_safe": True,
},
}
class TestNfoTagsPlugins:
def test_kodi_safe(self, kodi_safe_subscription_dict, music_video_config, output_directory):
subscription = Subscription.from_dict(
config=music_video_config,
preset_name="kodi_safe_xml",
preset_dict=kodi_safe_subscription_dict,
)
# Only dry run is needed to see if NFO values are kodi safe
transaction_log = subscription.download(dry_run=True)
assert_transaction_log_matches(
output_directory=output_directory,
transaction_log=transaction_log,
transaction_log_summary_file_name="plugins/test_kodi_safe_xml.txt",
regenerate_transaction_log=True,
)

16
tests/e2e/resources/transaction_log_summaries/plugins/test_kodi_safe_xml.txt Normal file
View file

@ -0,0 +1,16 @@
Files created in '{output_directory}'
----------------------------------------
Rick Beato - Can you hear the difference 🎸🔥 #shorts-thumb.jpg
Rick Beato - Can you hear the difference 🎸🔥 #shorts.3gp
Rick Beato - Can you hear the difference 🎸🔥 #shorts.nfo
NFO tags:
musicvideo:
album: Music Videos
artist: Rick Beato
kodi_safe_title □: Can you hear the difference? □□ #shorts
title: Can you hear the difference? □□ #shorts
year: 2022
test.nfo
NFO tags:
kodi_safe_root □:
kodi_safe_title □: kodi_safe_value □

2
tests/unit/validators/test_url_validator.py
View file

@ -20,6 +20,7 @@ class TestYoutubeVideoUrlValidator:
"https://www.youtube.com/watch?v=dQw4w9WgXcQ&feature=feedu",
"https://www.youtube.com/embed/dQw4w9WgXcQ",
"https://www.youtube.com/v/dQw4w9WgXcQ?version=3&amp;hl=en_US",
"https://www.youtube.com/shorts/dQw4w9WgXcQ",
],
)
def test_youtube_video_url_validator_success(self, url):
@ -36,6 +37,7 @@ class TestYoutubeVideoUrlValidator:
"youtube.com/watch?v=",
"youtu.be/",
"youtu.be",
"youtube.com/shorts",
],
)
def test_youtube_video_url_validator_fail(self, bad_url):