From 034855fbc46ebe0f8897f6ea14ec7cbd5d7b19a2 Mon Sep 17 00:00:00 2001 From: Jesse Bannon Date: Fri, 5 Aug 2022 16:53:36 -0700 Subject: [PATCH] [FEATURE+BUGFIX] Add `kodi_safe` arg for NFO plugins, support YT shorts (#157) --- src/ytdl_sub/plugins/nfo_tags.py | 47 +++++++++----- .../plugins/output_directory_nfo_tags.py | 34 ++++++++--- src/ytdl_sub/utils/xml.py | 61 +++++++++++++++++++ src/ytdl_sub/validators/url_validator.py | 18 +++--- tests/e2e/plugins/test_nfo_tags.py | 48 +++++++++++++++ .../plugins/test_kodi_safe_xml.txt | 16 +++++ tests/unit/validators/test_url_validator.py | 2 + 7 files changed, 197 insertions(+), 29 deletions(-) create mode 100644 src/ytdl_sub/utils/xml.py create mode 100644 tests/e2e/plugins/test_nfo_tags.py create mode 100644 tests/e2e/resources/transaction_log_summaries/plugins/test_kodi_safe_xml.txt diff --git a/src/ytdl_sub/plugins/nfo_tags.py b/src/ytdl_sub/plugins/nfo_tags.py index 38d8d15b..828fe3ad 100644 --- a/src/ytdl_sub/plugins/nfo_tags.py +++ b/src/ytdl_sub/plugins/nfo_tags.py @@ -1,14 +1,17 @@ import os from pathlib import Path - -import dicttoxml +from typing import Optional from ytdl_sub.entries.entry import Entry from ytdl_sub.plugins.plugin import Plugin from ytdl_sub.plugins.plugin import PluginOptions from ytdl_sub.utils.file_handler import FileMetadata +from ytdl_sub.utils.xml import to_max_3_byte_utf8_dict +from ytdl_sub.utils.xml import to_max_3_byte_utf8_string +from ytdl_sub.utils.xml import to_xml from ytdl_sub.validators.string_formatter_validators import DictFormatterValidator from ytdl_sub.validators.string_formatter_validators import StringFormatterValidator +from ytdl_sub.validators.validators import BoolValidator class NfoTagsOptions(PluginOptions): @@ -23,15 +26,19 @@ class NfoTagsOptions(PluginOptions): presets: my_example_preset: nfo: - nfo_name: "{title_sanitized}.nfo" - nfo_root: "episodedetails" - tags: - title: "{title}" - season: "{upload_year}" - episode: "{upload_month}{upload_day_padded}" + # required + nfo_name: "{title_sanitized}.nfo" + nfo_root: "episodedetails" + tags: + title: "{title}" + season: "{upload_year}" + episode: "{upload_month}{upload_day_padded}" + # optional + kodi_safe: False """ _required_keys = {"nfo_name", "nfo_root", "tags"} + _optional_keys = {"kodi_safe"} def __init__(self, name, value): super().__init__(name, value) @@ -39,6 +46,9 @@ class NfoTagsOptions(PluginOptions): self._nfo_name = self._validate_key(key="nfo_name", validator=StringFormatterValidator) self._nfo_root = self._validate_key(key="nfo_root", validator=StringFormatterValidator) self._tags = self._validate_key(key="tags", validator=DictFormatterValidator) + self._kodi_safe = self._validate_key_if_present( + key="kodi_safe", validator=BoolValidator, default=False + ).value @property def nfo_name(self) -> StringFormatterValidator: @@ -76,6 +86,15 @@ class NfoTagsOptions(PluginOptions): """ return self._tags + @property + def kodi_safe(self) -> Optional[bool]: + """ + Optional. Kodi does not support > 3-byte unicode characters, which include emojis and some + foreign language characters. Setting this to True will replace those characters with 'β–‘'. + Defaults to False. + """ + return self._kodi_safe + class NfoTagsPlugin(Plugin[NfoTagsOptions]): plugin_options_type = NfoTagsOptions @@ -98,12 +117,12 @@ class NfoTagsPlugin(Plugin[NfoTagsOptions]): nfo_root = self.overrides.apply_formatter( formatter=self.plugin_options.nfo_root, entry=entry ) - xml = dicttoxml.dicttoxml( - obj=nfo, - root=True, # We assume all NFOs have a root. Maybe we should not? - custom_root=nfo_root, - attr_type=False, - ) + + if self.plugin_options.kodi_safe: + nfo = to_max_3_byte_utf8_dict(nfo) + nfo_root = to_max_3_byte_utf8_string(nfo_root) + + xml = to_xml(nfo_dict=nfo, nfo_root=nfo_root) nfo_file_name = self.overrides.apply_formatter( formatter=self.plugin_options.nfo_name, entry=entry diff --git a/src/ytdl_sub/plugins/output_directory_nfo_tags.py b/src/ytdl_sub/plugins/output_directory_nfo_tags.py index 1d2a5dba..3b7a16a8 100644 --- a/src/ytdl_sub/plugins/output_directory_nfo_tags.py +++ b/src/ytdl_sub/plugins/output_directory_nfo_tags.py @@ -1,13 +1,16 @@ import os from pathlib import Path - -import dicttoxml +from typing import Optional from ytdl_sub.plugins.plugin import Plugin from ytdl_sub.plugins.plugin import PluginOptions from ytdl_sub.utils.file_handler import FileMetadata +from ytdl_sub.utils.xml import to_max_3_byte_utf8_dict +from ytdl_sub.utils.xml import to_max_3_byte_utf8_string +from ytdl_sub.utils.xml import to_xml from ytdl_sub.validators.string_formatter_validators import OverridesDictFormatterValidator from ytdl_sub.validators.string_formatter_validators import OverridesStringFormatterValidator +from ytdl_sub.validators.validators import BoolValidator class OutputDirectoryNfoTagsOptions(PluginOptions): @@ -22,13 +25,17 @@ class OutputDirectoryNfoTagsOptions(PluginOptions): presets: my_example_preset: output_directory_nfo_tags: + # required nfo_name: "tvshow.nfo" nfo_root: "tvshow" tags: title: "Sweet youtube TV show" + # optional + kodi_safe: False """ _required_keys = {"nfo_name", "nfo_root", "tags"} + _optional_keys = {"kodi_safe"} def __init__(self, name, value): super().__init__(name, value) @@ -41,6 +48,9 @@ class OutputDirectoryNfoTagsOptions(PluginOptions): key="nfo_root", validator=OverridesStringFormatterValidator ) self._tags = self._validate_key(key="tags", validator=OverridesDictFormatterValidator) + self._kodi_safe = self._validate_key_if_present( + key="kodi_safe", validator=BoolValidator, default=False + ).value @property def nfo_name(self) -> OverridesStringFormatterValidator: @@ -76,6 +86,15 @@ class OutputDirectoryNfoTagsOptions(PluginOptions): """ return self._tags + @property + def kodi_safe(self) -> Optional[bool]: + """ + Optional. Kodi does not support > 3-byte unicode characters, which include emojis and some + foreign language characters. Setting this to True will replace those characters with 'β–‘'. + Defaults to False. + """ + return self._kodi_safe + class OutputDirectoryNfoTagsPlugin(Plugin[OutputDirectoryNfoTagsOptions]): plugin_options_type = OutputDirectoryNfoTagsOptions @@ -91,13 +110,12 @@ class OutputDirectoryNfoTagsPlugin(Plugin[OutputDirectoryNfoTagsOptions]): # Write the nfo tags to XML with the nfo_root nfo_root = self.overrides.apply_formatter(formatter=self.plugin_options.nfo_root) - xml = dicttoxml.dicttoxml( - obj=nfo, - root=True, # We assume all NFOs have a root. Maybe we should not? - custom_root=nfo_root, - attr_type=False, - ) + if self.plugin_options.kodi_safe: + nfo = to_max_3_byte_utf8_dict(nfo) + nfo_root = to_max_3_byte_utf8_string(nfo_root) + + xml = to_xml(nfo_dict=nfo, nfo_root=nfo_root) nfo_file_name = self.overrides.apply_formatter(formatter=self.plugin_options.nfo_name) # Save the nfo's XML to file diff --git a/src/ytdl_sub/utils/xml.py b/src/ytdl_sub/utils/xml.py new file mode 100644 index 00000000..98248a4b --- /dev/null +++ b/src/ytdl_sub/utils/xml.py @@ -0,0 +1,61 @@ +from typing import Dict + +import dicttoxml + + +def _to_max_3_byte_utf8_char(char: str) -> str: + return "β–‘" if len(char.encode("utf-8")) > 3 else char + + +def to_max_3_byte_utf8_string(string: str) -> str: + """ + Parameters + ---------- + string + input string + + Returns + ------- + Casted unicode string + """ + return "".join(_to_max_3_byte_utf8_char(char) for char in string) + + +def to_max_3_byte_utf8_dict(string_dict: Dict[str, str]) -> Dict[str, str]: + """ + Parameters + ---------- + string_dict + Input string dict + + Returns + ------- + Casted dict + """ + return { + to_max_3_byte_utf8_string(key): to_max_3_byte_utf8_string(value) + for key, value in string_dict.items() + } + + +def to_xml(nfo_dict: Dict[str, str], nfo_root: str) -> bytes: + """ + Transforms a dict to XML + + Parameters + ---------- + nfo_dict + XML contents + nfo_root + Root of the XML + + Returns + ------- + XML bytes + """ + return dicttoxml.dicttoxml( + obj=nfo_dict, + root=True, + custom_root=nfo_root, + attr_type=False, + ) diff --git a/src/ytdl_sub/validators/url_validator.py b/src/ytdl_sub/validators/url_validator.py index a5998763..828c2e3f 100644 --- a/src/ytdl_sub/validators/url_validator.py +++ b/src/ytdl_sub/validators/url_validator.py @@ -14,10 +14,16 @@ class YoutubeVideoUrlValidator(StringValidator): def _get_video_id(cls, url: str) -> Optional[str]: """ Examples: - - https://youtu.be/SA2iWivDJiE - - https://www.youtube.com/watch?v=_oPAwA_Udwc&feature=feedu - - https://www.youtube.com/embed/SA2iWivDJiE - - https://www.youtube.com/v/SA2iWivDJiE?version=3&hl=en_US + + https://youtu.be/SA2iWivDJiE + https://www.youtube.com/watch?v=_oPAwA_Udwc&feature=feedu + https://www.youtube.com/embed/SA2iWivDJiE + https://www.youtube.com/v/SA2iWivDJiE?version=3&hl=en_US + https://www.youtube.com/shorts/ucYmEqmlhFw + + Returns + ------- + The video id if it is parsed correctly. None if parsing fails """ # If the url doesn't contain youtube or youtu.be, assume it is invalid if "youtube.com" not in url and "youtu.be" not in url: @@ -35,9 +41,7 @@ class YoutubeVideoUrlValidator(StringValidator): parsed_q = parse_qs(query.query) if "v" in parsed_q: return parsed_q["v"][0] - if query.path[:7] == "/embed/": - return query.path.split("/")[2] - if query.path[:3] == "/v/": + if query.path.startswith(("/embed/", "/v/", "/shorts/")): return query.path.split("/")[2] return None diff --git a/tests/e2e/plugins/test_nfo_tags.py b/tests/e2e/plugins/test_nfo_tags.py new file mode 100644 index 00000000..a8c6ba29 --- /dev/null +++ b/tests/e2e/plugins/test_nfo_tags.py @@ -0,0 +1,48 @@ +import pytest +from e2e.expected_transaction_log import assert_transaction_log_matches + +from ytdl_sub.subscriptions.subscription import Subscription + + +@pytest.fixture +def kodi_safe_subscription_dict(output_directory): + return { + "preset": "yt_music_video", + "youtube": {"video_url": "https://www.youtube.com/shorts/ucYmEqmlhFw"}, + # override the output directory with our fixture-generated dir + "output_options": {"output_directory": output_directory}, + # download the worst format so it is fast + "ytdl_options": { + "format": "best[height<=480]", + }, + "nfo_tags": { + "tags": { + "kodi_safe_title 🎸": "{title}", + }, + "kodi_safe": True, + }, + "output_directory_nfo_tags": { + "nfo_name": "test.nfo", + "nfo_root": "kodi_safe_root 🎸", + "tags": {"kodi_safe_title 🎸": "kodi_safe_value 🎸"}, + "kodi_safe": True, + }, + } + + +class TestNfoTagsPlugins: + def test_kodi_safe(self, kodi_safe_subscription_dict, music_video_config, output_directory): + subscription = Subscription.from_dict( + config=music_video_config, + preset_name="kodi_safe_xml", + preset_dict=kodi_safe_subscription_dict, + ) + + # Only dry run is needed to see if NFO values are kodi safe + transaction_log = subscription.download(dry_run=True) + assert_transaction_log_matches( + output_directory=output_directory, + transaction_log=transaction_log, + transaction_log_summary_file_name="plugins/test_kodi_safe_xml.txt", + regenerate_transaction_log=True, + ) diff --git a/tests/e2e/resources/transaction_log_summaries/plugins/test_kodi_safe_xml.txt b/tests/e2e/resources/transaction_log_summaries/plugins/test_kodi_safe_xml.txt new file mode 100644 index 00000000..9353f88a --- /dev/null +++ b/tests/e2e/resources/transaction_log_summaries/plugins/test_kodi_safe_xml.txt @@ -0,0 +1,16 @@ +Files created in '{output_directory}' +---------------------------------------- +Rick Beato - Can you hear the difference 🎸πŸ”₯ #shorts-thumb.jpg +Rick Beato - Can you hear the difference 🎸πŸ”₯ #shorts.3gp +Rick Beato - Can you hear the difference 🎸πŸ”₯ #shorts.nfo + NFO tags: + musicvideo: + album: Music Videos + artist: Rick Beato + kodi_safe_title β–‘: Can you hear the difference? β–‘β–‘ #shorts + title: Can you hear the difference? β–‘β–‘ #shorts + year: 2022 +test.nfo + NFO tags: + kodi_safe_root β–‘: + kodi_safe_title β–‘: kodi_safe_value β–‘ \ No newline at end of file diff --git a/tests/unit/validators/test_url_validator.py b/tests/unit/validators/test_url_validator.py index 742646bd..e592e231 100644 --- a/tests/unit/validators/test_url_validator.py +++ b/tests/unit/validators/test_url_validator.py @@ -20,6 +20,7 @@ class TestYoutubeVideoUrlValidator: "https://www.youtube.com/watch?v=dQw4w9WgXcQ&feature=feedu", "https://www.youtube.com/embed/dQw4w9WgXcQ", "https://www.youtube.com/v/dQw4w9WgXcQ?version=3&hl=en_US", + "https://www.youtube.com/shorts/dQw4w9WgXcQ", ], ) def test_youtube_video_url_validator_success(self, url): @@ -36,6 +37,7 @@ class TestYoutubeVideoUrlValidator: "youtube.com/watch?v=", "youtu.be/", "youtu.be", + "youtube.com/shorts", ], ) def test_youtube_video_url_validator_fail(self, bad_url):