[FEATURE+BUGFIX] Add kodi_safe arg for NFO plugins, support YT shorts (#157)

src/ytdl_sub/plugins/nfo_tags.py

@@ -1,14 +1,17 @@
 import os
 from pathlib import Path
-
+from typing import Optional
-import dicttoxml
 
 from ytdl_sub.entries.entry import Entry
 from ytdl_sub.plugins.plugin import Plugin
 from ytdl_sub.plugins.plugin import PluginOptions
 from ytdl_sub.utils.file_handler import FileMetadata
+from ytdl_sub.utils.xml import to_max_3_byte_utf8_dict
+from ytdl_sub.utils.xml import to_max_3_byte_utf8_string
+from ytdl_sub.utils.xml import to_xml
 from ytdl_sub.validators.string_formatter_validators import DictFormatterValidator
 from ytdl_sub.validators.string_formatter_validators import StringFormatterValidator
+from ytdl_sub.validators.validators import BoolValidator
 
 
 class NfoTagsOptions(PluginOptions):
@@ -23,15 +26,19 @@ class NfoTagsOptions(PluginOptions):
        presets:
          my_example_preset:
            nfo:
-            nfo_name: "{title_sanitized}.nfo"
+             # required
-            nfo_root: "episodedetails"
+             nfo_name: "{title_sanitized}.nfo"
-            tags:
+             nfo_root: "episodedetails"
-              title: "{title}"
+             tags:
-              season: "{upload_year}"
+               title: "{title}"
-              episode: "{upload_month}{upload_day_padded}"
+               season: "{upload_year}"
+               episode: "{upload_month}{upload_day_padded}"
+             # optional
+             kodi_safe: False
     """
 
     _required_keys = {"nfo_name", "nfo_root", "tags"}
+    _optional_keys = {"kodi_safe"}
 
     def __init__(self, name, value):
         super().__init__(name, value)
@@ -39,6 +46,9 @@ class NfoTagsOptions(PluginOptions):
         self._nfo_name = self._validate_key(key="nfo_name", validator=StringFormatterValidator)
         self._nfo_root = self._validate_key(key="nfo_root", validator=StringFormatterValidator)
         self._tags = self._validate_key(key="tags", validator=DictFormatterValidator)
+        self._kodi_safe = self._validate_key_if_present(
+            key="kodi_safe", validator=BoolValidator, default=False
+        ).value
 
     @property
     def nfo_name(self) -> StringFormatterValidator:
@@ -76,6 +86,15 @@ class NfoTagsOptions(PluginOptions):
         """
         return self._tags
 
+    @property
+    def kodi_safe(self) -> Optional[bool]:
+        """
+        Optional. Kodi does not support > 3-byte unicode characters, which include emojis and some
+        foreign language characters. Setting this to True will replace those characters with '□'.
+        Defaults to False.
+        """
+        return self._kodi_safe
+
 
 class NfoTagsPlugin(Plugin[NfoTagsOptions]):
     plugin_options_type = NfoTagsOptions
@@ -98,12 +117,12 @@ class NfoTagsPlugin(Plugin[NfoTagsOptions]):
         nfo_root = self.overrides.apply_formatter(
             formatter=self.plugin_options.nfo_root, entry=entry
         )
-        xml = dicttoxml.dicttoxml(
+
-            obj=nfo,
+        if self.plugin_options.kodi_safe:
-            root=True,  # We assume all NFOs have a root. Maybe we should not?
+            nfo = to_max_3_byte_utf8_dict(nfo)
-            custom_root=nfo_root,
+            nfo_root = to_max_3_byte_utf8_string(nfo_root)
-            attr_type=False,
+
-        )
+        xml = to_xml(nfo_dict=nfo, nfo_root=nfo_root)
 
         nfo_file_name = self.overrides.apply_formatter(
             formatter=self.plugin_options.nfo_name, entry=entry

src/ytdl_sub/plugins/output_directory_nfo_tags.py

@@ -1,13 +1,16 @@
 import os
 from pathlib import Path
-
+from typing import Optional
-import dicttoxml
 
 from ytdl_sub.plugins.plugin import Plugin
 from ytdl_sub.plugins.plugin import PluginOptions
 from ytdl_sub.utils.file_handler import FileMetadata
+from ytdl_sub.utils.xml import to_max_3_byte_utf8_dict
+from ytdl_sub.utils.xml import to_max_3_byte_utf8_string
+from ytdl_sub.utils.xml import to_xml
 from ytdl_sub.validators.string_formatter_validators import OverridesDictFormatterValidator
 from ytdl_sub.validators.string_formatter_validators import OverridesStringFormatterValidator
+from ytdl_sub.validators.validators import BoolValidator
 
 
 class OutputDirectoryNfoTagsOptions(PluginOptions):
@@ -22,13 +25,17 @@ class OutputDirectoryNfoTagsOptions(PluginOptions):
        presets:
          my_example_preset:
            output_directory_nfo_tags:
+             # required
              nfo_name: "tvshow.nfo"
              nfo_root: "tvshow"
              tags:
                title: "Sweet youtube TV show"
+             # optional
+             kodi_safe: False
     """
 
     _required_keys = {"nfo_name", "nfo_root", "tags"}
+    _optional_keys = {"kodi_safe"}
 
     def __init__(self, name, value):
         super().__init__(name, value)
@@ -41,6 +48,9 @@ class OutputDirectoryNfoTagsOptions(PluginOptions):
             key="nfo_root", validator=OverridesStringFormatterValidator
         )
         self._tags = self._validate_key(key="tags", validator=OverridesDictFormatterValidator)
+        self._kodi_safe = self._validate_key_if_present(
+            key="kodi_safe", validator=BoolValidator, default=False
+        ).value
 
     @property
     def nfo_name(self) -> OverridesStringFormatterValidator:
@@ -76,6 +86,15 @@ class OutputDirectoryNfoTagsOptions(PluginOptions):
         """
         return self._tags
 
+    @property
+    def kodi_safe(self) -> Optional[bool]:
+        """
+        Optional. Kodi does not support > 3-byte unicode characters, which include emojis and some
+        foreign language characters. Setting this to True will replace those characters with '□'.
+        Defaults to False.
+        """
+        return self._kodi_safe
+
 
 class OutputDirectoryNfoTagsPlugin(Plugin[OutputDirectoryNfoTagsOptions]):
     plugin_options_type = OutputDirectoryNfoTagsOptions
@@ -91,13 +110,12 @@ class OutputDirectoryNfoTagsPlugin(Plugin[OutputDirectoryNfoTagsOptions]):
 
         # Write the nfo tags to XML with the nfo_root
         nfo_root = self.overrides.apply_formatter(formatter=self.plugin_options.nfo_root)
-        xml = dicttoxml.dicttoxml(
-            obj=nfo,
-            root=True,  # We assume all NFOs have a root. Maybe we should not?
-            custom_root=nfo_root,
-            attr_type=False,
-        )
 
+        if self.plugin_options.kodi_safe:
+            nfo = to_max_3_byte_utf8_dict(nfo)
+            nfo_root = to_max_3_byte_utf8_string(nfo_root)
+
+        xml = to_xml(nfo_dict=nfo, nfo_root=nfo_root)
         nfo_file_name = self.overrides.apply_formatter(formatter=self.plugin_options.nfo_name)
 
         # Save the nfo's XML to file

src/ytdl_sub/utils/xml.py

@@ -0,0 +1,61 @@
+from typing import Dict
+
+import dicttoxml
+
+
+def _to_max_3_byte_utf8_char(char: str) -> str:
+    return "□" if len(char.encode("utf-8")) > 3 else char
+
+
+def to_max_3_byte_utf8_string(string: str) -> str:
+    """
+    Parameters
+    ----------
+    string
+        input string
+
+    Returns
+    -------
+    Casted unicode string
+    """
+    return "".join(_to_max_3_byte_utf8_char(char) for char in string)
+
+
+def to_max_3_byte_utf8_dict(string_dict: Dict[str, str]) -> Dict[str, str]:
+    """
+    Parameters
+    ----------
+    string_dict
+        Input string dict
+
+    Returns
+    -------
+    Casted dict
+    """
+    return {
+        to_max_3_byte_utf8_string(key): to_max_3_byte_utf8_string(value)
+        for key, value in string_dict.items()
+    }
+
+
+def to_xml(nfo_dict: Dict[str, str], nfo_root: str) -> bytes:
+    """
+    Transforms a dict to XML
+
+    Parameters
+    ----------
+    nfo_dict
+        XML contents
+    nfo_root
+        Root of the XML
+
+    Returns
+    -------
+    XML bytes
+    """
+    return dicttoxml.dicttoxml(
+        obj=nfo_dict,
+        root=True,
+        custom_root=nfo_root,
+        attr_type=False,
+    )

src/ytdl_sub/validators/url_validator.py

@@ -14,10 +14,16 @@ class YoutubeVideoUrlValidator(StringValidator):
     def _get_video_id(cls, url: str) -> Optional[str]:
         """
         Examples:
-        - https://youtu.be/SA2iWivDJiE
+
-        - https://www.youtube.com/watch?v=_oPAwA_Udwc&feature=feedu
+        https://youtu.be/SA2iWivDJiE
-        - https://www.youtube.com/embed/SA2iWivDJiE
+        https://www.youtube.com/watch?v=_oPAwA_Udwc&feature=feedu
-        - https://www.youtube.com/v/SA2iWivDJiE?version=3&hl=en_US
+        https://www.youtube.com/embed/SA2iWivDJiE
+        https://www.youtube.com/v/SA2iWivDJiE?version=3&hl=en_US
+        https://www.youtube.com/shorts/ucYmEqmlhFw
+
+        Returns
+        -------
+        The video id if it is parsed correctly. None if parsing fails
         """
         # If the url doesn't contain youtube or youtu.be, assume it is invalid
         if "youtube.com" not in url and "youtu.be" not in url:
@@ -35,9 +41,7 @@ class YoutubeVideoUrlValidator(StringValidator):
                 parsed_q = parse_qs(query.query)
                 if "v" in parsed_q:
                     return parsed_q["v"][0]
-            if query.path[:7] == "/embed/":
+            if query.path.startswith(("/embed/", "/v/", "/shorts/")):
-                return query.path.split("/")[2]
-            if query.path[:3] == "/v/":
                 return query.path.split("/")[2]
 
         return None

tests/e2e/plugins/test_nfo_tags.py

@@ -0,0 +1,48 @@
+import pytest
+from e2e.expected_transaction_log import assert_transaction_log_matches
+
+from ytdl_sub.subscriptions.subscription import Subscription
+
+
+@pytest.fixture
+def kodi_safe_subscription_dict(output_directory):
+    return {
+        "preset": "yt_music_video",
+        "youtube": {"video_url": "https://www.youtube.com/shorts/ucYmEqmlhFw"},
+        # override the output directory with our fixture-generated dir
+        "output_options": {"output_directory": output_directory},
+        # download the worst format so it is fast
+        "ytdl_options": {
+            "format": "best[height<=480]",
+        },
+        "nfo_tags": {
+            "tags": {
+                "kodi_safe_title 🎸": "{title}",
+            },
+            "kodi_safe": True,
+        },
+        "output_directory_nfo_tags": {
+            "nfo_name": "test.nfo",
+            "nfo_root": "kodi_safe_root 🎸",
+            "tags": {"kodi_safe_title 🎸": "kodi_safe_value 🎸"},
+            "kodi_safe": True,
+        },
+    }
+
+
+class TestNfoTagsPlugins:
+    def test_kodi_safe(self, kodi_safe_subscription_dict, music_video_config, output_directory):
+        subscription = Subscription.from_dict(
+            config=music_video_config,
+            preset_name="kodi_safe_xml",
+            preset_dict=kodi_safe_subscription_dict,
+        )
+
+        # Only dry run is needed to see if NFO values are kodi safe
+        transaction_log = subscription.download(dry_run=True)
+        assert_transaction_log_matches(
+            output_directory=output_directory,
+            transaction_log=transaction_log,
+            transaction_log_summary_file_name="plugins/test_kodi_safe_xml.txt",
+            regenerate_transaction_log=True,
+        )

tests/e2e/resources/transaction_log_summaries/plugins/test_kodi_safe_xml.txt

@@ -0,0 +1,16 @@
+Files created in '{output_directory}'
+----------------------------------------
+Rick Beato - Can you hear the difference 🎸🔥 #shorts-thumb.jpg
+Rick Beato - Can you hear the difference 🎸🔥 #shorts.3gp
+Rick Beato - Can you hear the difference 🎸🔥 #shorts.nfo
+  NFO tags:
+    musicvideo:
+      album: Music Videos
+      artist: Rick Beato
+      kodi_safe_title □: Can you hear the difference? □□ #shorts
+      title: Can you hear the difference? □□ #shorts
+      year: 2022
+test.nfo
+  NFO tags:
+    kodi_safe_root □:
+      kodi_safe_title □: kodi_safe_value □

tests/unit/validators/test_url_validator.py

@@ -20,6 +20,7 @@ class TestYoutubeVideoUrlValidator:
             "https://www.youtube.com/watch?v=dQw4w9WgXcQ&feature=feedu",
             "https://www.youtube.com/embed/dQw4w9WgXcQ",
             "https://www.youtube.com/v/dQw4w9WgXcQ?version=3&hl=en_US",
+            "https://www.youtube.com/shorts/dQw4w9WgXcQ",
         ],
     )
     def test_youtube_video_url_validator_success(self, url):
@@ -36,6 +37,7 @@ class TestYoutubeVideoUrlValidator:
             "youtube.com/watch?v=",
             "youtu.be/",
             "youtu.be",
+            "youtube.com/shorts",
         ],
     )
     def test_youtube_video_url_validator_fail(self, bad_url):