danvics/ytdl-sub
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

[FEATURE] NFO/XML attribute support (#198)

Browse source
This commit is contained in:
Jesse Bannon 2022-08-28 00:56:51 -07:00 committed by GitHub
parent d79e94e7e6
commit 9ea1be4c5f
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
7 changed files with 386 additions and 70 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

145
src/ytdl_sub/plugins/nfo_tags.py
View file

@ -1,6 +1,7 @@
import os
from abc import ABC
from pathlib import Path
from typing import Dict
from typing import Generic
from typing import Optional
from typing import Type
@ -10,21 +11,32 @@ from ytdl_sub.entries.entry import Entry
from ytdl_sub.plugins.plugin import Plugin
from ytdl_sub.plugins.plugin import PluginOptions
from ytdl_sub.utils.file_handler import FileMetadata
from ytdl_sub.utils.xml import XmlElement
from ytdl_sub.utils.xml import to_max_3_byte_utf8_dict
from ytdl_sub.utils.xml import to_max_3_byte_utf8_string
from ytdl_sub.utils.xml import to_xml
from ytdl_sub.validators.nfo_validators import NfoTagsValidator
from ytdl_sub.validators.nfo_validators import SharedNfoTagsValidator
from ytdl_sub.validators.nfo_validators import TDictFormatterValidator
from ytdl_sub.validators.nfo_validators import TStringFormatterValidator
from ytdl_sub.validators.string_formatter_validators import DictFormatterValidator
from ytdl_sub.validators.string_formatter_validators import StringFormatterValidator
from ytdl_sub.validators.validators import BoolValidator
TSharedNfoTagsValidator = TypeVar("TSharedNfoTagsValidator", bound=SharedNfoTagsValidator)
class SharedNfoTagsOptions(PluginOptions, ABC):
class SharedNfoTagsOptions(
PluginOptions,
Generic[TStringFormatterValidator, TDictFormatterValidator, TSharedNfoTagsValidator],
ABC,
):
"""
Shared code between NFO tags and Ouptut Directory NFO Tags
"""
_formatter_validator: Type[StringFormatterValidator]
_dict_formatter_validator: Type[DictFormatterValidator]
_formatter_validator: Type[TStringFormatterValidator]
_tags_validator: Type[TSharedNfoTagsValidator]
_required_keys = {"nfo_name", "nfo_root", "tags"}
_optional_keys = {"kodi_safe"}
@ -34,36 +46,93 @@ class SharedNfoTagsOptions(PluginOptions, ABC):
self._nfo_name = self._validate_key(key="nfo_name", validator=self._formatter_validator)
self._nfo_root = self._validate_key(key="nfo_root", validator=self._formatter_validator)
self._tags = self._validate_key(key="tags", validator=self._dict_formatter_validator)
self._tags = self._validate_key(key="tags", validator=self._tags_validator)
self._kodi_safe = self._validate_key_if_present(
key="kodi_safe", validator=BoolValidator, default=False
).value
@property
def nfo_name(self) -> StringFormatterValidator:
"""
The NFO file name.
"""
return self._nfo_name
TSharedNfoTagsOptions = TypeVar("TSharedNfoTagsOptions", bound=SharedNfoTagsOptions)
@property
def nfo_root(self) -> StringFormatterValidator:
"""
OVERRIDE DOC IN CHILD CLASSES
"""
return self._nfo_root
@property
def tags(self) -> TSharedNfoTagsValidator:
"""
OVERRIDE DOC IN CHILD CLASSES
"""
return self._tags
@property
def kodi_safe(self) -> Optional[bool]:
"""
Optional. Kodi does not support > 3-byte unicode characters, which include emojis and some
foreign language characters. Setting this to True will replace those characters with '□'.
Defaults to False.
"""
return self._kodi_safe
class SharedNfoTagsPlugin(Plugin[TSharedNfoTagsOptions], Generic[TSharedNfoTagsOptions], ABC):
class SharedNfoTagsPlugin(
Plugin[
SharedNfoTagsOptions[
TStringFormatterValidator, TDictFormatterValidator, TSharedNfoTagsValidator
]
],
Generic[TStringFormatterValidator, TDictFormatterValidator, TSharedNfoTagsValidator],
ABC,
):
"""
Shared code between NFO tags and Ouptut Directory NFO Tags
"""
def _get_xml_element_dict(self, entry: Optional[Entry]) -> Dict[str, XmlElement]:
nfo_tags: Dict[str, XmlElement] = {}
for key, string_tag in self.plugin_options.tags.string_tags.items():
nfo_tags[key] = XmlElement(
text=self.overrides.apply_formatter(formatter=string_tag, entry=entry),
attributes={},
)
for key, attribute_tag in self.plugin_options.tags.attribute_tags.items():
nfo_tags[key] = XmlElement(
text=self.overrides.apply_formatter(formatter=attribute_tag.tag, entry=entry),
attributes={
attr_name: self.overrides.apply_formatter(formatter=attr_formatter, entry=entry)
for attr_name, attr_formatter in attribute_tag.attributes.dict.items()
},
)
return nfo_tags
def _create_nfo(self, entry: Optional[Entry] = None) -> None:
nfo = {}
for tag, tag_formatter in sorted(self.plugin_options.tags.dict.items()):
nfo[tag] = self.overrides.apply_formatter(formatter=tag_formatter, entry=entry)
# Write the nfo tags to XML with the nfo_root
nfo_root = self.overrides.apply_formatter(
formatter=self.plugin_options.nfo_root, entry=entry
)
nfo_tags = self._get_xml_element_dict(entry=entry)
if self.plugin_options.kodi_safe:
nfo = to_max_3_byte_utf8_dict(nfo)
nfo_root = to_max_3_byte_utf8_string(nfo_root)
nfo_tags = {
to_max_3_byte_utf8_string(key): XmlElement(
text=to_max_3_byte_utf8_string(xml_elem.text),
attributes=to_max_3_byte_utf8_dict(xml_elem.attributes),
)
for key, xml_elem in nfo_tags.items()
}
xml = to_xml(nfo_dict=nfo, nfo_root=nfo_root)
xml = to_xml(nfo_dict=nfo_tags, nfo_root=nfo_root)
nfo_file_name = self.overrides.apply_formatter(
formatter=self.plugin_options.nfo_name, entry=entry
@ -76,11 +145,18 @@ class SharedNfoTagsPlugin(Plugin[TSharedNfoTagsOptions], Generic[TSharedNfoTagsO
nfo_file.write(xml)
# Save the nfo file and log its metadata
nfo_metadata = FileMetadata.from_dict(value_dict={nfo_root: nfo}, title="NFO tags:")
nfo_metadata = FileMetadata.from_dict(
value_dict={
nfo_root: {key: xml_elem.to_dict_value() for key, xml_elem in nfo_tags.items()}
},
title="NFO tags:",
)
self.save_file(file_name=nfo_file_name, file_metadata=nfo_metadata, entry=entry)
class NfoTagsOptions(SharedNfoTagsOptions):
class NfoTagsOptions(
SharedNfoTagsOptions[StringFormatterValidator, DictFormatterValidator, NfoTagsValidator]
):
"""
Adds an NFO file for every download file. An NFO file is simply an XML file
with a ``.nfo`` extension. You can add any values into the NFO.
@ -105,13 +181,7 @@ class NfoTagsOptions(SharedNfoTagsOptions):
_formatter_validator = StringFormatterValidator
_dict_formatter_validator = DictFormatterValidator
@property
def nfo_name(self) -> StringFormatterValidator:
"""
The NFO file name.
"""
return self._nfo_name
_tags_validator = NfoTagsValidator
@property
def nfo_root(self) -> StringFormatterValidator:
@ -127,7 +197,7 @@ class NfoTagsOptions(SharedNfoTagsOptions):
return self._nfo_root
@property
def tags(self) -> DictFormatterValidator:
def tags(self) -> NfoTagsValidator:
"""
Tags within the nfo_root tag. In the usage above, it would look like
@ -139,20 +209,29 @@ class NfoTagsOptions(SharedNfoTagsOptions):
<season>2022</season>
<episode>502</episode>
</episodedetails>
Also supports xml attributes:
.. code-block:: yaml
tags:
season:
attributes:
name: "Best Year"
tag: "{upload_year}"
Which translates to
.. code-block:: xml
<season name="Best Year">2022</season>
"""
return self._tags
@property
def kodi_safe(self) -> Optional[bool]:
"""
Optional. Kodi does not support > 3-byte unicode characters, which include emojis and some
foreign language characters. Setting this to True will replace those characters with '□'.
Defaults to False.
"""
return self._kodi_safe
class NfoTagsPlugin(SharedNfoTagsPlugin[NfoTagsOptions]):
class NfoTagsPlugin(
SharedNfoTagsPlugin[StringFormatterValidator, DictFormatterValidator, NfoTagsValidator]
):
plugin_options_type = NfoTagsOptions
def post_process_entry(self, entry: Entry) -> None:

59
src/ytdl_sub/plugins/output_directory_nfo_tags.py
View file

@ -1,15 +1,19 @@
from typing import Optional
from ytdl_sub.plugins.nfo_tags import NfoTagsValidator
from ytdl_sub.plugins.nfo_tags import SharedNfoTagsOptions
from ytdl_sub.plugins.nfo_tags import SharedNfoTagsPlugin
from ytdl_sub.validators.nfo_validators import NfoOverrideTagsValidator
from ytdl_sub.validators.string_formatter_validators import OverridesDictFormatterValidator
from ytdl_sub.validators.string_formatter_validators import OverridesStringFormatterValidator
class OutputDirectoryNfoTagsOptions(SharedNfoTagsOptions):
class OutputDirectoryNfoTagsOptions(
SharedNfoTagsOptions[
OverridesStringFormatterValidator, OverridesDictFormatterValidator, NfoOverrideTagsValidator
]
):
"""
Adds a single NFO file in the output directory. An NFO file is simply an XML file with a
``.nfo`` extension. You can add any values into the NFO.
Adds a single NFO file in the output directory. An NFO file is simply an XML file with a
``.nfo`` extension. You can add any strings or override variables into this NFO.
Usage:
@ -29,13 +33,7 @@ class OutputDirectoryNfoTagsOptions(SharedNfoTagsOptions):
_formatter_validator = OverridesStringFormatterValidator
_dict_formatter_validator = OverridesDictFormatterValidator
@property
def nfo_name(self) -> OverridesStringFormatterValidator:
"""
The NFO file name.
"""
return self._nfo_name
_tags_validator = NfoOverrideTagsValidator
@property
def nfo_root(self) -> OverridesStringFormatterValidator:
@ -51,7 +49,9 @@ class OutputDirectoryNfoTagsOptions(SharedNfoTagsOptions):
return self._nfo_root
@property
def tags(self) -> OverridesDictFormatterValidator:
def tags(
self,
) -> NfoTagsValidator:
"""
Tags within the nfo_root tag. In the usage above, it would look like
@ -61,20 +61,33 @@ class OutputDirectoryNfoTagsOptions(SharedNfoTagsOptions):
<tvshow>
<title>Sweet youtube TV show</title>
</tvshow>
Also supports xml attributes:
.. code-block:: yaml
tags:
title:
attributes:
year: "2022"
tag: "Sweet youtube TV show"
Which translates to
.. code-block:: xml
<title year="2022">Sweet youtube TV show</season>
"""
return self._tags
@property
def kodi_safe(self) -> Optional[bool]:
"""
Optional. Kodi does not support > 3-byte unicode characters, which include emojis and some
foreign language characters. Setting this to True will replace those characters with '□'.
Defaults to False.
"""
return self._kodi_safe
class OutputDirectoryNfoTagsPlugin(SharedNfoTagsPlugin[OutputDirectoryNfoTagsOptions]):
class OutputDirectoryNfoTagsPlugin(
SharedNfoTagsPlugin[
OverridesStringFormatterValidator,
OverridesDictFormatterValidator,
OutputDirectoryNfoTagsOptions,
]
):
plugin_options_type = OutputDirectoryNfoTagsOptions
def post_process_subscription(self):

31
src/ytdl_sub/utils/xml.py
View file

@ -1,5 +1,27 @@
import xml.etree.ElementTree as et
from dataclasses import dataclass
from typing import Any
from typing import Dict
from typing import Union
@dataclass
class XmlElement:
text: str
attributes: Dict[str, str]
def to_dict_value(self) -> Union[str, Dict[str, Any]]:
"""
Returns
-------
Only the tag if no attributes, otherwise a dict containing both attributes and the tag
"""
if not self.attributes:
return self.text
return {
"attributes": self.attributes,
"tag": self.text,
}
def _to_max_3_byte_utf8_char(char: str) -> str:
@ -37,7 +59,7 @@ def to_max_3_byte_utf8_dict(string_dict: Dict[str, str]) -> Dict[str, str]:
}
def to_xml(nfo_dict: Dict[str, str], nfo_root: str) -> bytes:
def to_xml(nfo_dict: Dict[str, XmlElement], nfo_root: str) -> bytes:
"""
Transforms a dict to XML
@ -53,9 +75,10 @@ def to_xml(nfo_dict: Dict[str, str], nfo_root: str) -> bytes:
XML bytes
"""
xml_root = et.Element(nfo_root)
for key, value in nfo_dict.items():
sub_element = et.SubElement(xml_root, key)
sub_element.text = value
for key, xml_elem in sorted(nfo_dict.items()):
sorted_attr = dict(sorted(xml_elem.attributes.items()))
sub_element = et.SubElement(xml_root, key, sorted_attr)
sub_element.text = xml_elem.text
et.indent(tree=xml_root, space=" ", level=0)
return et.tostring(element=xml_root, encoding="utf-8", xml_declaration=True)

124
src/ytdl_sub/validators/nfo_validators.py Normal file
View file

@ -0,0 +1,124 @@
from abc import ABC
from typing import Dict
from typing import Generic
from typing import Type
from typing import TypeVar
from ytdl_sub.validators.strict_dict_validator import StrictDictValidator
from ytdl_sub.validators.string_formatter_validators import DictFormatterValidator
from ytdl_sub.validators.string_formatter_validators import OverridesDictFormatterValidator
from ytdl_sub.validators.string_formatter_validators import OverridesStringFormatterValidator
from ytdl_sub.validators.string_formatter_validators import StringFormatterValidator
from ytdl_sub.validators.validators import DictValidator
TStringFormatterValidator = TypeVar("TStringFormatterValidator", bound=StringFormatterValidator)
TDictFormatterValidator = TypeVar("TDictFormatterValidator", bound=DictFormatterValidator)
class _NfoTagsWithAttributesValidator(
StrictDictValidator, Generic[TStringFormatterValidator, TDictFormatterValidator], ABC
):
_required_keys = {"attributes", "tag"}
formatter_validator: Type[TStringFormatterValidator]
dict_formatter_validator: Type[TDictFormatterValidator]
def __init__(self, name, value):
super().__init__(name, value)
self._attributes = self._validate_key(
key="attributes", validator=self.dict_formatter_validator
)
self._tag = self._validate_key(key="tag", validator=self.formatter_validator)
@property
def attributes(self) -> TDictFormatterValidator:
"""
Returns
-------
The attributes for this NFO tag
"""
return self._attributes
@property
def tag(self) -> TStringFormatterValidator:
"""
Returns
-------
The value for this NFO tag
"""
return self._tag
class NfoTagsWithAttributesValidator(
_NfoTagsWithAttributesValidator[StringFormatterValidator, DictFormatterValidator]
):
formatter_validator = StringFormatterValidator
dict_formatter_validator = DictFormatterValidator
class NfoOverrideTagsWithAttributesValidator(
_NfoTagsWithAttributesValidator[
OverridesStringFormatterValidator, OverridesDictFormatterValidator
]
):
formatter_validator = OverridesStringFormatterValidator
dict_formatter_validator = OverridesDictFormatterValidator
TNfoTagsWithAttributesValidator = _NfoTagsWithAttributesValidator[
TStringFormatterValidator, TDictFormatterValidator
]
class SharedNfoTagsValidator(
DictValidator, Generic[TStringFormatterValidator, TDictFormatterValidator], ABC
):
_tags_with_attributes_validator: Type[TNfoTagsWithAttributesValidator]
def __init__(self, name, value):
super().__init__(name, value)
self._string_tags: Dict[str, StringFormatterValidator] = {}
self._attribute_tags: Dict[str, TNfoTagsWithAttributesValidator] = {}
for key, tag_value in self._dict.items():
if isinstance(tag_value, str):
self._string_tags[key] = self._validate_key(
key=key, validator=self._tags_with_attributes_validator.formatter_validator
)
elif isinstance(tag_value, dict):
self._attribute_tags[key] = self._validate_key(
key=key, validator=self._tags_with_attributes_validator
)
else:
raise self._validation_exception("must either be a string or attributes object")
@property
def string_tags(self) -> Dict[str, StringFormatterValidator]:
"""
Returns
-------
Tags with no attributes
"""
return self._string_tags
@property
def attribute_tags(self) -> Dict[str, TNfoTagsWithAttributesValidator]:
"""
Returns
-------
Tags with attributes
"""
return self._attribute_tags
class NfoTagsValidator(SharedNfoTagsValidator[StringFormatterValidator, DictFormatterValidator]):
_tags_with_attributes_validator = NfoTagsWithAttributesValidator
class NfoOverrideTagsValidator(
SharedNfoTagsValidator[OverridesStringFormatterValidator, OverridesDictFormatterValidator]
):
_tags_with_attributes_validator = NfoOverrideTagsWithAttributesValidator

45
tests/e2e/plugins/test_nfo_tags.py
View file

@ -2,10 +2,11 @@ import pytest
from e2e.expected_transaction_log import assert_transaction_log_matches
from ytdl_sub.subscriptions.subscription import Subscription
from ytdl_sub.utils.exceptions import ValidationException
@pytest.fixture
def kodi_safe_subscription_dict(output_directory):
def subscription_dict(output_directory):
return {
"preset": "yt_music_video",
"youtube": {"video_url": "https://www.youtube.com/shorts/ucYmEqmlhFw"},
@ -17,25 +18,40 @@ def kodi_safe_subscription_dict(output_directory):
},
"nfo_tags": {
"tags": {
"kodi_safe_title 🎸": "{title}",
"kodi_safe_title 🎸": "kodi_safe_value 🎸",
"kodi_safe_title_with_attrs": {
"attributes": {"🎸?": "value\nnewlines 🎸"},
"tag": "the \n tag 🎸🎸",
},
},
"kodi_safe": True,
},
"output_directory_nfo_tags": {
"nfo_name": "test.nfo",
"nfo_root": "kodi_safe_root 🎸",
"tags": {"kodi_safe_title 🎸": "kodi_safe_value 🎸"},
"kodi_safe": True,
"tags": {
"kodi_safe_title 🎸": "kodi_safe_value 🎸",
"kodi_safe_title_with_attrs": {
"attributes": {"🎸?": "value\nnewlines 🎸"},
"tag": "the \n tag 🎸🎸",
},
},
},
}
class TestNfoTagsPlugins:
def test_kodi_safe(self, kodi_safe_subscription_dict, music_video_config, output_directory):
@pytest.mark.parametrize("kodi_safe", [True, False])
def test_nfo_tags(self, subscription_dict, music_video_config, output_directory, kodi_safe):
transaction_log_file_name = "test_nfo.txt"
if kodi_safe:
transaction_log_file_name = "test_nfo_kodi_safe.txt"
subscription_dict["nfo_tags"]["kodi_safe"] = True
subscription_dict["output_directory_nfo_tags"]["kodi_safe"] = True
subscription = Subscription.from_dict(
config=music_video_config,
preset_name="kodi_safe_xml",
preset_dict=kodi_safe_subscription_dict,
preset_dict=subscription_dict,
)
# Only dry run is needed to see if NFO values are kodi safe
@ -43,5 +59,18 @@ class TestNfoTagsPlugins:
assert_transaction_log_matches(
output_directory=output_directory,
transaction_log=transaction_log,
transaction_log_summary_file_name="plugins/test_kodi_safe_xml.txt",
transaction_log_summary_file_name=f"plugins/nfo_tags/{transaction_log_file_name}",
)
def test_source_variable_in_output_directory_nfo_tags_errors(
self, subscription_dict, music_video_config
):
subscription_dict["output_directory_nfo_tags"]["tags"]["kodi_safe_title_with_attrs"][
"attributes"
]["tag"] = "{title}"
with pytest.raises(ValidationException):
Subscription.from_dict(
config=music_video_config,
preset_name="kodi_safe_xml",
preset_dict=subscription_dict,
)

32
tests/e2e/resources/transaction_log_summaries/plugins/nfo_tags/test_nfo.txt Normal file
View file

@ -0,0 +1,32 @@
Files created in '{output_directory}'
----------------------------------------
Rick Beato - Can you hear the difference？ 🎸🔥 #shorts-thumb.jpg
Rick Beato - Can you hear the difference？ 🎸🔥 #shorts.3gp
Rick Beato - Can you hear the difference？ 🎸🔥 #shorts.nfo
NFO tags:
musicvideo:
album: Music Videos
artist: Rick Beato
kodi_safe_title 🎸: kodi_safe_value 🎸
kodi_safe_title_with_attrs:
attributes:
🎸?:
value
newlines 🎸
tag:
the
tag 🎸🎸
title: Can you hear the difference? 🎸🔥 #shorts
year: 2022
test.nfo
NFO tags:
kodi_safe_root 🎸:
kodi_safe_title 🎸: kodi_safe_value 🎸
kodi_safe_title_with_attrs:
attributes:
🎸?:
value
newlines 🎸
tag:
the
tag 🎸🎸

20
tests/e2e/resources/transaction_log_summaries/plugins/test_kodi_safe_xml.txt → tests/e2e/resources/transaction_log_summaries/plugins/nfo_tags/test_nfo_kodi_safe.txt
View file

@ -7,10 +7,26 @@ Rick Beato - Can you hear the difference？ 🎸🔥 #shorts.nfo
musicvideo:
album: Music Videos
artist: Rick Beato
kodi_safe_title □: Can you hear the difference? □□ #shorts
kodi_safe_title □: kodi_safe_value □
kodi_safe_title_with_attrs:
attributes:
□?:
value
newlines □
tag:
the
tag □□
title: Can you hear the difference? □□ #shorts
year: 2022
test.nfo
NFO tags:
kodi_safe_root □:
kodi_safe_title □: kodi_safe_value □
kodi_safe_title □: kodi_safe_value □
kodi_safe_title_with_attrs:
attributes:
□?:
value
newlines □
tag:
the
tag □□