danvics/ytdl-sub
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

[BACKEND] output_directory_nfo_tags to support source vars (#239)

Browse source
This commit is contained in:
Jesse Bannon 2022-09-17 11:11:37 -07:00 committed by GitHub
parent 46c14bd842
commit 64d4be0a42
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
13 changed files with 70 additions and 273 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

9
src/ytdl_sub/config/preset.py
View file

@ -131,10 +131,6 @@ class Preset(StrictDictValidator):
def _source_variables(self) -> List[str]:
return self.downloader.downloader_entry_type.source_variables()
@property
def _added_override_variables(self) -> List[str]:
return self.downloader.added_override_variables()
def __validate_and_get_downloader(self, downloader_source: str) -> Type[Downloader]:
return self._validate_key(key=downloader_source, validator=DownloadStrategyValidator).get(
downloader_source=downloader_source
@ -225,10 +221,7 @@ class Preset(StrictDictValidator):
formatter_validator: Union[StringFormatterValidator, OverridesStringFormatterValidator],
):
# Set the formatter variables to be the overrides
variable_dict = dict(
self.overrides.dict_with_format_strings,
**{added_override: "dummy_string" for added_override in self._added_override_variables},
)
variable_dict = copy.deepcopy(self.overrides.dict_with_format_strings)
# If the formatter supports source variables, set the formatter variables to include
# both source and override variables

15
src/ytdl_sub/config/preset_options.py
View file

@ -121,21 +121,6 @@ class Overrides(DictFormatterValidator):
sanitize=True,
)
def add_override_variables(self, variables_to_add: Dict[str, str]) -> None:
"""
Parameters
----------
variables_to_add
Override variables to add
"""
for key_name, override_var_value in variables_to_add.items():
for sanitize in [False, True]:
self._add_override_variable(
key_name=key_name,
format_string=override_var_value,
sanitize=sanitize,
)
def apply_formatter(
self,
formatter: StringFormatterValidator,

9
src/ytdl_sub/downloaders/downloader.py
View file

@ -111,15 +111,6 @@ class Downloader(DownloadArchiver, Generic[DownloaderOptionsT, DownloaderEntryT]
"""
return {"ignoreerrors": True}
@classmethod
def added_override_variables(cls) -> List[str]:
"""
Returns
-------
List of override variables that this downloader adds
"""
return []
def __init__(
self,
download_options: DownloaderOptionsT,

30
src/ytdl_sub/downloaders/youtube/channel.py
View file

@ -1,7 +1,6 @@
from pathlib import Path
from typing import Dict
from typing import Generator
from typing import List
from typing import Optional
from ytdl_sub.downloaders.downloader import download_logger
@ -109,20 +108,6 @@ class YoutubeChannelDownloader(YoutubeDownloader[YoutubeChannelDownloaderOptions
},
)
@classmethod
def added_override_variables(cls) -> List[str]:
"""
Adds the following :ref:`override <overrides>` variables:
.. code-block:: yaml
overrides:
source_uploader: # The channel's name. NOTE: sometimes it's empty, use with caution
source_title: # The channel's name
source_description: # The channel's description
"""
return ["source_uploader", "source_title", "source_description"]
# pylint: enable=line-too-long
@property
@ -135,20 +120,7 @@ class YoutubeChannelDownloader(YoutubeDownloader[YoutubeChannelDownloaderOptions
"""
Downloads all videos from a channel
"""
collection_url = self.collection.collection_urls.list[0]
_, orphans = super()._download_url_metadata(collection_url=collection_url)
assert not orphans
# TODO: Handle this better
self.overrides.add_override_variables(
variables_to_add={
"source_uploader": self.channel.kwargs_get("uploader", "__failed_to_scrape__"),
"source_title": self.channel.kwargs("title"),
"source_description": self.channel.kwargs_get("description", ""),
}
)
for entry in super()._download(parents=self.parents):
for entry in super().download():
yield entry.to_type(YoutubeVideo)
def _download_thumbnail(

30
src/ytdl_sub/downloaders/youtube/playlist.py
View file

@ -1,6 +1,5 @@
from typing import Dict
from typing import Generator
from typing import List
from ytdl_sub.downloaders.generic.collection_validator import CollectionValidator
from ytdl_sub.downloaders.youtube.abc import YoutubeDownloader
@ -74,20 +73,6 @@ class YoutubePlaylistDownloader(
**{"break_on_existing": True},
)
@classmethod
def added_override_variables(cls) -> List[str]:
"""
Adds the following :ref:`override <overrides>` variables:
.. code-block:: yaml
overrides:
source_uploader: # The playlist's owner's channel name. NOTE: sometimes it's empty, use with caution
source_title: # The playlist's title
source_description: # The playlist's description
"""
return ["source_uploader", "source_title", "source_description"]
# pylint: enable=line-too-long
@property
@ -100,18 +85,5 @@ class YoutubePlaylistDownloader(
"""
Downloads all videos in a Youtube playlist.
"""
collection_url = self.collection.collection_urls.list[0]
_, orphans = super()._download_url_metadata(collection_url)
assert not orphans
# TODO: Handle this better
self.overrides.add_override_variables(
variables_to_add={
"source_title": self.playlist.title,
"source_uploader": self.playlist.kwargs_get("uploader", "__failed_to_scrape__"),
"source_description": self.playlist.kwargs_get("description", ""),
}
)
for entry in super()._download(parents=self.parents):
for entry in super().download():
yield entry.to_type(YoutubePlaylistVideo)

16
src/ytdl_sub/entries/variables/entry_variables.py
View file

@ -238,9 +238,9 @@ class EntryVariables(BaseEntryVariables):
Returns
-------
str
The playlist uploader id if it exists, otherwise returns the playlist unique ID.
The playlist uploader id if it exists, otherwise returns the entry uploader ID.
"""
return self.kwargs_get(PLAYLIST_UPLOADER_ID, self.playlist_uid)
return self.kwargs_get(PLAYLIST_UPLOADER_ID, self.uploader_id)
@property
def playlist_uploader(self: Self) -> str:
@ -248,9 +248,9 @@ class EntryVariables(BaseEntryVariables):
Returns
-------
str
The playlist uploader if it exists, otherwise return the playlist uploader ID.
The playlist uploader if it exists, otherwise return the entry uploader.
"""
return self.kwargs_get(PLAYLIST_UPLOADER, self.playlist_uploader_id)
return self.kwargs_get(PLAYLIST_UPLOADER, self.uploader)
@property
def playlist_uploader_url(self: Self) -> str:
@ -268,9 +268,9 @@ class EntryVariables(BaseEntryVariables):
Returns
-------
str
The source uploader id if it exists, otherwise returns the source unique ID.
The source uploader id if it exists, otherwise returns the playlist_uploader_id
"""
return self.kwargs_get(SOURCE_UPLOADER_ID, self.source_uid)
return self.kwargs_get(SOURCE_UPLOADER_ID, self.playlist_uploader_id)
@property
def source_uploader(self: Self) -> str:
@ -278,9 +278,9 @@ class EntryVariables(BaseEntryVariables):
Returns
-------
str
The source uploader if it exists, otherwise return the source uploader ID.
The source uploader if it exists, otherwise return the playlist_uploader
"""
return self.kwargs_get(SOURCE_UPLOADER, self.source_uploader_id)
return self.kwargs_get(SOURCE_UPLOADER, self.playlist_uploader)
@property
def source_uploader_url(self: Self) -> str:

53
src/ytdl_sub/plugins/nfo_tags.py
View file

@ -3,11 +3,8 @@ from abc import ABC
from collections import defaultdict
from pathlib import Path
from typing import Dict
from typing import Generic
from typing import List
from typing import Optional
from typing import Type
from typing import TypeVar
from ytdl_sub.entries.entry import Entry
from ytdl_sub.plugins.plugin import Plugin
@ -18,37 +15,25 @@ from ytdl_sub.utils.xml import to_max_3_byte_utf8_dict
from ytdl_sub.utils.xml import to_max_3_byte_utf8_string
from ytdl_sub.utils.xml import to_xml
from ytdl_sub.validators.nfo_validators import NfoTagsValidator
from ytdl_sub.validators.nfo_validators import SharedNfoTagsValidator
from ytdl_sub.validators.nfo_validators import TDictFormatterValidator
from ytdl_sub.validators.nfo_validators import TStringFormatterValidator
from ytdl_sub.validators.string_formatter_validators import DictFormatterValidator
from ytdl_sub.validators.string_formatter_validators import StringFormatterValidator
from ytdl_sub.validators.validators import BoolValidator
TSharedNfoTagsValidator = TypeVar("TSharedNfoTagsValidator", bound=SharedNfoTagsValidator)
class SharedNfoTagsOptions(
PluginOptions,
Generic[TStringFormatterValidator, TDictFormatterValidator, TSharedNfoTagsValidator],
ABC,
):
class SharedNfoTagsOptions(PluginOptions):
"""
Shared code between NFO tags and Ouptut Directory NFO Tags
"""
_formatter_validator: Type[TStringFormatterValidator]
_tags_validator: Type[TSharedNfoTagsValidator]
_required_keys = {"nfo_name", "nfo_root", "tags"}
_optional_keys = {"kodi_safe"}
def __init__(self, name, value):
super().__init__(name, value)
self._nfo_name = self._validate_key(key="nfo_name", validator=self._formatter_validator)
self._nfo_root = self._validate_key(key="nfo_root", validator=self._formatter_validator)
self._tags = self._validate_key(key="tags", validator=self._tags_validator)
self._nfo_name = self._validate_key(key="nfo_name", validator=StringFormatterValidator)
self._nfo_root = self._validate_key(key="nfo_root", validator=StringFormatterValidator)
self._tags = self._validate_key(key="tags", validator=NfoTagsValidator)
self._kodi_safe = self._validate_key_if_present(
key="kodi_safe", validator=BoolValidator, default=False
).value
@ -68,7 +53,7 @@ class SharedNfoTagsOptions(
return self._nfo_root
@property
def tags(self) -> TSharedNfoTagsValidator:
def tags(self) -> NfoTagsValidator:
"""
OVERRIDE DOC IN CHILD CLASSES
"""
@ -84,20 +69,12 @@ class SharedNfoTagsOptions(
return self._kodi_safe
class SharedNfoTagsPlugin(
Plugin[
SharedNfoTagsOptions[
TStringFormatterValidator, TDictFormatterValidator, TSharedNfoTagsValidator
]
],
Generic[TStringFormatterValidator, TDictFormatterValidator, TSharedNfoTagsValidator],
ABC,
):
class SharedNfoTagsPlugin(Plugin[SharedNfoTagsOptions], ABC):
"""
Shared code between NFO tags and Ouptut Directory NFO Tags
"""
def _get_xml_element_dict(self, entry: Optional[Entry]) -> Dict[str, List[XmlElement]]:
def _get_xml_element_dict(self, entry: Entry) -> Dict[str, List[XmlElement]]:
nfo_tags: Dict[str, List[XmlElement]] = defaultdict(list)
for key, string_tags in self.plugin_options.tags.string_tags.items():
@ -125,7 +102,7 @@ class SharedNfoTagsPlugin(
return nfo_tags
def _create_nfo(self, entry: Optional[Entry] = None) -> None:
def _create_nfo(self, entry: Entry, save_to_entry: bool = True) -> None:
# Write the nfo tags to XML with the nfo_root
nfo_root = self.overrides.apply_formatter(
formatter=self.plugin_options.nfo_root, entry=entry
@ -171,12 +148,14 @@ class SharedNfoTagsPlugin(
},
title="NFO tags",
)
self.save_file(file_name=nfo_file_name, file_metadata=nfo_metadata, entry=entry)
if save_to_entry:
self.save_file(file_name=nfo_file_name, file_metadata=nfo_metadata, entry=entry)
else:
self.save_file(file_name=nfo_file_name, file_metadata=nfo_metadata)
class NfoTagsOptions(
SharedNfoTagsOptions[StringFormatterValidator, DictFormatterValidator, NfoTagsValidator]
):
class NfoTagsOptions(SharedNfoTagsOptions):
"""
Adds an NFO file for every download file. An NFO file is simply an XML file
with a ``.nfo`` extension. You can add any values into the NFO.
@ -254,9 +233,7 @@ class NfoTagsOptions(
return self._tags
class NfoTagsPlugin(
SharedNfoTagsPlugin[StringFormatterValidator, DictFormatterValidator, NfoTagsValidator]
):
class NfoTagsPlugin(SharedNfoTagsPlugin):
plugin_options_type = NfoTagsOptions
def post_process_entry(self, entry: Entry) -> None:

56
src/ytdl_sub/plugins/output_directory_nfo_tags.py
View file

@ -1,19 +1,19 @@
from typing import Optional
from ytdl_sub.config.preset_options import Overrides
from ytdl_sub.entries.entry import Entry
from ytdl_sub.plugins.nfo_tags import NfoTagsValidator
from ytdl_sub.plugins.nfo_tags import SharedNfoTagsOptions
from ytdl_sub.plugins.nfo_tags import SharedNfoTagsPlugin
from ytdl_sub.validators.nfo_validators import NfoOverrideTagsValidator
from ytdl_sub.validators.string_formatter_validators import OverridesDictFormatterValidator
from ytdl_sub.validators.string_formatter_validators import OverridesStringFormatterValidator
from ytdl_sub.validators.string_formatter_validators import StringFormatterValidator
from ytdl_sub.ytdl_additions.enhanced_download_archive import EnhancedDownloadArchive
class OutputDirectoryNfoTagsOptions(
SharedNfoTagsOptions[
OverridesStringFormatterValidator, OverridesDictFormatterValidator, NfoOverrideTagsValidator
]
):
class OutputDirectoryNfoTagsOptions(SharedNfoTagsOptions):
"""
Adds a single NFO file in the output directory. An NFO file is simply an XML file with a
``.nfo`` extension. You can add any strings or override variables into this NFO.
``.nfo`` extension. It uses the last entry's source variables which can change per download
invocation. Be cautious of which variables you use.
Usage:
@ -31,12 +31,8 @@ class OutputDirectoryNfoTagsOptions(
kodi_safe: False
"""
_formatter_validator = OverridesStringFormatterValidator
_dict_formatter_validator = OverridesDictFormatterValidator
_tags_validator = NfoOverrideTagsValidator
@property
def nfo_root(self) -> OverridesStringFormatterValidator:
def nfo_root(self) -> StringFormatterValidator:
"""
The root tag of the NFO's XML. In the usage above, it would look like
@ -49,9 +45,7 @@ class OutputDirectoryNfoTagsOptions(
return self._nfo_root
@property
def tags(
self,
) -> NfoTagsValidator:
def tags(self) -> NfoTagsValidator:
"""
Tags within the nfo_root tag. In the usage above, it would look like
@ -86,17 +80,27 @@ class OutputDirectoryNfoTagsOptions(
return self._tags
class OutputDirectoryNfoTagsPlugin(
SharedNfoTagsPlugin[
OverridesStringFormatterValidator,
OverridesDictFormatterValidator,
OutputDirectoryNfoTagsOptions,
]
):
class OutputDirectoryNfoTagsPlugin(SharedNfoTagsPlugin):
plugin_options_type = OutputDirectoryNfoTagsOptions
def __init__(
self,
plugin_options: OutputDirectoryNfoTagsOptions,
overrides: Overrides,
enhanced_download_archive: EnhancedDownloadArchive,
):
super().__init__(plugin_options, overrides, enhanced_download_archive)
self._last_entry: Optional[Entry] = None
def post_process_entry(self, entry: Entry) -> None:
"""
Tracks the last entry processed
"""
self._last_entry = entry
def post_process_subscription(self):
"""
Creates an NFO file in the root of the output directory
Creates an NFO file in the root of the output directory using the last entry
"""
self._create_nfo()
if self._last_entry:
self._create_nfo(entry=self._last_entry, save_to_entry=False)

2
src/ytdl_sub/plugins/plugin.py
View file

@ -6,7 +6,6 @@ from typing import Optional
from typing import Tuple
from typing import Type
from typing import TypeVar
from typing import final
from ytdl_sub.config.preset_options import AddsVariablesMixin
from ytdl_sub.config.preset_options import Overrides
@ -60,7 +59,6 @@ class Plugin(DownloadArchiver, Generic[PluginOptionsT], ABC):
# If the plugin creates multile entries from a single entry
is_split_plugin: bool = False
@final
def __init__(
self,
plugin_options: PluginOptionsT,

91
src/ytdl_sub/validators/nfo_validators.py
View file

@ -1,43 +1,27 @@
from abc import ABC
from collections import defaultdict
from typing import Dict
from typing import Generic
from typing import List
from typing import Type
from typing import TypeVar
from ytdl_sub.validators.strict_dict_validator import StrictDictValidator
from ytdl_sub.validators.string_formatter_validators import DictFormatterValidator
from ytdl_sub.validators.string_formatter_validators import ListFormatterValidator
from ytdl_sub.validators.string_formatter_validators import ListOverridesFormatterValidator
from ytdl_sub.validators.string_formatter_validators import OverridesDictFormatterValidator
from ytdl_sub.validators.string_formatter_validators import OverridesStringFormatterValidator
from ytdl_sub.validators.string_formatter_validators import StringFormatterValidator
from ytdl_sub.validators.validators import DictValidator
from ytdl_sub.validators.validators import ListValidator
TStringFormatterValidator = TypeVar("TStringFormatterValidator", bound=StringFormatterValidator)
TDictFormatterValidator = TypeVar("TDictFormatterValidator", bound=DictFormatterValidator)
class _NfoTagsWithAttributesValidator(
StrictDictValidator, Generic[TStringFormatterValidator, TDictFormatterValidator], ABC
):
class NfoTagsWithAttributesValidator(StrictDictValidator):
_required_keys = {"attributes", "tag"}
formatter_validator: Type[TStringFormatterValidator]
dict_formatter_validator: Type[TDictFormatterValidator]
def __init__(self, name, value):
super().__init__(name, value)
self._attributes = self._validate_key(
key="attributes", validator=self.dict_formatter_validator
)
self._tag = self._validate_key(key="tag", validator=self.formatter_validator)
self._attributes = self._validate_key(key="attributes", validator=DictFormatterValidator)
self._tag = self._validate_key(key="tag", validator=StringFormatterValidator)
@property
def attributes(self) -> TDictFormatterValidator:
def attributes(self) -> DictFormatterValidator:
"""
Returns
-------
@ -46,7 +30,7 @@ class _NfoTagsWithAttributesValidator(
return self._attributes
@property
def tag(self) -> TStringFormatterValidator:
def tag(self) -> StringFormatterValidator:
"""
Returns
-------
@ -55,59 +39,18 @@ class _NfoTagsWithAttributesValidator(
return self._tag
class NfoTagsWithAttributesValidator(
_NfoTagsWithAttributesValidator[StringFormatterValidator, DictFormatterValidator]
):
"""TagsWithAttributes for the entry NFO validator"""
formatter_validator = StringFormatterValidator
dict_formatter_validator = DictFormatterValidator
class NfoTagsWithAttributesListValidator(ListValidator[NfoTagsWithAttributesValidator]):
"""TagsWithAttributes list for the entry NFO validator"""
_inner_list_type = NfoTagsWithAttributesValidator
class NfoOverrideTagsWithAttributesValidator(
_NfoTagsWithAttributesValidator[
OverridesStringFormatterValidator, OverridesDictFormatterValidator
]
):
"""TagsWithAttributes for the output directory NFO validator"""
formatter_validator = OverridesStringFormatterValidator
dict_formatter_validator = OverridesDictFormatterValidator
class NfoOverrideTagsWithAttributesListValidator(
ListValidator[NfoOverrideTagsWithAttributesValidator]
):
"""TagsWithAttributes list for the output directory NFO validator"""
_inner_list_type = NfoOverrideTagsWithAttributesValidator
# Generic TagsWithAttribute to use for SharedNfoTagsValidator
TNfoTagsWithAttributesValidator = _NfoTagsWithAttributesValidator[
TStringFormatterValidator, TDictFormatterValidator
]
# List validators
TNfoTagsWithAttributesListValidator = ListValidator[TNfoTagsWithAttributesValidator]
TNfoTagsListValidator = ListValidator[TStringFormatterValidator]
class SharedNfoTagsValidator(DictValidator, ABC):
_tags_validator: Type[TNfoTagsListValidator]
_tags_with_attributes_validator: Type[TNfoTagsWithAttributesListValidator]
class NfoTagsValidator(DictValidator, ABC):
def __init__(self, name, value):
super().__init__(name, value)
self._string_tags: Dict[str, List[TStringFormatterValidator]] = defaultdict(list)
self._attribute_tags: Dict[str, List[TNfoTagsWithAttributesValidator]] = defaultdict(list)
self._string_tags: Dict[str, List[StringFormatterValidator]] = defaultdict(list)
self._attribute_tags: Dict[str, List[NfoTagsWithAttributesValidator]] = defaultdict(list)
for key, tag_value in self._dict.items():
# Turn each value into a list if it's not
@ -118,12 +61,12 @@ class SharedNfoTagsValidator(DictValidator, ABC):
self._string_tags[key].extend(
self._validate_key(
key=key,
validator=self._tags_validator,
validator=ListFormatterValidator,
).list
)
elif isinstance(tag_value[0], dict):
self._attribute_tags[key].extend(
self._validate_key(key=key, validator=self._tags_with_attributes_validator).list
self._validate_key(key=key, validator=NfoTagsWithAttributesListValidator).list
)
else:
raise self._validation_exception(
@ -131,7 +74,7 @@ class SharedNfoTagsValidator(DictValidator, ABC):
)
@property
def string_tags(self) -> Dict[str, List[TStringFormatterValidator]]:
def string_tags(self) -> Dict[str, List[StringFormatterValidator]]:
"""
Returns
-------
@ -140,20 +83,10 @@ class SharedNfoTagsValidator(DictValidator, ABC):
return self._string_tags
@property
def attribute_tags(self) -> Dict[str, List[TNfoTagsWithAttributesValidator]]:
def attribute_tags(self) -> Dict[str, List[NfoTagsWithAttributesValidator]]:
"""
Returns
-------
Tags with attributes
"""
return self._attribute_tags
class NfoTagsValidator(SharedNfoTagsValidator):
_tags_validator = ListFormatterValidator
_tags_with_attributes_validator = NfoTagsWithAttributesListValidator
class NfoOverrideTagsValidator(SharedNfoTagsValidator):
_tags_validator = ListOverridesFormatterValidator
_tags_with_attributes_validator = NfoOverrideTagsWithAttributesListValidator

27
tests/e2e/plugins/test_nfo_tags.py
View file

@ -83,30 +83,3 @@ class TestNfoTagsPlugins:
transaction_log=transaction_log,
transaction_log_summary_file_name=f"plugins/nfo_tags/{transaction_log_file_name}",
)
def test_source_variable_in_output_directory_nfo_tags_errors(
self, subscription_dict, music_video_config
):
subscription_dict["output_directory_nfo_tags"]["tags"]["kodi_safe_title_with_attrs"][
"attributes"
]["tag"] = "{title}"
with pytest.raises(ValidationException):
Subscription.from_dict(
config=music_video_config,
preset_name="kodi_safe_xml",
preset_dict=subscription_dict,
)
def test_source_variable_in_output_directory_nfo_tags_list_errors(
self, subscription_dict, music_video_config
):
subscription_dict["output_directory_nfo_tags"]["tags"]["kodi_safe_title_with_attrs"] = [
"okay value",
"not okay {title}",
]
with pytest.raises(ValidationException):
Subscription.from_dict(
config=music_video_config,
preset_name="kodi_safe_xml",
preset_dict=subscription_dict,
)

3
tests/e2e/resources/expected_downloads_summaries/plugins/date_range/no_downloads.json
View file

@ -1,6 +1,5 @@
{
".ytdl-sub-recent-download-archive.json": "99914b932bd37a50b983c5e7c90ae93b",
"fanart.jpg": "129c6639b47299bc48062f0365e670ee",
"poster.jpg": "5de28eea5a921a041452ab3ce1041f73",
"tvshow.nfo": "c1c888ff6691f36328d1fb9d8c43adff"
"poster.jpg": "5de28eea5a921a041452ab3ce1041f73"
}

2
tests/unit/config/test_preset.py
View file

@ -230,7 +230,7 @@ class TestPreset:
):
with pytest.raises(
StringFormattingVariableNotFoundException,
match="Override variable 'dne_var' does not exist",
match="Format variable 'dne_var' does not exist",
):
_ = Preset(
config=config_file,