danvics/ytdl-sub
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

[BACKEND] output_directory_nfo_tags to support source vars

Browse source
This commit is contained in:
Jesse Bannon 2022-09-17 09:05:54 -07:00
parent 46c14bd842
commit 030974c074
8 changed files with 28 additions and 227 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

9
src/ytdl_sub/config/preset.py
View file

@ -131,10 +131,6 @@ class Preset(StrictDictValidator):
def _source_variables(self) -> List[str]: def _source_variables(self) -> List[str]:
return self.downloader.downloader_entry_type.source_variables() return self.downloader.downloader_entry_type.source_variables()
@property
def _added_override_variables(self) -> List[str]:
return self.downloader.added_override_variables()
def __validate_and_get_downloader(self, downloader_source: str) -> Type[Downloader]: def __validate_and_get_downloader(self, downloader_source: str) -> Type[Downloader]:
return self._validate_key(key=downloader_source, validator=DownloadStrategyValidator).get( return self._validate_key(key=downloader_source, validator=DownloadStrategyValidator).get(
downloader_source=downloader_source downloader_source=downloader_source
@ -225,10 +221,7 @@ class Preset(StrictDictValidator):
formatter_validator: Union[StringFormatterValidator, OverridesStringFormatterValidator], formatter_validator: Union[StringFormatterValidator, OverridesStringFormatterValidator],
): ):
# Set the formatter variables to be the overrides # Set the formatter variables to be the overrides
variable_dict = dict( variable_dict = copy.deepcopy(self.overrides.dict_with_format_strings)
self.overrides.dict_with_format_strings,
**{added_override: "dummy_string" for added_override in self._added_override_variables},
)
# If the formatter supports source variables, set the formatter variables to include # If the formatter supports source variables, set the formatter variables to include
# both source and override variables # both source and override variables

15
src/ytdl_sub/config/preset_options.py
View file

@ -121,21 +121,6 @@ class Overrides(DictFormatterValidator):
sanitize=True, sanitize=True,
) )
def add_override_variables(self, variables_to_add: Dict[str, str]) -> None:
"""
Parameters
----------
variables_to_add
Override variables to add
"""
for key_name, override_var_value in variables_to_add.items():
for sanitize in [False, True]:
self._add_override_variable(
key_name=key_name,
format_string=override_var_value,
sanitize=sanitize,
)
def apply_formatter( def apply_formatter(
self, self,
formatter: StringFormatterValidator, formatter: StringFormatterValidator,

9
src/ytdl_sub/downloaders/downloader.py
View file

@ -111,15 +111,6 @@ class Downloader(DownloadArchiver, Generic[DownloaderOptionsT, DownloaderEntryT]
""" """
return {"ignoreerrors": True} return {"ignoreerrors": True}
@classmethod
def added_override_variables(cls) -> List[str]:
"""
Returns
-------
List of override variables that this downloader adds
"""
return []
def __init__( def __init__(
self, self,
download_options: DownloaderOptionsT, download_options: DownloaderOptionsT,

30
src/ytdl_sub/downloaders/youtube/channel.py
View file

@ -1,7 +1,6 @@
from pathlib import Path from pathlib import Path
from typing import Dict from typing import Dict
from typing import Generator from typing import Generator
from typing import List
from typing import Optional from typing import Optional
from ytdl_sub.downloaders.downloader import download_logger from ytdl_sub.downloaders.downloader import download_logger
@ -109,20 +108,6 @@ class YoutubeChannelDownloader(YoutubeDownloader[YoutubeChannelDownloaderOptions
}, },
) )
@classmethod
def added_override_variables(cls) -> List[str]:
"""
Adds the following :ref:`override <overrides>` variables:
.. code-block:: yaml
overrides:
source_uploader: # The channel's name. NOTE: sometimes it's empty, use with caution
source_title: # The channel's name
source_description: # The channel's description
"""
return ["source_uploader", "source_title", "source_description"]
# pylint: enable=line-too-long # pylint: enable=line-too-long
@property @property
@ -135,20 +120,7 @@ class YoutubeChannelDownloader(YoutubeDownloader[YoutubeChannelDownloaderOptions
""" """
Downloads all videos from a channel Downloads all videos from a channel
""" """
collection_url = self.collection.collection_urls.list[0] for entry in super().download():
_, orphans = super()._download_url_metadata(collection_url=collection_url)
assert not orphans
# TODO: Handle this better
self.overrides.add_override_variables(
variables_to_add={
"source_uploader": self.channel.kwargs_get("uploader", "__failed_to_scrape__"),
"source_title": self.channel.kwargs("title"),
"source_description": self.channel.kwargs_get("description", ""),
}
)
for entry in super()._download(parents=self.parents):
yield entry.to_type(YoutubeVideo) yield entry.to_type(YoutubeVideo)
def _download_thumbnail( def _download_thumbnail(

30
src/ytdl_sub/downloaders/youtube/playlist.py
View file

@ -1,6 +1,5 @@
from typing import Dict from typing import Dict
from typing import Generator from typing import Generator
from typing import List
from ytdl_sub.downloaders.generic.collection_validator import CollectionValidator from ytdl_sub.downloaders.generic.collection_validator import CollectionValidator
from ytdl_sub.downloaders.youtube.abc import YoutubeDownloader from ytdl_sub.downloaders.youtube.abc import YoutubeDownloader
@ -74,20 +73,6 @@ class YoutubePlaylistDownloader(
**{"break_on_existing": True}, **{"break_on_existing": True},
) )
@classmethod
def added_override_variables(cls) -> List[str]:
"""
Adds the following :ref:`override <overrides>` variables:
.. code-block:: yaml
overrides:
source_uploader: # The playlist's owner's channel name. NOTE: sometimes it's empty, use with caution
source_title: # The playlist's title
source_description: # The playlist's description
"""
return ["source_uploader", "source_title", "source_description"]
# pylint: enable=line-too-long # pylint: enable=line-too-long
@property @property
@ -100,18 +85,5 @@ class YoutubePlaylistDownloader(
""" """
Downloads all videos in a Youtube playlist. Downloads all videos in a Youtube playlist.
""" """
collection_url = self.collection.collection_urls.list[0] for entry in super().download():
_, orphans = super()._download_url_metadata(collection_url)
assert not orphans
# TODO: Handle this better
self.overrides.add_override_variables(
variables_to_add={
"source_title": self.playlist.title,
"source_uploader": self.playlist.kwargs_get("uploader", "__failed_to_scrape__"),
"source_description": self.playlist.kwargs_get("description", ""),
}
)
for entry in super()._download(parents=self.parents):
yield entry.to_type(YoutubePlaylistVideo) yield entry.to_type(YoutubePlaylistVideo)

43
src/ytdl_sub/plugins/nfo_tags.py
View file

@ -3,11 +3,8 @@ from abc import ABC
from collections import defaultdict from collections import defaultdict
from pathlib import Path from pathlib import Path
from typing import Dict from typing import Dict
from typing import Generic
from typing import List from typing import List
from typing import Optional from typing import Optional
from typing import Type
from typing import TypeVar
from ytdl_sub.entries.entry import Entry from ytdl_sub.entries.entry import Entry
from ytdl_sub.plugins.plugin import Plugin from ytdl_sub.plugins.plugin import Plugin
@ -18,37 +15,25 @@ from ytdl_sub.utils.xml import to_max_3_byte_utf8_dict
from ytdl_sub.utils.xml import to_max_3_byte_utf8_string from ytdl_sub.utils.xml import to_max_3_byte_utf8_string
from ytdl_sub.utils.xml import to_xml from ytdl_sub.utils.xml import to_xml
from ytdl_sub.validators.nfo_validators import NfoTagsValidator from ytdl_sub.validators.nfo_validators import NfoTagsValidator
from ytdl_sub.validators.nfo_validators import SharedNfoTagsValidator
from ytdl_sub.validators.nfo_validators import TDictFormatterValidator
from ytdl_sub.validators.nfo_validators import TStringFormatterValidator
from ytdl_sub.validators.string_formatter_validators import DictFormatterValidator from ytdl_sub.validators.string_formatter_validators import DictFormatterValidator
from ytdl_sub.validators.string_formatter_validators import StringFormatterValidator from ytdl_sub.validators.string_formatter_validators import StringFormatterValidator
from ytdl_sub.validators.validators import BoolValidator from ytdl_sub.validators.validators import BoolValidator
TSharedNfoTagsValidator = TypeVar("TSharedNfoTagsValidator", bound=SharedNfoTagsValidator)
class SharedNfoTagsOptions(PluginOptions):
class SharedNfoTagsOptions(
PluginOptions,
Generic[TStringFormatterValidator, TDictFormatterValidator, TSharedNfoTagsValidator],
ABC,
):
""" """
Shared code between NFO tags and Ouptut Directory NFO Tags Shared code between NFO tags and Ouptut Directory NFO Tags
""" """
_formatter_validator: Type[TStringFormatterValidator]
_tags_validator: Type[TSharedNfoTagsValidator]
_required_keys = {"nfo_name", "nfo_root", "tags"} _required_keys = {"nfo_name", "nfo_root", "tags"}
_optional_keys = {"kodi_safe"} _optional_keys = {"kodi_safe"}
def __init__(self, name, value): def __init__(self, name, value):
super().__init__(name, value) super().__init__(name, value)
self._nfo_name = self._validate_key(key="nfo_name", validator=self._formatter_validator) self._nfo_name = self._validate_key(key="nfo_name", validator=StringFormatterValidator)
self._nfo_root = self._validate_key(key="nfo_root", validator=self._formatter_validator) self._nfo_root = self._validate_key(key="nfo_root", validator=StringFormatterValidator)
self._tags = self._validate_key(key="tags", validator=self._tags_validator) self._tags = self._validate_key(key="tags", validator=NfoTagsValidator)
self._kodi_safe = self._validate_key_if_present( self._kodi_safe = self._validate_key_if_present(
key="kodi_safe", validator=BoolValidator, default=False key="kodi_safe", validator=BoolValidator, default=False
).value ).value
@ -68,7 +53,7 @@ class SharedNfoTagsOptions(
return self._nfo_root return self._nfo_root
@property @property
def tags(self) -> TSharedNfoTagsValidator: def tags(self) -> NfoTagsValidator:
""" """
OVERRIDE DOC IN CHILD CLASSES OVERRIDE DOC IN CHILD CLASSES
""" """
@ -84,15 +69,7 @@ class SharedNfoTagsOptions(
return self._kodi_safe return self._kodi_safe
class SharedNfoTagsPlugin( class SharedNfoTagsPlugin(Plugin[SharedNfoTagsOptions], ABC):
Plugin[
SharedNfoTagsOptions[
TStringFormatterValidator, TDictFormatterValidator, TSharedNfoTagsValidator
]
],
Generic[TStringFormatterValidator, TDictFormatterValidator, TSharedNfoTagsValidator],
ABC,
):
""" """
Shared code between NFO tags and Ouptut Directory NFO Tags Shared code between NFO tags and Ouptut Directory NFO Tags
""" """
@ -174,9 +151,7 @@ class SharedNfoTagsPlugin(
self.save_file(file_name=nfo_file_name, file_metadata=nfo_metadata, entry=entry) self.save_file(file_name=nfo_file_name, file_metadata=nfo_metadata, entry=entry)
class NfoTagsOptions( class NfoTagsOptions(SharedNfoTagsOptions):
SharedNfoTagsOptions[StringFormatterValidator, DictFormatterValidator, NfoTagsValidator]
):
""" """
Adds an NFO file for every download file. An NFO file is simply an XML file Adds an NFO file for every download file. An NFO file is simply an XML file
with a ``.nfo`` extension. You can add any values into the NFO. with a ``.nfo`` extension. You can add any values into the NFO.
@ -254,9 +229,7 @@ class NfoTagsOptions(
return self._tags return self._tags
class NfoTagsPlugin( class NfoTagsPlugin(SharedNfoTagsPlugin):
SharedNfoTagsPlugin[StringFormatterValidator, DictFormatterValidator, NfoTagsValidator]
):
plugin_options_type = NfoTagsOptions plugin_options_type = NfoTagsOptions
def post_process_entry(self, entry: Entry) -> None: def post_process_entry(self, entry: Entry) -> None:

28
src/ytdl_sub/plugins/output_directory_nfo_tags.py
View file

@ -1,16 +1,10 @@
from ytdl_sub.plugins.nfo_tags import NfoTagsValidator from ytdl_sub.plugins.nfo_tags import NfoTagsValidator
from ytdl_sub.plugins.nfo_tags import SharedNfoTagsOptions from ytdl_sub.plugins.nfo_tags import SharedNfoTagsOptions
from ytdl_sub.plugins.nfo_tags import SharedNfoTagsPlugin from ytdl_sub.plugins.nfo_tags import SharedNfoTagsPlugin
from ytdl_sub.validators.nfo_validators import NfoOverrideTagsValidator from ytdl_sub.validators.string_formatter_validators import StringFormatterValidator
from ytdl_sub.validators.string_formatter_validators import OverridesDictFormatterValidator
from ytdl_sub.validators.string_formatter_validators import OverridesStringFormatterValidator
class OutputDirectoryNfoTagsOptions( class OutputDirectoryNfoTagsOptions(SharedNfoTagsOptions):
SharedNfoTagsOptions[
OverridesStringFormatterValidator, OverridesDictFormatterValidator, NfoOverrideTagsValidator
]
):
""" """
Adds a single NFO file in the output directory. An NFO file is simply an XML file with a Adds a single NFO file in the output directory. An NFO file is simply an XML file with a
``.nfo`` extension. You can add any strings or override variables into this NFO. ``.nfo`` extension. You can add any strings or override variables into this NFO.
@ -31,12 +25,8 @@ class OutputDirectoryNfoTagsOptions(
kodi_safe: False kodi_safe: False
""" """
_formatter_validator = OverridesStringFormatterValidator
_dict_formatter_validator = OverridesDictFormatterValidator
_tags_validator = NfoOverrideTagsValidator
@property @property
def nfo_root(self) -> OverridesStringFormatterValidator: def nfo_root(self) -> StringFormatterValidator:
""" """
The root tag of the NFO's XML. In the usage above, it would look like The root tag of the NFO's XML. In the usage above, it would look like
@ -49,9 +39,7 @@ class OutputDirectoryNfoTagsOptions(
return self._nfo_root return self._nfo_root
@property @property
def tags( def tags(self) -> NfoTagsValidator:
self,
) -> NfoTagsValidator:
""" """
Tags within the nfo_root tag. In the usage above, it would look like Tags within the nfo_root tag. In the usage above, it would look like
@ -86,13 +74,7 @@ class OutputDirectoryNfoTagsOptions(
return self._tags return self._tags
class OutputDirectoryNfoTagsPlugin( class OutputDirectoryNfoTagsPlugin(SharedNfoTagsPlugin):
SharedNfoTagsPlugin[
OverridesStringFormatterValidator,
OverridesDictFormatterValidator,
OutputDirectoryNfoTagsOptions,
]
):
plugin_options_type = OutputDirectoryNfoTagsOptions plugin_options_type = OutputDirectoryNfoTagsOptions
def post_process_subscription(self): def post_process_subscription(self):

91
src/ytdl_sub/validators/nfo_validators.py
View file

@ -1,43 +1,27 @@
from abc import ABC from abc import ABC
from collections import defaultdict from collections import defaultdict
from typing import Dict from typing import Dict
from typing import Generic
from typing import List from typing import List
from typing import Type
from typing import TypeVar
from ytdl_sub.validators.strict_dict_validator import StrictDictValidator from ytdl_sub.validators.strict_dict_validator import StrictDictValidator
from ytdl_sub.validators.string_formatter_validators import DictFormatterValidator from ytdl_sub.validators.string_formatter_validators import DictFormatterValidator
from ytdl_sub.validators.string_formatter_validators import ListFormatterValidator from ytdl_sub.validators.string_formatter_validators import ListFormatterValidator
from ytdl_sub.validators.string_formatter_validators import ListOverridesFormatterValidator
from ytdl_sub.validators.string_formatter_validators import OverridesDictFormatterValidator
from ytdl_sub.validators.string_formatter_validators import OverridesStringFormatterValidator
from ytdl_sub.validators.string_formatter_validators import StringFormatterValidator from ytdl_sub.validators.string_formatter_validators import StringFormatterValidator
from ytdl_sub.validators.validators import DictValidator from ytdl_sub.validators.validators import DictValidator
from ytdl_sub.validators.validators import ListValidator from ytdl_sub.validators.validators import ListValidator
TStringFormatterValidator = TypeVar("TStringFormatterValidator", bound=StringFormatterValidator)
TDictFormatterValidator = TypeVar("TDictFormatterValidator", bound=DictFormatterValidator)
class NfoTagsWithAttributesValidator(StrictDictValidator):
class _NfoTagsWithAttributesValidator(
StrictDictValidator, Generic[TStringFormatterValidator, TDictFormatterValidator], ABC
):
_required_keys = {"attributes", "tag"} _required_keys = {"attributes", "tag"}
formatter_validator: Type[TStringFormatterValidator]
dict_formatter_validator: Type[TDictFormatterValidator]
def __init__(self, name, value): def __init__(self, name, value):
super().__init__(name, value) super().__init__(name, value)
self._attributes = self._validate_key( self._attributes = self._validate_key(key="attributes", validator=DictFormatterValidator)
key="attributes", validator=self.dict_formatter_validator self._tag = self._validate_key(key="tag", validator=StringFormatterValidator)
)
self._tag = self._validate_key(key="tag", validator=self.formatter_validator)
@property @property
def attributes(self) -> TDictFormatterValidator: def attributes(self) -> DictFormatterValidator:
""" """
Returns Returns
------- -------
@ -46,7 +30,7 @@ class _NfoTagsWithAttributesValidator(
return self._attributes return self._attributes
@property @property
def tag(self) -> TStringFormatterValidator: def tag(self) -> StringFormatterValidator:
""" """
Returns Returns
------- -------
@ -55,59 +39,18 @@ class _NfoTagsWithAttributesValidator(
return self._tag return self._tag
class NfoTagsWithAttributesValidator(
_NfoTagsWithAttributesValidator[StringFormatterValidator, DictFormatterValidator]
):
"""TagsWithAttributes for the entry NFO validator"""
formatter_validator = StringFormatterValidator
dict_formatter_validator = DictFormatterValidator
class NfoTagsWithAttributesListValidator(ListValidator[NfoTagsWithAttributesValidator]): class NfoTagsWithAttributesListValidator(ListValidator[NfoTagsWithAttributesValidator]):
"""TagsWithAttributes list for the entry NFO validator""" """TagsWithAttributes list for the entry NFO validator"""
_inner_list_type = NfoTagsWithAttributesValidator _inner_list_type = NfoTagsWithAttributesValidator
class NfoOverrideTagsWithAttributesValidator( class NfoTagsValidator(DictValidator, ABC):
_NfoTagsWithAttributesValidator[
OverridesStringFormatterValidator, OverridesDictFormatterValidator
]
):
"""TagsWithAttributes for the output directory NFO validator"""
formatter_validator = OverridesStringFormatterValidator
dict_formatter_validator = OverridesDictFormatterValidator
class NfoOverrideTagsWithAttributesListValidator(
ListValidator[NfoOverrideTagsWithAttributesValidator]
):
"""TagsWithAttributes list for the output directory NFO validator"""
_inner_list_type = NfoOverrideTagsWithAttributesValidator
# Generic TagsWithAttribute to use for SharedNfoTagsValidator
TNfoTagsWithAttributesValidator = _NfoTagsWithAttributesValidator[
TStringFormatterValidator, TDictFormatterValidator
]
# List validators
TNfoTagsWithAttributesListValidator = ListValidator[TNfoTagsWithAttributesValidator]
TNfoTagsListValidator = ListValidator[TStringFormatterValidator]
class SharedNfoTagsValidator(DictValidator, ABC):
_tags_validator: Type[TNfoTagsListValidator]
_tags_with_attributes_validator: Type[TNfoTagsWithAttributesListValidator]
def __init__(self, name, value): def __init__(self, name, value):
super().__init__(name, value) super().__init__(name, value)
self._string_tags: Dict[str, List[TStringFormatterValidator]] = defaultdict(list) self._string_tags: Dict[str, List[StringFormatterValidator]] = defaultdict(list)
self._attribute_tags: Dict[str, List[TNfoTagsWithAttributesValidator]] = defaultdict(list) self._attribute_tags: Dict[str, List[NfoTagsWithAttributesValidator]] = defaultdict(list)
for key, tag_value in self._dict.items(): for key, tag_value in self._dict.items():
# Turn each value into a list if it's not # Turn each value into a list if it's not
@ -118,12 +61,12 @@ class SharedNfoTagsValidator(DictValidator, ABC):
self._string_tags[key].extend( self._string_tags[key].extend(
self._validate_key( self._validate_key(
key=key, key=key,
validator=self._tags_validator, validator=ListFormatterValidator,
).list ).list
) )
elif isinstance(tag_value[0], dict): elif isinstance(tag_value[0], dict):
self._attribute_tags[key].extend( self._attribute_tags[key].extend(
self._validate_key(key=key, validator=self._tags_with_attributes_validator).list self._validate_key(key=key, validator=NfoTagsWithAttributesListValidator).list
) )
else: else:
raise self._validation_exception( raise self._validation_exception(
@ -131,7 +74,7 @@ class SharedNfoTagsValidator(DictValidator, ABC):
) )
@property @property
def string_tags(self) -> Dict[str, List[TStringFormatterValidator]]: def string_tags(self) -> Dict[str, List[StringFormatterValidator]]:
""" """
Returns Returns
------- -------
@ -140,20 +83,10 @@ class SharedNfoTagsValidator(DictValidator, ABC):
return self._string_tags return self._string_tags
@property @property
def attribute_tags(self) -> Dict[str, List[TNfoTagsWithAttributesValidator]]: def attribute_tags(self) -> Dict[str, List[NfoTagsWithAttributesValidator]]:
""" """
Returns Returns
------- -------
Tags with attributes Tags with attributes
""" """
return self._attribute_tags return self._attribute_tags
class NfoTagsValidator(SharedNfoTagsValidator):
_tags_validator = ListFormatterValidator
_tags_with_attributes_validator = NfoTagsWithAttributesListValidator
class NfoOverrideTagsValidator(SharedNfoTagsValidator):
_tags_validator = ListOverridesFormatterValidator
_tags_with_attributes_validator = NfoOverrideTagsWithAttributesListValidator