danvics/ytdl-sub
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

[BACKEND] More flexible formatter post_process (#1421)

Browse source 
Makes the underlying formatting process better handle native (non-string) types in a more flexible way.
This commit is contained in:
Jesse Bannon 2026-01-23 16:36:17 -08:00 committed by GitHub
parent 264e458c1c
commit 60cdbad8c7
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
15 changed files with 74 additions and 100 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

53
src/ytdl_sub/config/overrides.py
View file

@ -3,6 +3,8 @@ from typing import Dict
from typing import Iterable
from typing import Optional
from typing import Set
from typing import Type
from typing import TypeVar
from ytdl_sub.entries.entry import Entry
from ytdl_sub.entries.script.variable_definitions import VARIABLES
@ -20,10 +22,11 @@ from ytdl_sub.utils.exceptions import StringFormattingException
from ytdl_sub.utils.exceptions import ValidationException
from ytdl_sub.utils.script import ScriptUtils
from ytdl_sub.utils.scriptable import Scriptable
from ytdl_sub.validators.string_formatter_validators import OverridesStringFormatterValidator
from ytdl_sub.validators.string_formatter_validators import StringFormatterValidator
from ytdl_sub.validators.string_formatter_validators import UnstructuredDictFormatterValidator
ExpectedT = TypeVar("ExpectedT")
class Overrides(UnstructuredDictFormatterValidator, Scriptable):
"""
@ -207,7 +210,8 @@ class Overrides(UnstructuredDictFormatterValidator, Scriptable):
formatter: StringFormatterValidator,
entry: Optional[Entry] = None,
function_overrides: Optional[Dict[str, str]] = None,
) -> str:
expected_type: Type[ExpectedT] = str,
) -> ExpectedT:
"""
Parameters
----------
@ -217,6 +221,8 @@ class Overrides(UnstructuredDictFormatterValidator, Scriptable):
Optional. Entry to add source variables to the formatter
function_overrides
Optional. Explicit values to override the overrides themselves and source variables
expected_type
The expected type that should return. Defaults to string.
Returns
-------
@ -227,42 +233,15 @@ class Overrides(UnstructuredDictFormatterValidator, Scriptable):
StringFormattingException
If the formatter that is trying to be resolved cannot
"""
return formatter.post_process(
str(
self._apply_to_resolvable(
formatter=formatter, entry=entry, function_overrides=function_overrides
)
)
)
def apply_overrides_formatter_to_native(
self,
formatter: OverridesStringFormatterValidator,
function_overrides: Optional[Dict[str, str]] = None,
) -> Any:
"""
Parameters
----------
formatter
Overrides formatter to apply
function_overrides
Optional. Explicit values to override the overrides themselves and source variables
Returns
-------
The native python form of the resolved variable
"""
return formatter.post_process_native(
out = formatter.post_process(
self._apply_to_resolvable(
formatter=formatter, entry=None, function_overrides=function_overrides
formatter=formatter, entry=entry, function_overrides=function_overrides
).native
)
def evaluate_boolean(
self, formatter: StringFormatterValidator, entry: Optional[Entry] = None
) -> bool:
"""
Apply a formatter, and evaluate it to a boolean
"""
output = self.apply_formatter(formatter=formatter, entry=entry)
return ScriptUtils.bool_formatter_output(output)
if not isinstance(out, expected_type):
raise StringFormattingException(
f"Expected type {expected_type.__name__}, but received '{out.__class__.__name__}'"
)
return out

2
src/ytdl_sub/config/plugin/plugin.py
View file

@ -48,7 +48,7 @@ class Plugin(BasePlugin[OptionsValidatorT], Generic[OptionsValidatorT], ABC):
Returns True if enabled, False if disabled.
"""
if isinstance(self.plugin_options, ToggleableOptionsDictValidator):
return self.overrides.evaluate_boolean(self.plugin_options.enable)
return self.overrides.apply_formatter(self.plugin_options.enable, expected_type=bool)
return True
def ytdl_options_match_filters(self) -> Tuple[List[str], List[str]]:

5
src/ytdl_sub/config/preset_options.py
View file

@ -62,10 +62,7 @@ class YTDLOptions(UnstructuredOverridesDictFormatterValidator):
Materializes the entire ytdl-options dict from OverrideStringFormatters into
native python.
"""
out = {
key: overrides.apply_overrides_formatter_to_native(val)
for key, val in self.dict.items()
}
out = {key: overrides.apply_formatter(val) for key, val in self.dict.items()}
if "cookiefile" in out:
if not FileHandler.is_file_existent(out["cookiefile"]):
raise ValidationException(

14
src/ytdl_sub/downloaders/url/downloader.py
View file

@ -52,12 +52,12 @@ class UrlDownloaderBasePluginExtension(SourcePluginExtension[MultiUrlValidator])
if 0 <= input_url_idx < len(self.plugin_options.urls.list):
validator = self.plugin_options.urls.list[input_url_idx]
if entry_input_url in self.overrides.apply_overrides_formatter_to_native(validator.url):
if entry_input_url in self.overrides.apply_formatter(validator.url, expected_type=list):
return validator
# Match the first validator based on the URL, if one exists
for validator in self.plugin_options.urls.list:
if entry_input_url in self.overrides.apply_overrides_formatter_to_native(validator.url):
if entry_input_url in self.overrides.apply_formatter(validator.url, expected_type=list):
return validator
# Return the first validator if none exist
@ -382,7 +382,7 @@ class MultiUrlDownloader(SourcePlugin[MultiUrlValidator]):
entries_to_iter: List[Optional[Entry]] = entries
indices = list(range(len(entries_to_iter)))
if self.overrides.evaluate_boolean(validator.download_reverse):
if self.overrides.apply_formatter(validator.download_reverse, expected_type=bool):
indices = reversed(indices)
for idx in indices:
@ -461,8 +461,8 @@ class MultiUrlDownloader(SourcePlugin[MultiUrlValidator]):
ytdl_option_overrides=validator.ytdl_options.to_native_dict(self.overrides)
)
include_sibling_metadata = self.overrides.evaluate_boolean(
validator.include_sibling_metadata
include_sibling_metadata = self.overrides.apply_formatter(
validator.include_sibling_metadata, expected_type=bool
)
parents, orphan_entries = self._download_url_metadata(
@ -487,11 +487,9 @@ class MultiUrlDownloader(SourcePlugin[MultiUrlValidator]):
# download the bottom-most urls first since they are top-priority
for idx, url_validator in reversed(list(enumerate(self.collection.urls.list))):
# URLs can be empty. If they are, then skip
if not (urls := self.overrides.apply_overrides_formatter_to_native(url_validator.url)):
if not (urls := self.overrides.apply_formatter(url_validator.url, expected_type=list)):
continue
assert isinstance(urls, list)
for url in reversed(urls):
assert isinstance(url, str)

3
src/ytdl_sub/downloaders/url/validators.py
View file

@ -1,5 +1,6 @@
from typing import Any
from typing import Dict
from typing import List
from typing import Optional
from typing import Set
@ -44,7 +45,7 @@ class UrlThumbnailListValidator(ListValidator[UrlThumbnailValidator]):
class OverridesOneOrManyUrlValidator(OverridesStringFormatterValidator):
def post_process_native(self, resolved: Any) -> Any:
def post_process(self, resolved: Any) -> List[str]:
if isinstance(resolved, str):
return [resolved]
if isinstance(resolved, list):

2
src/ytdl_sub/plugins/date_range.py
View file

@ -116,7 +116,7 @@ class DateRangePlugin(Plugin[DateRangeOptions]):
date_validator=self.plugin_options.after, overrides=self.overrides
)
after_filter = f"{date_type} >= {after_str}"
if self.overrides.evaluate_boolean(self.plugin_options.breaks):
if self.overrides.apply_formatter(self.plugin_options.breaks, expected_type=bool):
breaking_match_filters.append(after_filter)
else:
match_filters.append(after_filter)

2
src/ytdl_sub/plugins/embed_thumbnail.py
View file

@ -33,7 +33,7 @@ class EmbedThumbnailPlugin(Plugin[EmbedThumbnailOptions]):
@property
def _embed_thumbnail(self) -> bool:
return self.overrides.evaluate_boolean(self.plugin_options)
return self.overrides.apply_formatter(self.plugin_options, expected_type=bool)
@classmethod
def _embed_video_thumbnail(cls, entry: Entry) -> None:

11
src/ytdl_sub/plugins/filter_exclude.py
View file

@ -7,13 +7,14 @@ from ytdl_sub.config.validators.options import OptionsValidator
from ytdl_sub.entries.entry import Entry
from ytdl_sub.utils.exceptions import StringFormattingException
from ytdl_sub.utils.logger import Logger
from ytdl_sub.validators.string_formatter_validators import ListFormatterValidator
from ytdl_sub.validators.string_formatter_validators import BooleanFormatterValidator
from ytdl_sub.validators.validators import ListValidator
from ytdl_sub.ytdl_additions.enhanced_download_archive import EnhancedDownloadArchive
logger = Logger.get("filter-exclude")
class FilterExcludeOptions(ListFormatterValidator, OptionsValidator):
class FilterExcludeOptions(ListValidator[BooleanFormatterValidator], OptionsValidator):
"""
Applies a conditional OR on any number of filters comprised of either variables or scripts.
If any filter evaluates to True, the entry will be excluded.
@ -29,6 +30,8 @@ class FilterExcludeOptions(ListFormatterValidator, OptionsValidator):
{ %contains( %lower(description), '#short' ) }
"""
_inner_list_type = BooleanFormatterValidator
class FilterExcludePlugin(Plugin[FilterExcludeOptions]):
plugin_options_type = FilterExcludeOptions
@ -52,7 +55,9 @@ class FilterExcludePlugin(Plugin[FilterExcludeOptions]):
return entry
for formatter in self.plugin_options.list:
should_exclude = self.overrides.evaluate_boolean(formatter=formatter, entry=entry)
should_exclude = self.overrides.apply_formatter(
formatter=formatter, entry=entry, expected_type=bool
)
if should_exclude:
logger.info(

12
src/ytdl_sub/plugins/filter_include.py
View file

@ -7,14 +7,14 @@ from ytdl_sub.config.validators.options import OptionsValidator
from ytdl_sub.entries.entry import Entry
from ytdl_sub.utils.exceptions import StringFormattingException
from ytdl_sub.utils.logger import Logger
from ytdl_sub.utils.script import ScriptUtils
from ytdl_sub.validators.string_formatter_validators import ListFormatterValidator
from ytdl_sub.validators.string_formatter_validators import BooleanFormatterValidator
from ytdl_sub.validators.validators import ListValidator
from ytdl_sub.ytdl_additions.enhanced_download_archive import EnhancedDownloadArchive
logger = Logger.get("filter-include")
class FilterIncludeOptions(ListFormatterValidator, OptionsValidator):
class FilterIncludeOptions(ListValidator[BooleanFormatterValidator], OptionsValidator):
"""
Applies a conditional AND on any number of filters comprised of either variables or scripts.
If all filters evaluate to True, the entry will be included.
@ -38,6 +38,8 @@ class FilterIncludeOptions(ListFormatterValidator, OptionsValidator):
}
"""
_inner_list_type = BooleanFormatterValidator
class FilterIncludePlugin(Plugin[FilterIncludeOptions]):
plugin_options_type = FilterIncludeOptions
@ -61,8 +63,8 @@ class FilterIncludePlugin(Plugin[FilterIncludeOptions]):
return entry
for formatter in self.plugin_options.list:
should_exclude = ScriptUtils.bool_formatter_output(
self.overrides.apply_formatter(formatter=formatter, entry=entry)
should_exclude = self.overrides.apply_formatter(
formatter=formatter, entry=entry, expected_type=bool
)
if not should_exclude:
logger.info(

2
src/ytdl_sub/plugins/nfo_tags.py
View file

@ -140,7 +140,7 @@ class SharedNfoTagsPlugin(Plugin[SharedNfoTagsOptions], ABC):
if not nfo_tags:
return
if self.overrides.evaluate_boolean(self.plugin_options.kodi_safe):
if self.overrides.apply_formatter(self.plugin_options.kodi_safe, expected_type=bool):
nfo_root = to_max_3_byte_utf8_string(nfo_root)
nfo_tags = {
to_max_3_byte_utf8_string(key): [

2
src/ytdl_sub/plugins/square_thumbnail.py
View file

@ -31,7 +31,7 @@ class SquareThumbnailPlugin(Plugin[SquareThumbnailOptions]):
@property
def _square_thumbnail(self) -> bool:
return self.overrides.evaluate_boolean(self.plugin_options)
return self.overrides.apply_formatter(self.plugin_options, expected_type=bool)
@classmethod
def _convert_to_square_thumbnail(cls, entry: Entry) -> None:

6
src/ytdl_sub/plugins/throttle_protection.py
View file

@ -42,8 +42,8 @@ class _RandomizedRangeValidator(StrictDictValidator, ABC):
)
def _randomized_float(self, overrides: Overrides, entry: Optional[Entry] = None) -> float:
actualized_min = float(overrides.apply_formatter(self._min, entry=entry))
actualized_max = float(overrides.apply_formatter(self._max, entry=entry))
actualized_min = overrides.apply_formatter(self._min, entry=entry, expected_type=float)
actualized_max = overrides.apply_formatter(self._max, entry=entry, expected_type=float)
if actualized_min < 0:
raise self._validation_exception(
@ -70,7 +70,7 @@ class _RandomizedRangeValidator(StrictDictValidator, ABC):
-------
Max possible value
"""
actualized_max = float(overrides.apply_formatter(self._max, entry=entry))
actualized_max = overrides.apply_formatter(self._max, entry=entry, expected_type=float)
if actualized_max < 0:
raise self._validation_exception(
f"max must be greater than zero, received {actualized_max}"

41
src/ytdl_sub/validators/string_formatter_validators.py
View file

@ -82,35 +82,27 @@ class StringFormatterValidator(StringValidator):
"""
return self._parsed
def post_process(self, resolved: str) -> str:
def post_process(self, resolved: Any) -> Any:
"""
Returns
-------
Apply any post processing to the resolved value
Apply any post processing to the resolved value. Defaults to casting it to string.
"""
return resolved
def post_process_native(self, resolved: Any) -> Any:
"""
Returns
-------
Apply any post processing to the resolved native value.
"""
return resolved
return str(resolved)
class FloatFormatterValidator(StringFormatterValidator):
_expected_value_type_name = "float"
def post_process(self, resolved: str) -> str:
def post_process(self, resolved: str) -> float:
try:
float(resolved)
out = float(resolved)
except Exception as exc:
raise self._validation_exception(
f"Expected a float, but received '{resolved}'"
) from exc
return resolved
return out
class StandardizedDateValidator(StringFormatterValidator):
@ -127,6 +119,13 @@ class StandardizedDateValidator(StringFormatterValidator):
return resolved
class BooleanFormatterValidator(StringFormatterValidator):
_expected_value_type_name = "boolean"
def post_process(self, resolved: Any) -> bool:
return ScriptUtils.bool_formatter_output(output=str(resolved))
# pylint: disable=line-too-long
class OverridesStringFormatterValidator(StringFormatterValidator):
"""
@ -146,15 +145,14 @@ class OverridesStringFormatterValidator(StringFormatterValidator):
class OverridesIntegerFormatterValidator(OverridesStringFormatterValidator):
_expected_value_type_name = "integer"
def post_process(self, resolved: str) -> str:
def post_process(self, resolved: str) -> int:
try:
int(resolved)
out = int(resolved)
except Exception as exc:
raise self._validation_exception(
f"Expected an integer, but received '{resolved}'"
) from exc
return resolved
return out
class OverridesFloatFormatterValidator(FloatFormatterValidator, OverridesStringFormatterValidator):
@ -163,9 +161,14 @@ class OverridesFloatFormatterValidator(FloatFormatterValidator, OverridesStringF
"""
class OverridesBooleanFormatterValidator(OverridesStringFormatterValidator):
class OverridesBooleanFormatterValidator(
BooleanFormatterValidator, OverridesStringFormatterValidator
):
_expected_value_type_name = "boolean"
def post_process(self, resolved: Any) -> bool:
return ScriptUtils.bool_formatter_output(output=str(resolved))
class ListFormatterValidator(ListValidator[StringFormatterValidator]):
_inner_list_type = StringFormatterValidator

13
tests/unit/config/test_subscription.py
View file

@ -475,19 +475,6 @@ def test_advanced_tv_show_subscriptions(
assert overrides.script.get("subscription_name").native == "Gardening with Ciscoe"
assert overrides.apply_overrides_formatter_to_native(overrides.dict["subscription_array"]) == [
"https://www.youtube.com/@gardeningwithciscoe4430",
"https://www.youtube.com/playlist?list=PLi8V8UemxeG6lo5if5H5g5EbsteELcb0_",
"https://www.youtube.com/playlist?list=PLsJlQSR-KjmaQqqJ9jq18cF6XXXAR4kyn",
"https://www.youtube.com/watch?v=2vq-vPubS5I",
]
assert overrides.apply_overrides_formatter_to_native(overrides.dict["urls"]) == [
"https://www.youtube.com/@gardeningwithciscoe4430",
"https://www.youtube.com/playlist?list=PLi8V8UemxeG6lo5if5H5g5EbsteELcb0_",
"https://www.youtube.com/playlist?list=PLsJlQSR-KjmaQqqJ9jq18cF6XXXAR4kyn",
"https://www.youtube.com/watch?v=2vq-vPubS5I",
]
def test_music_subscriptions(default_config: ConfigFile, music_subscriptions_path: Path):
subs = Subscription.from_file_path(

6
tests/unit/prebuilt_presets/test_tv_show_collection.py
View file

@ -57,12 +57,13 @@ class TestTvShowCollectionPreset:
# is_bilateral
if i == 0:
url = sub.overrides.apply_overrides_formatter_to_native(
url = sub.overrides.apply_formatter(
url_list[itr].url,
function_overrides={
# mock so bilateral url gets enabled
"subscription_has_download_archive": "True"
},
expected_type=list,
)
assert url == [
f"youtube.com/playlist?url_{season_num}_{i}"
@ -81,12 +82,13 @@ class TestTvShowCollectionPreset:
# not bilateral
else:
for j in range(2):
url = sub.overrides.apply_overrides_formatter_to_native(
url = sub.overrides.apply_formatter(
url_list[itr + j].url,
function_overrides={
# mock so bilateral url gets enabled
"subscription_has_download_archive": "True"
},
expected_type=list,
)
# First instance is the first url to get thumbnails