Merge branch 'master' into prebuilt_presets

This commit is contained in:
Qualis Svagtlys 2024-01-11 07:22:58 -06:00
commit 89f13e59d0
31 changed files with 473 additions and 103 deletions

View file

@ -22,6 +22,15 @@ Extracts audio from a video file.
opus, vorbis, wav, and best to grab the best possible format at runtime. opus, vorbis, wav, and best to grab the best possible format at runtime.
``enable``
:expected type: Optional[OverridesFormatter]
:description:
Can typically be left undefined to always default to enable. For preset convenience,
this field can be set using an override variable to easily toggle whether this plugin
is enabled or not via Boolean.
``quality`` ``quality``
:expected type: Float :expected type: Float
@ -76,6 +85,15 @@ chapters and remove specific ones. Can also remove chapters using regex.
Defaults to True. Embed chapters into the file. Defaults to True. Embed chapters into the file.
``enable``
:expected type: Optional[OverridesFormatter]
:description:
Can typically be left undefined to always default to enable. For preset convenience,
this field can be set using an override variable to easily toggle whether this plugin
is enabled or not via Boolean.
``force_key_frames`` ``force_key_frames``
:expected type: Optional[Boolean] :expected type: Optional[Boolean]
@ -148,6 +166,15 @@ granularity possible.
Only download videos before this datetime. Only download videos before this datetime.
``enable``
:expected type: Optional[OverridesFormatter]
:description:
Can typically be left undefined to always default to enable. For preset convenience,
this field can be set using an override variable to easily toggle whether this plugin
is enabled or not via Boolean.
---------------------------------------------------------------------------------------------------- ----------------------------------------------------------------------------------------------------
download download
@ -259,6 +286,15 @@ Also supports custom ffmpeg conversions:
with ``ffmpeg_post_process_args``. Defaults to ``yt-dlp``. with ``ffmpeg_post_process_args``. Defaults to ``yt-dlp``.
``enable``
:expected type: Optional[OverridesFormatter]
:description:
Can typically be left undefined to always default to enable. For preset convenience,
this field can be set using an override variable to easily toggle whether this plugin
is enabled or not via Boolean.
``ffmpeg_post_process_args`` ``ffmpeg_post_process_args``
:expected type: Optional[OverridesFormatter] :expected type: Optional[OverridesFormatter]
@ -395,6 +431,15 @@ with a ``.nfo`` extension. You can add any values into the NFO.
episode: "{upload_month}{upload_day_padded}" episode: "{upload_month}{upload_day_padded}"
kodi_safe: False kodi_safe: False
``enable``
:expected type: Optional[OverridesFormatter]
:description:
Can typically be left undefined to always default to enable. For preset convenience,
this field can be set using an override variable to easily toggle whether this plugin
is enabled or not via Boolean.
``kodi_safe`` ``kodi_safe``
:expected type: Optional[Boolean] :expected type: Optional[Boolean]
@ -484,6 +529,15 @@ Usage:
# optional # optional
kodi_safe: False kodi_safe: False
``enable``
:expected type: Optional[OverridesFormatter]
:description:
Can typically be left undefined to always default to enable. For preset convenience,
this field can be set using an override variable to easily toggle whether this plugin
is enabled or not via Boolean.
``kodi_safe`` ``kodi_safe``
:expected type: Optional[Boolean] :expected type: Optional[Boolean]
@ -800,6 +854,15 @@ and using ``title_and_description`` can regex match/exclude from either ``title`
- "{upload_month}" - "{upload_month}"
- "{upload_day}" - "{upload_day}"
``enable``
:expected type: Optional[OverridesFormatter]
:description:
Can typically be left undefined to always default to enable. For preset convenience,
this field can be set using an override variable to easily toggle whether this plugin
is enabled or not via Boolean.
``skip_if_match_fails`` ``skip_if_match_fails``
:expected type: Optional[Boolean] :expected type: Optional[Boolean]
@ -880,6 +943,15 @@ It will set the respective language to the correct subtitle file.
webm files can only embed "vtt" subtitle types. webm files can only embed "vtt" subtitle types.
``enable``
:expected type: Optional[OverridesFormatter]
:description:
Can typically be left undefined to always default to enable. For preset convenience,
this field can be set using an override variable to easily toggle whether this plugin
is enabled or not via Boolean.
``languages`` ``languages``
:expected type: Optional[List[String]] :expected type: Optional[List[String]]
@ -931,6 +1003,15 @@ scripted.
max: 36 max: 36
subscription_download_probability: 1.0 subscription_download_probability: 1.0
``enable``
:expected type: Optional[OverridesFormatter]
:description:
Can typically be left undefined to always default to enable. For preset convenience,
this field can be set using an override variable to easily toggle whether this plugin
is enabled or not via Boolean.
``max_downloads_per_subscription`` ``max_downloads_per_subscription``
:expected type: Optional[Range] :expected type: Optional[Range]

View file

@ -107,15 +107,10 @@ def _download_subscriptions_from_yaml_files(
subscriptions += Subscription.from_file_path( subscriptions += Subscription.from_file_path(
config=config, config=config,
subscription_path=path, subscription_path=path,
subscription_matches=subscription_matches,
subscription_override_dict=subscription_override_dict, subscription_override_dict=subscription_override_dict,
) )
if subscriptions and subscription_matches:
logger.info("Filtering subscriptions by name based on --match arguments")
subscriptions = [
sub for sub in subscriptions if any(match in sub.name for match in subscription_matches)
]
for subscription in subscriptions: for subscription in subscriptions:
with subscription.exception_handling(): with subscription.exception_handling():
logger.info( logger.info(

View file

@ -1,5 +1,6 @@
from abc import ABC from abc import ABC
from abc import abstractmethod from abc import abstractmethod
from functools import cached_property
from typing import Dict from typing import Dict
from typing import Generic from typing import Generic
from typing import List from typing import List
@ -8,9 +9,11 @@ from typing import Tuple
from typing import Type from typing import Type
from ytdl_sub.config.overrides import Overrides from ytdl_sub.config.overrides import Overrides
from ytdl_sub.config.validators.options import ToggleableOptionsDictValidator
from ytdl_sub.config.validators.options import TOptionsValidator from ytdl_sub.config.validators.options import TOptionsValidator
from ytdl_sub.entries.entry import Entry from ytdl_sub.entries.entry import Entry
from ytdl_sub.utils.file_handler import FileMetadata from ytdl_sub.utils.file_handler import FileMetadata
from ytdl_sub.utils.script import ScriptUtils
from ytdl_sub.ytdl_additions.enhanced_download_archive import DownloadArchiver from ytdl_sub.ytdl_additions.enhanced_download_archive import DownloadArchiver
from ytdl_sub.ytdl_additions.enhanced_download_archive import EnhancedDownloadArchive from ytdl_sub.ytdl_additions.enhanced_download_archive import EnhancedDownloadArchive
@ -40,6 +43,17 @@ class Plugin(BasePlugin[TOptionsValidator], Generic[TOptionsValidator], ABC):
Class to define the new plugin functionality Class to define the new plugin functionality
""" """
@cached_property
def is_enabled(self) -> bool:
"""
Returns True if enabled, False if disabled.
"""
if isinstance(self.plugin_options, ToggleableOptionsDictValidator):
return ScriptUtils.bool_formatter_output(
self.overrides.apply_formatter(self.plugin_options.enable)
)
return True
def ytdl_options_match_filters(self) -> Tuple[List[str], List[str]]: def ytdl_options_match_filters(self) -> Tuple[List[str], List[str]]:
""" """
Returns Returns

View file

@ -6,6 +6,7 @@ from typing import TypeVar
from ytdl_sub.config.plugin.plugin_operation import PluginOperation from ytdl_sub.config.plugin.plugin_operation import PluginOperation
from ytdl_sub.utils.exceptions import ValidationException from ytdl_sub.utils.exceptions import ValidationException
from ytdl_sub.validators.strict_dict_validator import StrictDictValidator from ytdl_sub.validators.strict_dict_validator import StrictDictValidator
from ytdl_sub.validators.string_formatter_validators import OverridesBooleanFormatterValidator
from ytdl_sub.validators.validators import Validator from ytdl_sub.validators.validators import Validator
# pylint: disable=no-self-use # pylint: disable=no-self-use
@ -57,3 +58,28 @@ TOptionsValidator = TypeVar("TOptionsValidator", bound=OptionsValidator)
class OptionsDictValidator(StrictDictValidator, OptionsValidator, ABC): class OptionsDictValidator(StrictDictValidator, OptionsValidator, ABC):
pass pass
class ToggleableOptionsDictValidator(OptionsDictValidator):
_optional_keys = {"enable"}
def __init__(self, name, value):
assert (
"enable" in self._optional_keys
), f"{self.__class__.__name__} does not have enable as an optional field"
super().__init__(name, value)
self._enable = self._validate_key(
key="enable", validator=OverridesBooleanFormatterValidator, default="True"
)
@property
def enable(self) -> OverridesBooleanFormatterValidator:
"""
:expected type: Optional[OverridesFormatter]
:description:
Can typically be left undefined to always default to enable. For preset convenience,
this field can be set using an override variable to easily toggle whether this plugin
is enabled or not via Boolean.
"""
return self._enable

View file

@ -13,9 +13,10 @@ from ytdl_sub.config.plugin.preset_plugins import PresetPlugins
from ytdl_sub.config.preset_options import OutputOptions from ytdl_sub.config.preset_options import OutputOptions
from ytdl_sub.config.validators.options import OptionsValidator from ytdl_sub.config.validators.options import OptionsValidator
from ytdl_sub.downloaders.url.validators import MultiUrlValidator from ytdl_sub.downloaders.url.validators import MultiUrlValidator
from ytdl_sub.entries.script.variable_definitions import VARIABLE_SCRIPTS
from ytdl_sub.entries.variables.override_variables import SubscriptionVariables from ytdl_sub.entries.variables.override_variables import SubscriptionVariables
from ytdl_sub.script.script import Script from ytdl_sub.script.script import Script
from ytdl_sub.utils.scriptable import BASE_SCRIPT
from ytdl_sub.validators.string_formatter_validators import to_variable_dependency_format_string
from ytdl_sub.validators.string_formatter_validators import validate_formatters from ytdl_sub.validators.string_formatter_validators import validate_formatters
@ -32,10 +33,10 @@ def _add_dummy_overrides(overrides: Overrides) -> Dict[str, str]:
# Have the dummy override variable contain all variable deps that it uses in the string # Have the dummy override variable contain all variable deps that it uses in the string
dummy_overrides: Dict[str, str] = {} dummy_overrides: Dict[str, str] = {}
for override_name in _override_variables(overrides): for override_name in _override_variables(overrides):
dummy_overrides[override_name] = ""
# pylint: disable=protected-access # pylint: disable=protected-access
for variable_dependency in overrides.script._variables[override_name].variables: dummy_overrides[override_name] = to_variable_dependency_format_string(
dummy_overrides[override_name] += f"{{ {variable_dependency.name } }}" script=overrides.script, parsed_format_string=overrides.script._variables[override_name]
)
# pylint: enable=protected-access # pylint: enable=protected-access
return dummy_overrides return dummy_overrides
@ -72,8 +73,15 @@ def _override_variables(overrides: Overrides) -> Set[str]:
} }
def _entry_variables() -> Set[str]: _DUMMY_ENTRY_VARIABLES: Dict[str, str] = {
return set(list(VARIABLE_SCRIPTS.keys())) name: to_variable_dependency_format_string(
# pylint: disable=protected-access
script=BASE_SCRIPT,
parsed_format_string=BASE_SCRIPT._variables[name]
# pylint: enable=protected-access
)
for name in BASE_SCRIPT.variable_names
}
class VariableValidation: class VariableValidation:
@ -97,12 +105,12 @@ class VariableValidation:
""" """
Do some gymnastics to initialize the Overrides script. Do some gymnastics to initialize the Overrides script.
""" """
entry_variables = _entry_variables()
override_variables = _override_variables(overrides) override_variables = _override_variables(overrides)
# Set resolved variables as all entry + override variables # Set resolved variables as all entry + override variables
# at this point to generate every possible added/modified variable # at this point to generate every possible added/modified variable
self.resolved_variables = entry_variables | override_variables self.resolved_variables = set(_DUMMY_ENTRY_VARIABLES.keys()) | override_variables
plugin_variables: Set[str] = set()
for ( for (
plugin_options, plugin_options,
@ -125,6 +133,7 @@ class VariableValidation:
# Set unresolved as variables that are added but do not exist as # Set unresolved as variables that are added but do not exist as
# entry/override variables since they are created at run-time # entry/override variables since they are created at run-time
self.unresolved_variables |= added_variables | modified_variables self.unresolved_variables |= added_variables | modified_variables
plugin_variables |= added_variables | modified_variables
# Then update resolved variables to reflect that # Then update resolved variables to reflect that
self.resolved_variables -= self.unresolved_variables self.resolved_variables -= self.unresolved_variables
@ -137,10 +146,11 @@ class VariableValidation:
) )
# copy the script and mock entry variables # copy the script and mock entry variables
self.script = copy.deepcopy(overrides.script).add(_add_dummy_variables(entry_variables)) self.script = copy.deepcopy(overrides.script)
self.script.add( self.script.add(
variables=_add_dummy_overrides(overrides=overrides), variables=_add_dummy_overrides(overrides=overrides)
unresolvable=self.unresolved_variables, | _add_dummy_variables(variables=plugin_variables)
| _DUMMY_ENTRY_VARIABLES
) )
return self return self
@ -158,7 +168,6 @@ class VariableValidation:
resolved_variables = added_variables | modified_variables resolved_variables = added_variables | modified_variables
self.script.add(_add_dummy_variables(resolved_variables))
self.resolved_variables |= resolved_variables self.resolved_variables |= resolved_variables
self.unresolved_variables -= resolved_variables self.unresolved_variables -= resolved_variables
@ -177,13 +186,10 @@ class VariableValidation:
): ):
self._add_variables(PluginOperation.MODIFY_ENTRY_METADATA, options=plugin_options) self._add_variables(PluginOperation.MODIFY_ENTRY_METADATA, options=plugin_options)
self._update_script()
for plugin_options in PluginMapping.order_options_by( for plugin_options in PluginMapping.order_options_by(
self.plugins.zipped(), PluginOperation.MODIFY_ENTRY self.plugins.zipped(), PluginOperation.MODIFY_ENTRY
): ):
added = self._add_variables(PluginOperation.MODIFY_ENTRY, options=plugin_options) self._add_variables(PluginOperation.MODIFY_ENTRY, options=plugin_options)
if added:
self._update_script()
# Validate that any formatter in the plugin options can resolve # Validate that any formatter in the plugin options can resolve
validate_formatters( validate_formatters(

View file

@ -102,7 +102,7 @@ class YTDLP:
) )
if is_downloaded and is_thumbnail_downloaded: if is_downloaded and is_thumbnail_downloaded:
return entry_dict return entry_dict or {} # in-case yt-dlp returns None
# Always add check_formats # Always add check_formats
# See https://github.com/yt-dlp/yt-dlp/issues/502 # See https://github.com/yt-dlp/yt-dlp/issues/502
@ -216,14 +216,11 @@ class YTDLP:
**kwargs **kwargs
arguments passed directory to YoutubeDL extract_info arguments passed directory to YoutubeDL extract_info
""" """
parent_dict: Dict = {}
try: try:
with cls._listen_and_log_downloaded_info_json( with cls._listen_and_log_downloaded_info_json(
working_directory=working_directory, log_prefix=log_prefix_on_info_json_dl working_directory=working_directory, log_prefix=log_prefix_on_info_json_dl
): ):
parent_dict = cls.extract_info( cls.extract_info(ytdl_options_overrides=ytdl_options_overrides, **kwargs)
ytdl_options_overrides=ytdl_options_overrides, **kwargs
)
except RejectedVideoReached: except RejectedVideoReached:
cls.logger.debug( cls.logger.debug(
"RejectedVideoReached, stopping additional downloads " "RejectedVideoReached, stopping additional downloads "
@ -237,19 +234,34 @@ class YTDLP:
except MaxDownloadsReached: except MaxDownloadsReached:
cls.logger.info("MaxDownloadsReached, stopping additional downloads.") cls.logger.info("MaxDownloadsReached, stopping additional downloads.")
# For YouTube playlists in particular, channel metadata is not fetched. Attempt to get parent_dicts: List[Dict] = []
# channel metadata via grabbing uploader_url info json a max of 3 times entry_dicts = cls._get_entry_dicts_from_info_json_files(working_directory=working_directory)
current_iter = 0 entry_ids = {entry_dict.get("id") for entry_dict in entry_dicts}
url = kwargs.get("url")
uploader_url = parent_dict.get("uploader_url")
while current_iter < 3 and uploader_url and url != uploader_url:
cls.logger.debug("Attempting to get parent metadata from URL %s", uploader_url)
parent_dict = cls.extract_info(
ytdl_options_overrides=ytdl_options_overrides | {"playlist_items": "0:0"},
url=uploader_url,
)
current_iter += 1
url = uploader_url
uploader_url = parent_dict.get("uploader_url")
return cls._get_entry_dicts_from_info_json_files(working_directory=working_directory) # Try to get additional uploader (source) metadata that yt-dlp does not fetch
# in a single request
for entry_dict in entry_dicts:
if not (uploader_id := entry_dict.get("uploader_id")):
continue
if uploader_id in entry_ids or not (uploader_url := entry_dict.get("uploader_url")):
continue
cls.logger.debug("Attempting to get parent metadata from URL %s", uploader_url)
try:
parent_dict = cls.extract_info(
ytdl_options_overrides=ytdl_options_overrides | {"playlist_items": "0:0"},
url=uploader_url,
)
except Exception: # pylint: disable=broad-except
# Do not try this uploader_id again
entry_ids.add(uploader_id)
break
if isinstance(parent_dict, dict):
parent_id = parent_dict.get("id")
parent_dicts.append(parent_dict)
entry_ids |= {uploader_id, parent_id}
cls.logger.debug("Adding parent metadata with ids [%s, %s]", uploader_id, parent_id)
return entry_dicts + parent_dicts

View file

@ -6,7 +6,7 @@ from typing import Set
from ytdl_sub.config.plugin.plugin import Plugin from ytdl_sub.config.plugin.plugin import Plugin
from ytdl_sub.config.plugin.plugin_operation import PluginOperation from ytdl_sub.config.plugin.plugin_operation import PluginOperation
from ytdl_sub.config.validators.options import OptionsDictValidator from ytdl_sub.config.validators.options import ToggleableOptionsDictValidator
from ytdl_sub.downloaders.ytdl_options_builder import YTDLOptionsBuilder from ytdl_sub.downloaders.ytdl_options_builder import YTDLOptionsBuilder
from ytdl_sub.entries.entry import Entry from ytdl_sub.entries.entry import Entry
from ytdl_sub.entries.script.variable_definitions import VARIABLES from ytdl_sub.entries.script.variable_definitions import VARIABLES
@ -21,7 +21,7 @@ from ytdl_sub.validators.validators import FloatValidator
v: VariableDefinitions = VARIABLES v: VariableDefinitions = VARIABLES
class AudioExtractOptions(OptionsDictValidator): class AudioExtractOptions(ToggleableOptionsDictValidator):
""" """
Extracts audio from a video file. Extracts audio from a video file.
@ -35,7 +35,7 @@ class AudioExtractOptions(OptionsDictValidator):
""" """
_required_keys = {"codec"} _required_keys = {"codec"}
_optional_keys = {"quality"} _optional_keys = {"enable", "quality"}
@classmethod @classmethod
def partial_validate(cls, name: str, value: Any) -> None: def partial_validate(cls, name: str, value: Any) -> None:

View file

@ -7,7 +7,7 @@ from typing import Set
from ytdl_sub.config.plugin.plugin import Plugin from ytdl_sub.config.plugin.plugin import Plugin
from ytdl_sub.config.plugin.plugin_operation import PluginOperation from ytdl_sub.config.plugin.plugin_operation import PluginOperation
from ytdl_sub.config.validators.options import OptionsDictValidator from ytdl_sub.config.validators.options import ToggleableOptionsDictValidator
from ytdl_sub.downloaders.ytdl_options_builder import YTDLOptionsBuilder from ytdl_sub.downloaders.ytdl_options_builder import YTDLOptionsBuilder
from ytdl_sub.entries.entry import Entry from ytdl_sub.entries.entry import Entry
from ytdl_sub.entries.entry import ytdl_sub_chapters_from_comments from ytdl_sub.entries.entry import ytdl_sub_chapters_from_comments
@ -59,7 +59,7 @@ class SponsorBlockCategoryListValidator(ListValidator[SponsorBlockCategoriesVali
_inner_list_type = SponsorBlockCategoriesValidator _inner_list_type = SponsorBlockCategoriesValidator
class ChaptersOptions(OptionsDictValidator): class ChaptersOptions(ToggleableOptionsDictValidator):
""" """
Embeds chapters to video files if they are present. Additional options to add SponsorBlock Embeds chapters to video files if they are present. Additional options to add SponsorBlock
chapters and remove specific ones. Can also remove chapters using regex. chapters and remove specific ones. Can also remove chapters using regex.
@ -90,6 +90,7 @@ class ChaptersOptions(OptionsDictValidator):
""" """
_optional_keys = { _optional_keys = {
"enable",
"embed_chapters", "embed_chapters",
"allow_chapters_from_comments", "allow_chapters_from_comments",
"sponsorblock_categories", "sponsorblock_categories",

View file

@ -3,12 +3,12 @@ from typing import Optional
from typing import Tuple from typing import Tuple
from ytdl_sub.config.plugin.plugin import Plugin from ytdl_sub.config.plugin.plugin import Plugin
from ytdl_sub.config.validators.options import OptionsDictValidator from ytdl_sub.config.validators.options import ToggleableOptionsDictValidator
from ytdl_sub.utils.datetime import to_date_str from ytdl_sub.utils.datetime import to_date_str
from ytdl_sub.validators.string_datetime import StringDatetimeValidator from ytdl_sub.validators.string_datetime import StringDatetimeValidator
class DateRangeOptions(OptionsDictValidator): class DateRangeOptions(ToggleableOptionsDictValidator):
""" """
Only download files uploaded within the specified date range. Only download files uploaded within the specified date range.
Dates must adhere to a yt-dlp datetime. From their docs: Dates must adhere to a yt-dlp datetime. From their docs:
@ -31,7 +31,7 @@ class DateRangeOptions(OptionsDictValidator):
after: "today-2weeks" after: "today-2weeks"
""" """
_optional_keys = {"before", "after"} _optional_keys = {"enable", "before", "after"}
def __init__(self, name, value): def __init__(self, name, value):
super().__init__(name, value) super().__init__(name, value)

View file

@ -7,7 +7,7 @@ from typing import Set
from ytdl_sub.config.overrides import Overrides from ytdl_sub.config.overrides import Overrides
from ytdl_sub.config.plugin.plugin import Plugin from ytdl_sub.config.plugin.plugin import Plugin
from ytdl_sub.config.plugin.plugin_operation import PluginOperation from ytdl_sub.config.plugin.plugin_operation import PluginOperation
from ytdl_sub.config.validators.options import OptionsDictValidator from ytdl_sub.config.validators.options import ToggleableOptionsDictValidator
from ytdl_sub.entries.entry import Entry from ytdl_sub.entries.entry import Entry
from ytdl_sub.entries.script.variable_definitions import VARIABLES from ytdl_sub.entries.script.variable_definitions import VARIABLES
from ytdl_sub.entries.script.variable_definitions import VariableDefinitions from ytdl_sub.entries.script.variable_definitions import VariableDefinitions
@ -28,7 +28,7 @@ class FileConvertWithValidator(StringSelectValidator):
_select_values = {"yt-dlp", "ffmpeg"} _select_values = {"yt-dlp", "ffmpeg"}
class FileConvertOptions(OptionsDictValidator): class FileConvertOptions(ToggleableOptionsDictValidator):
""" """
Converts video files from one extension to another. Converts video files from one extension to another.
@ -56,7 +56,7 @@ class FileConvertOptions(OptionsDictValidator):
""" """
_required_keys = {"convert_to"} _required_keys = {"convert_to"}
_optional_keys = {"convert_with", "ffmpeg_post_process_args"} _optional_keys = {"enable", "convert_with", "ffmpeg_post_process_args"}
@classmethod @classmethod
def partial_validate(cls, name: str, value: Any) -> None: def partial_validate(cls, name: str, value: Any) -> None:

View file

@ -1,4 +1,3 @@
import json
from typing import Dict from typing import Dict
from typing import Optional from typing import Optional
@ -8,6 +7,7 @@ from ytdl_sub.config.validators.options import OptionsValidator
from ytdl_sub.entries.entry import Entry from ytdl_sub.entries.entry import Entry
from ytdl_sub.utils.exceptions import StringFormattingException from ytdl_sub.utils.exceptions import StringFormattingException
from ytdl_sub.utils.logger import Logger from ytdl_sub.utils.logger import Logger
from ytdl_sub.utils.script import ScriptUtils
from ytdl_sub.validators.string_formatter_validators import ListFormatterValidator from ytdl_sub.validators.string_formatter_validators import ListFormatterValidator
from ytdl_sub.ytdl_additions.enhanced_download_archive import EnhancedDownloadArchive from ytdl_sub.ytdl_additions.enhanced_download_archive import EnhancedDownloadArchive
@ -53,7 +53,9 @@ class FilterExcludePlugin(Plugin[FilterExcludeOptions]):
return entry return entry
for formatter in self.plugin_options.list: for formatter in self.plugin_options.list:
out = json.loads(self.overrides.apply_formatter(formatter=formatter, entry=entry)) out = ScriptUtils.bool_formatter_output(
self.overrides.apply_formatter(formatter=formatter, entry=entry)
)
if bool(out): if bool(out):
logger.info( logger.info(
"Filtering '%s' from the filter %s evaluating to True", "Filtering '%s' from the filter %s evaluating to True",

View file

@ -1,4 +1,3 @@
import json
from typing import Dict from typing import Dict
from typing import Optional from typing import Optional
@ -8,6 +7,7 @@ from ytdl_sub.config.validators.options import OptionsValidator
from ytdl_sub.entries.entry import Entry from ytdl_sub.entries.entry import Entry
from ytdl_sub.utils.exceptions import StringFormattingException from ytdl_sub.utils.exceptions import StringFormattingException
from ytdl_sub.utils.logger import Logger from ytdl_sub.utils.logger import Logger
from ytdl_sub.utils.script import ScriptUtils
from ytdl_sub.validators.string_formatter_validators import ListFormatterValidator from ytdl_sub.validators.string_formatter_validators import ListFormatterValidator
from ytdl_sub.ytdl_additions.enhanced_download_archive import EnhancedDownloadArchive from ytdl_sub.ytdl_additions.enhanced_download_archive import EnhancedDownloadArchive
@ -61,7 +61,9 @@ class FilterIncludePlugin(Plugin[FilterIncludeOptions]):
return entry return entry
for formatter in self.plugin_options.list: for formatter in self.plugin_options.list:
out = json.loads(self.overrides.apply_formatter(formatter=formatter, entry=entry)) out = ScriptUtils.bool_formatter_output(
self.overrides.apply_formatter(formatter=formatter, entry=entry)
)
if not bool(out): if not bool(out):
logger.info( logger.info(
"Filtering '%s' from the filter %s evaluating to False", "Filtering '%s' from the filter %s evaluating to False",

View file

@ -4,7 +4,7 @@ from typing import List
from typing import Tuple from typing import Tuple
from ytdl_sub.config.plugin.plugin import Plugin from ytdl_sub.config.plugin.plugin import Plugin
from ytdl_sub.config.validators.options import OptionsDictValidator from ytdl_sub.config.validators.options import ToggleableOptionsDictValidator
from ytdl_sub.utils.logger import Logger from ytdl_sub.utils.logger import Logger
from ytdl_sub.validators.validators import StringListValidator from ytdl_sub.validators.validators import StringListValidator
@ -56,7 +56,7 @@ def combine_filters(filters: List[str], to_combine: List[str]) -> List[str]:
return output_filters return output_filters
class MatchFiltersOptions(OptionsDictValidator): class MatchFiltersOptions(ToggleableOptionsDictValidator):
""" """
Set ``--match-filters`` to pass into yt-dlp to filter entries from being downloaded. Set ``--match-filters`` to pass into yt-dlp to filter entries from being downloaded.
Uses the same syntax as yt-dlp. An entry will be downloaded if any one of the filters are met. Uses the same syntax as yt-dlp. An entry will be downloaded if any one of the filters are met.
@ -74,7 +74,7 @@ class MatchFiltersOptions(OptionsDictValidator):
# - "availability=?public" # - "availability=?public"
""" """
_optional_keys = {"filters"} _optional_keys = {"enable", "filters"}
@classmethod @classmethod
def partial_validate(cls, name: str, value: Any) -> None: def partial_validate(cls, name: str, value: Any) -> None:

View file

@ -8,7 +8,7 @@ from typing import List
from typing import Optional from typing import Optional
from ytdl_sub.config.plugin.plugin import Plugin from ytdl_sub.config.plugin.plugin import Plugin
from ytdl_sub.config.validators.options import OptionsDictValidator from ytdl_sub.config.validators.options import ToggleableOptionsDictValidator
from ytdl_sub.entries.entry import Entry from ytdl_sub.entries.entry import Entry
from ytdl_sub.utils.file_handler import FileHandler from ytdl_sub.utils.file_handler import FileHandler
from ytdl_sub.utils.file_handler import FileMetadata from ytdl_sub.utils.file_handler import FileMetadata
@ -23,13 +23,13 @@ from ytdl_sub.validators.string_formatter_validators import StringFormatterValid
from ytdl_sub.validators.validators import BoolValidator from ytdl_sub.validators.validators import BoolValidator
class SharedNfoTagsOptions(OptionsDictValidator): class SharedNfoTagsOptions(ToggleableOptionsDictValidator):
""" """
Shared code between NFO tags and Ouptut Directory NFO Tags Shared code between NFO tags and Ouptut Directory NFO Tags
""" """
_required_keys = {"nfo_name", "nfo_root", "tags"} _required_keys = {"nfo_name", "nfo_root", "tags"}
_optional_keys = {"kodi_safe"} _optional_keys = {"enable", "kodi_safe"}
@classmethod @classmethod
def partial_validate(cls, name: str, value: Any) -> None: def partial_validate(cls, name: str, value: Any) -> None:

View file

@ -32,7 +32,7 @@ class OutputDirectoryNfoTagsOptions(SharedNfoTagsOptions):
# Hack to make it so collection named seasons do not error # Hack to make it so collection named seasons do not error
# when adding output_directory_nfo info for plex # when adding output_directory_nfo info for plex
_required_keys = set() _required_keys = set()
_optional_keys = {"kodi_safe", "nfo_name", "nfo_root", "tags"} _optional_keys = {"enable", "kodi_safe", "nfo_name", "nfo_root", "tags"}
@property @property
def nfo_root(self) -> StringFormatterValidator: def nfo_root(self) -> StringFormatterValidator:

View file

@ -8,7 +8,7 @@ from typing import Set
from ytdl_sub.config.overrides import Overrides from ytdl_sub.config.overrides import Overrides
from ytdl_sub.config.plugin.plugin import Plugin from ytdl_sub.config.plugin.plugin import Plugin
from ytdl_sub.config.plugin.plugin_operation import PluginOperation from ytdl_sub.config.plugin.plugin_operation import PluginOperation
from ytdl_sub.config.validators.options import OptionsDictValidator from ytdl_sub.config.validators.options import ToggleableOptionsDictValidator
from ytdl_sub.entries.entry import Entry from ytdl_sub.entries.entry import Entry
from ytdl_sub.script.parser import parse from ytdl_sub.script.parser import parse
from ytdl_sub.script.utils.exceptions import RuntimeException from ytdl_sub.script.utils.exceptions import RuntimeException
@ -120,7 +120,7 @@ class FromSourceVariablesRegex(DictValidator):
} }
class RegexOptions(OptionsDictValidator): class RegexOptions(ToggleableOptionsDictValidator):
r""" r"""
.. attention:: .. attention::
@ -218,7 +218,7 @@ class RegexOptions(OptionsDictValidator):
""" """
_required_keys = {"from"} _required_keys = {"from"}
_optional_keys = {"skip_if_match_fails"} _optional_keys = {"enable", "skip_if_match_fails"}
@classmethod @classmethod
def partial_validate(cls, name: str, value: Any) -> None: def partial_validate(cls, name: str, value: Any) -> None:

View file

@ -6,7 +6,7 @@ from typing import Set
from ytdl_sub.config.plugin.plugin import Plugin from ytdl_sub.config.plugin.plugin import Plugin
from ytdl_sub.config.plugin.plugin_operation import PluginOperation from ytdl_sub.config.plugin.plugin_operation import PluginOperation
from ytdl_sub.config.validators.options import OptionsDictValidator from ytdl_sub.config.validators.options import ToggleableOptionsDictValidator
from ytdl_sub.downloaders.ytdl_options_builder import YTDLOptionsBuilder from ytdl_sub.downloaders.ytdl_options_builder import YTDLOptionsBuilder
from ytdl_sub.entries.entry import Entry from ytdl_sub.entries.entry import Entry
from ytdl_sub.entries.script.variable_definitions import VARIABLES from ytdl_sub.entries.script.variable_definitions import VARIABLES
@ -31,7 +31,7 @@ class SubtitlesTypeValidator(StringSelectValidator):
_select_values = SUBTITLE_EXTENSIONS _select_values = SUBTITLE_EXTENSIONS
class SubtitleOptions(OptionsDictValidator): class SubtitleOptions(ToggleableOptionsDictValidator):
""" """
Defines how to download and store subtitles. Using this plugin creates two new variables: Defines how to download and store subtitles. Using this plugin creates two new variables:
``lang`` and ``subtitles_ext``. ``lang`` is dynamic since you can download multiple subtitles. ``lang`` and ``subtitles_ext``. ``lang`` is dynamic since you can download multiple subtitles.
@ -52,6 +52,7 @@ class SubtitleOptions(OptionsDictValidator):
""" """
_optional_keys = { _optional_keys = {
"enable",
"subtitles_name", "subtitles_name",
"subtitles_type", "subtitles_type",
"embed_subtitles", "embed_subtitles",

View file

@ -6,7 +6,7 @@ from typing import Tuple
from ytdl_sub.config.overrides import Overrides from ytdl_sub.config.overrides import Overrides
from ytdl_sub.config.plugin.plugin import Plugin from ytdl_sub.config.plugin.plugin import Plugin
from ytdl_sub.config.validators.options import OptionsDictValidator from ytdl_sub.config.validators.options import ToggleableOptionsDictValidator
from ytdl_sub.entries.entry import Entry from ytdl_sub.entries.entry import Entry
from ytdl_sub.utils.file_handler import FileMetadata from ytdl_sub.utils.file_handler import FileMetadata
from ytdl_sub.utils.logger import Logger from ytdl_sub.utils.logger import Logger
@ -59,7 +59,7 @@ class RandomizedRangeValidator(StrictDictValidator):
return int(self.randomized_float()) return int(self.randomized_float())
class ThrottleProtectionOptions(OptionsDictValidator): class ThrottleProtectionOptions(ToggleableOptionsDictValidator):
""" """
Provides options to make ytdl-sub look more 'human-like' to protect from throttling. For Provides options to make ytdl-sub look more 'human-like' to protect from throttling. For
range-based values, a random number will be chosen within the range to avoid sleeps looking range-based values, a random number will be chosen within the range to avoid sleeps looking
@ -85,6 +85,7 @@ class ThrottleProtectionOptions(OptionsDictValidator):
""" """
_optional_keys = { _optional_keys = {
"enable",
"sleep_per_download_s", "sleep_per_download_s",
"sleep_per_subscription_s", "sleep_per_subscription_s",
"max_downloads_per_subscription", "max_downloads_per_subscription",

View file

@ -3,7 +3,7 @@ from typing import Any
from typing import Dict from typing import Dict
from ytdl_sub.config.plugin.plugin import Plugin from ytdl_sub.config.plugin.plugin import Plugin
from ytdl_sub.config.validators.options import OptionsDictValidator from ytdl_sub.config.validators.options import ToggleableOptionsDictValidator
from ytdl_sub.entries.entry import Entry from ytdl_sub.entries.entry import Entry
from ytdl_sub.utils.ffmpeg import add_ffmpeg_metadata_key_values from ytdl_sub.utils.ffmpeg import add_ffmpeg_metadata_key_values
from ytdl_sub.utils.file_handler import FileMetadata from ytdl_sub.utils.file_handler import FileMetadata
@ -13,7 +13,7 @@ from ytdl_sub.validators.string_formatter_validators import DictFormatterValidat
logger = Logger.get("video-tags") logger = Logger.get("video-tags")
class VideoTagsOptions(OptionsDictValidator): class VideoTagsOptions(ToggleableOptionsDictValidator):
""" """
Adds tags to every downloaded video file using ffmpeg ``-metadata key=value`` args. Adds tags to every downloaded video file using ffmpeg ``-metadata key=value`` args.
@ -27,7 +27,7 @@ class VideoTagsOptions(OptionsDictValidator):
description: "{description}" description: "{description}"
""" """
_optional_keys = {"tags"} _optional_keys = {"enable", "tags"}
_allow_extra_keys = True _allow_extra_keys = True
@classmethod @classmethod

View file

@ -248,6 +248,52 @@ class Script:
for variable_name, resolved in resolved_variables.items(): for variable_name, resolved in resolved_variables.items():
self._variables[variable_name] = SyntaxTree(ast=[resolved]) self._variables[variable_name] = SyntaxTree(ast=[resolved])
def _recursive_get_unresolved_output_filter_variables(
self, current_var: SyntaxTree, subset_to_resolve: Set[str], unresolvable: Set[Variable]
) -> Set[str]:
for var_dep in current_var.variables:
if var_dep in unresolvable:
raise ScriptVariableNotResolved(
f"Output filter variable contains the variable {var_dep} "
f"which is set as unresolvable"
)
subset_to_resolve.add(var_dep.name)
subset_to_resolve |= self._recursive_get_unresolved_output_filter_variables(
current_var=self._variables[var_dep.name],
subset_to_resolve=subset_to_resolve,
unresolvable=unresolvable,
)
return subset_to_resolve
def _get_unresolved_output_filter(
self,
unresolved: Dict[Variable, SyntaxTree],
output_filter: Set[str],
unresolvable: Set[Variable],
) -> Dict[Variable, SyntaxTree]:
"""
When an output filter is applied, only a subset of variables that the filter
depends on need to be resolved.
"""
subset_to_resolve: Set[str] = set()
for output_filter_variable in output_filter:
subset_to_resolve.add(output_filter_variable)
if output_filter_variable not in self._variables:
raise ScriptVariableNotResolved(
"Tried to specify an output filter variable that does not exist"
)
subset_to_resolve |= self._recursive_get_unresolved_output_filter_variables(
current_var=self._variables[output_filter_variable],
subset_to_resolve=subset_to_resolve,
unresolvable=unresolvable,
)
return {var: syntax for var, syntax in unresolved.items() if var.name in subset_to_resolve}
def _resolve( def _resolve(
self, self,
pre_resolved: Optional[Dict[str, Resolvable]] = None, pre_resolved: Optional[Dict[str, Resolvable]] = None,
@ -288,6 +334,13 @@ class Script:
if Variable(name) not in unresolved_filter if Variable(name) not in unresolved_filter
} }
if output_filter:
unresolved = self._get_unresolved_output_filter(
unresolved=unresolved,
output_filter=output_filter,
unresolvable=unresolvable,
)
while unresolved: while unresolved:
unresolved_count: int = len(unresolved) unresolved_count: int = len(unresolved)

View file

@ -75,6 +75,7 @@ class Subscription(SubscriptionDownload):
cls, cls,
config: ConfigFile, config: ConfigFile,
subscription_path: str | Path, subscription_path: str | Path,
subscription_matches: Optional[List[str]] = None,
subscription_override_dict: Optional[Dict] = None, subscription_override_dict: Optional[Dict] = None,
) -> List["Subscription"]: ) -> List["Subscription"]:
""" """
@ -86,6 +87,8 @@ class Subscription(SubscriptionDownload):
Validated instance of the config Validated instance of the config
subscription_path: subscription_path:
File path to the subscription yaml file File path to the subscription yaml file
subscription_matches:
Optional list, only output subscriptions that match one or more of these values
subscription_override_dict: subscription_override_dict:
Optional dict containing overrides to every subscription Optional dict containing overrides to every subscription
@ -99,16 +102,16 @@ class Subscription(SubscriptionDownload):
If subscription file is misconfigured If subscription file is misconfigured
""" """
subscriptions: List["Subscription"] = [] subscriptions: List["Subscription"] = []
subscription_dict = load_yaml(file_path=subscription_path) subscription_object = load_yaml(file_path=subscription_path)
has_file_preset = FILE_PRESET_APPLY_KEY in subscription_dict has_file_preset = FILE_PRESET_APPLY_KEY in subscription_object
# If a file preset is present... # If a file preset is present...
if has_file_preset: if has_file_preset:
# Validate it (make sure it is a dict) # Validate it (make sure it is a dict)
file_preset = LiteralDictValidator( file_preset = LiteralDictValidator(
name=f"{subscription_path}.{FILE_PRESET_APPLY_KEY}", name=f"{subscription_path}.{FILE_PRESET_APPLY_KEY}",
value=subscription_dict[FILE_PRESET_APPLY_KEY], value=subscription_object[FILE_PRESET_APPLY_KEY],
) )
# Deep copy the config and add this file preset to its preset list # Deep copy the config and add this file preset to its preset list
@ -116,7 +119,9 @@ class Subscription(SubscriptionDownload):
config.presets.dict[FILE_PRESET_APPLY_KEY] = file_preset.dict config.presets.dict[FILE_PRESET_APPLY_KEY] = file_preset.dict
subscriptions_dict: Dict[str, Any] = { subscriptions_dict: Dict[str, Any] = {
key: obj for key, obj in subscription_dict.items() if key not in [FILE_PRESET_APPLY_KEY] key: obj
for key, obj in subscription_object.items()
if key not in [FILE_PRESET_APPLY_KEY]
} }
subscriptions_dicts = SubscriptionValidator( subscriptions_dicts = SubscriptionValidator(
@ -129,7 +134,16 @@ class Subscription(SubscriptionDownload):
global_presets_to_apply=[FILE_PRESET_APPLY_KEY] if has_file_preset else [] global_presets_to_apply=[FILE_PRESET_APPLY_KEY] if has_file_preset else []
) )
for subscription_key, subscription_object in subscriptions_dicts.items(): if subscriptions_dicts and subscription_matches:
logger.info("Filtering subscriptions by name based on --match arguments")
subscriptions_dicts = {
subscription_name: subscription_object
for subscription_name, subscription_object in subscriptions_dicts.items()
if any(match in subscription_name for match in subscription_matches)
}
for subscription_name, subscription_object in subscriptions_dicts.items():
# Hard-override subscriptions here # Hard-override subscriptions here
mergedeep.merge( mergedeep.merge(
subscription_object, subscription_object,
@ -140,7 +154,7 @@ class Subscription(SubscriptionDownload):
subscriptions.append( subscriptions.append(
cls.from_dict( cls.from_dict(
config=config, config=config,
preset_name=subscription_key, preset_name=subscription_name,
preset_dict=subscription_object, preset_dict=subscription_object,
) )
) )

View file

@ -190,7 +190,7 @@ class SubscriptionDownload(BaseSubscription, ABC):
------- -------
List of plugins defined in the subscription, initialized and ready to use. List of plugins defined in the subscription, initialized and ready to use.
""" """
return [ plugins = [
plugin_type( plugin_type(
options=plugin_options, options=plugin_options,
overrides=self.overrides, overrides=self.overrides,
@ -198,6 +198,7 @@ class SubscriptionDownload(BaseSubscription, ABC):
) )
for plugin_type, plugin_options in self.plugins.zipped() for plugin_type, plugin_options in self.plugins.zipped()
] ]
return [plugin for plugin in plugins if plugin.is_enabled]
@classmethod @classmethod
def _cleanup_entry_files(cls, entry: Entry): def _cleanup_entry_files(cls, entry: Entry):

View file

@ -95,10 +95,12 @@ class SubscriptionYTDLOptions:
if self._preset.output_options.maintain_download_archive: if self._preset.output_options.maintain_download_archive:
ytdl_options["download_archive"] = self._enhanced_download_archive.working_file_path ytdl_options["download_archive"] = self._enhanced_download_archive.working_file_path
if self._preset.output_options.keep_max_files: if self._preset.output_options.keep_max_files:
# yt-dlp has a weird bug with max_downloads=1, set to 2 for safe measure keep_max_files = int(
ytdl_options["max_downloads"] = max( self._overrides.apply_formatter(self._preset.output_options.keep_max_files)
int(self._overrides.apply_formatter(self._preset.output_options.keep_max_files)), 2
) )
if keep_max_files > 0:
# yt-dlp has a weird bug with max_downloads=1, set to 2 for safe measure
ytdl_options["max_downloads"] = max(keep_max_files, 2)
return ytdl_options return ytdl_options

View file

@ -38,3 +38,15 @@ class ScriptUtils:
out = f"{{%from_json('''{dumped_json}''')}}" out = f"{{%from_json('''{dumped_json}''')}}"
return out return out
@classmethod
def bool_formatter_output(cls, output: str) -> bool:
"""
Translate formatter output to a boolean
"""
if not output or output.lower() == "false":
return False
try:
return bool(json.loads(output))
except Exception: # pylint: disable=broad-except
return True

View file

@ -14,10 +14,8 @@ from ytdl_sub.script.utils.exceptions import RuntimeException
from ytdl_sub.utils.exceptions import StringFormattingException from ytdl_sub.utils.exceptions import StringFormattingException
from ytdl_sub.utils.script import ScriptUtils from ytdl_sub.utils.script import ScriptUtils
_BASE_SCRIPT: Script = Script( BASE_SCRIPT: Script = Script(
ScriptUtils.add_sanitized_variables( dict(ScriptUtils.add_sanitized_variables(VARIABLE_SCRIPTS), **CUSTOM_FUNCTION_SCRIPTS)
dict(copy.deepcopy(VARIABLE_SCRIPTS), **copy.deepcopy(CUSTOM_FUNCTION_SCRIPTS))
)
) )
@ -37,7 +35,7 @@ class Scriptable(ABC):
""" """
Initializes with base values Initializes with base values
""" """
self._script = copy.deepcopy(_BASE_SCRIPT) self._script = copy.deepcopy(BASE_SCRIPT)
self._unresolvable = copy.deepcopy(UNRESOLVED_VARIABLES) self._unresolvable = copy.deepcopy(UNRESOLVED_VARIABLES)
@property @property

View file

@ -6,8 +6,10 @@ from typing import final
from ytdl_sub.entries.script.variable_definitions import VARIABLES from ytdl_sub.entries.script.variable_definitions import VARIABLES
from ytdl_sub.script.parser import parse from ytdl_sub.script.parser import parse
from ytdl_sub.script.script import Script from ytdl_sub.script.script import Script
from ytdl_sub.script.types.syntax_tree import SyntaxTree
from ytdl_sub.script.utils.exceptions import RuntimeException
from ytdl_sub.script.utils.exceptions import ScriptVariableNotResolved
from ytdl_sub.script.utils.exceptions import UserException from ytdl_sub.script.utils.exceptions import UserException
from ytdl_sub.script.utils.exceptions import VariableDoesNotExist
from ytdl_sub.utils.exceptions import StringFormattingVariableNotFoundException from ytdl_sub.utils.exceptions import StringFormattingVariableNotFoundException
from ytdl_sub.validators.validators import DictValidator from ytdl_sub.validators.validators import DictValidator
from ytdl_sub.validators.validators import ListValidator from ytdl_sub.validators.validators import ListValidator
@ -90,7 +92,7 @@ class OverridesStringFormatterValidator(StringFormatterValidator):
# pylint: enable=line-too-long # pylint: enable=line-too-long
class OverridesIntegerFormatterValidator(StringFormatterValidator): class OverridesIntegerFormatterValidator(OverridesStringFormatterValidator):
_expected_value_type_name = "integer" _expected_value_type_name = "integer"
def post_process(self, resolved: str) -> str: def post_process(self, resolved: str) -> str:
@ -104,6 +106,10 @@ class OverridesIntegerFormatterValidator(StringFormatterValidator):
return resolved return resolved
class OverridesBooleanFormatterValidator(OverridesStringFormatterValidator):
_expected_value_type_name = "boolean"
class ListFormatterValidator(ListValidator[StringFormatterValidator]): class ListFormatterValidator(ListValidator[StringFormatterValidator]):
_inner_list_type = StringFormatterValidator _inner_list_type = StringFormatterValidator
@ -142,21 +148,67 @@ class OverridesDictFormatterValidator(DictFormatterValidator):
_key_validator = OverridesStringFormatterValidator _key_validator = OverridesStringFormatterValidator
def to_variable_dependency_format_string(script: Script, parsed_format_string: SyntaxTree) -> str:
"""
Create a dummy format string that contains all variable deps as a string.
"""
dummy_format_string = ""
for var in parsed_format_string.variables:
dummy_format_string += f"{{ {var.name} }}"
# pylint: disable=protected-access
for variable_dependency in script._variables[var.name].variables:
dummy_format_string += f"{{ {variable_dependency.name} }}"
# pylint: enable=protected-access
return dummy_format_string
def _validate_formatter( def _validate_formatter(
mock_script: Script, mock_script: Script,
unresolved_variables: Set[str], unresolved_variables: Set[str],
formatter_validator: Union[StringFormatterValidator, OverridesStringFormatterValidator], formatter_validator: Union[StringFormatterValidator, OverridesStringFormatterValidator],
) -> None: ) -> None:
try: is_static_formatter = False
unresolvable = unresolved_variables unresolvable = unresolved_variables
if isinstance(formatter_validator, OverridesStringFormatterValidator): if isinstance(formatter_validator, OverridesStringFormatterValidator):
unresolvable = unresolved_variables.union({VARIABLES.entry_metadata.variable_name}) is_static_formatter = True
unresolvable = unresolved_variables.union({VARIABLES.entry_metadata.variable_name})
parsed = parse(
text=formatter_validator.format_string,
)
variable_names = {var.name for var in parsed.variables}
custom_function_names = {f"%{func.name}" for func in parsed.custom_functions}
if not variable_names.issubset(mock_script.variable_names):
raise StringFormattingVariableNotFoundException(
"contains the following variables that do not exist: "
f"{', '.join(sorted(variable_names - mock_script.variable_names))}"
)
if not custom_function_names.issubset(mock_script.function_names):
raise StringFormattingVariableNotFoundException(
"contains the following custom functions that do not exist: "
f"{', '.join(sorted(custom_function_names - mock_script.function_names))}"
)
if unresolved := variable_names.intersection(unresolvable):
raise StringFormattingVariableNotFoundException(
"contains the following variables that are unresolved when executing this "
f"formatter: {', '.join(sorted(unresolved))}"
)
try:
mock_script.resolve_once( mock_script.resolve_once(
{"tmp_var": formatter_validator.format_string}, {
"tmp_var": to_variable_dependency_format_string(
script=mock_script, parsed_format_string=parsed
)
},
unresolvable=unresolvable, unresolvable=unresolvable,
) )
except VariableDoesNotExist as exc: except RuntimeException as exc:
if isinstance(exc, ScriptVariableNotResolved) and is_static_formatter:
raise StringFormattingVariableNotFoundException(
"static formatters must contain variables that have no dependency to "
"entry variables"
) from exc
raise StringFormattingVariableNotFoundException(exc) from exc raise StringFormattingVariableNotFoundException(exc) from exc

View file

@ -113,7 +113,7 @@ def assert_logs(
for call_args in patched_debug.call_args_list: for call_args in patched_debug.call_args_list:
occurrences += int(expected_message in call_args.args[0]) occurrences += int(expected_message in call_args.args[0])
if expected_occurrences: if expected_occurrences is not None:
assert ( assert (
occurrences == expected_occurrences occurrences == expected_occurrences
), f"{expected_message} was expected {expected_occurrences} times, got {occurrences}" ), f"{expected_message} was expected {expected_occurrences} times, got {occurrences}"

View file

@ -30,7 +30,7 @@
"Project Zombie/Season 2011/s2011.e063001 - Project Zombie Fin.nfo": "54ea4a48116aa98480a79495036c25e9", "Project Zombie/Season 2011/s2011.e063001 - Project Zombie Fin.nfo": "54ea4a48116aa98480a79495036c25e9",
"Project Zombie/Season 2011/s2011.e112101 - Skyrim 'Ultra HD wMods' [PC]-thumb.jpg": "1718599d5189c65f7d8cf6acfa5ea851", "Project Zombie/Season 2011/s2011.e112101 - Skyrim 'Ultra HD wMods' [PC]-thumb.jpg": "1718599d5189c65f7d8cf6acfa5ea851",
"Project Zombie/Season 2011/s2011.e112101 - Skyrim 'Ultra HD wMods' [PC].info.json": "INFO_JSON", "Project Zombie/Season 2011/s2011.e112101 - Skyrim 'Ultra HD wMods' [PC].info.json": "INFO_JSON",
"Project Zombie/Season 2011/s2011.e112101 - Skyrim 'Ultra HD wMods' [PC].mp4": "2a18cc4baa198edecdc5154082f73eff", "Project Zombie/Season 2011/s2011.e112101 - Skyrim 'Ultra HD wMods' [PC].mp4": "554d46311112228c22ca9532007803cd",
"Project Zombie/Season 2011/s2011.e112101 - Skyrim 'Ultra HD wMods' [PC].nfo": "04f6aad56d85b1f65b81b6b8000f6479", "Project Zombie/Season 2011/s2011.e112101 - Skyrim 'Ultra HD wMods' [PC].nfo": "04f6aad56d85b1f65b81b6b8000f6479",
"Project Zombie/Season 2012/s2012.e012301 - Project Zombie Map Trailer-thumb.jpg": "54ebe9df801b278fdd17b21afa8373a6", "Project Zombie/Season 2012/s2012.e012301 - Project Zombie Map Trailer-thumb.jpg": "54ebe9df801b278fdd17b21afa8373a6",
"Project Zombie/Season 2012/s2012.e012301 - Project Zombie Map Trailer.info.json": "INFO_JSON", "Project Zombie/Season 2012/s2012.e012301 - Project Zombie Map Trailer.info.json": "INFO_JSON",

View file

@ -129,7 +129,7 @@ class TestPreset:
): ):
with pytest.raises( with pytest.raises(
StringFormattingVariableNotFoundException, StringFormattingVariableNotFoundException,
match="Variable dne_var does not exist.", match="contains the following variables that do not exist: dne_var",
): ):
_ = Preset( _ = Preset(
config=config_file, config=config_file,
@ -145,7 +145,7 @@ class TestPreset:
): ):
with pytest.raises( with pytest.raises(
StringFormattingVariableNotFoundException, StringFormattingVariableNotFoundException,
match="Variable dne_var does not exist", match="contains the following variables that do not exist: dne_var",
): ):
_ = Preset( _ = Preset(
config=config_file, config=config_file,
@ -161,7 +161,7 @@ class TestPreset:
): ):
with pytest.raises( with pytest.raises(
StringFormattingVariableNotFoundException, StringFormattingVariableNotFoundException,
match="Variable dne_var does not exist", match="contains the following variables that do not exist: dne_var",
): ):
_ = Preset( _ = Preset(
config=config_file, config=config_file,
@ -182,7 +182,7 @@ class TestPreset:
): ):
with pytest.raises( with pytest.raises(
StringFormattingVariableNotFoundException, StringFormattingVariableNotFoundException,
match="Variable dne_var does not exist", match="contains the following variables that do not exist: dne_var",
): ):
_ = Preset( _ = Preset(
config=config_file, config=config_file,
@ -198,6 +198,26 @@ class TestPreset:
}, },
) )
def test_preset_error__dict_override_variable_not_static(
self, config_file, output_options, youtube_video
):
with pytest.raises(
StringFormattingVariableNotFoundException,
match="static formatters must contain variables that "
"have no dependency to entry variables",
):
_ = Preset(
config=config_file,
name="test",
value={
"download": youtube_video,
"output_options": {
"output_directory": "{title}",
"file_name": "{uid}",
},
},
)
def test_preset_with_multi_url__contains_empty_url(self, config_file, output_options): def test_preset_with_multi_url__contains_empty_url(self, config_file, output_options):
_ = Preset( _ = Preset(
config=config_file, config=config_file,

View file

@ -1,3 +1,4 @@
import pytest
from conftest import assert_logs from conftest import assert_logs
from ytdl_sub.plugins.throttle_protection import logger as throttle_protection_logger from ytdl_sub.plugins.throttle_protection import logger as throttle_protection_logger
@ -57,3 +58,59 @@ class TestThrottleProtectionPlugin:
expected_occurrences=1, expected_occurrences=1,
): ):
_ = subscription.download(dry_run=False) _ = subscription.download(dry_run=False)
@pytest.mark.parametrize(
"disable_value",
[
"",
False,
"{tp_bool_string}",
"{tp_empty_string}",
],
)
def test_disabled(
self,
config,
subscription_name,
output_directory,
mock_download_collection_entries,
disable_value,
):
preset_dict = {
"preset": [
"Kodi Music Videos",
],
"overrides": {
"url": "https://your.name.here",
"music_video_directory": output_directory,
"tp_bool_string": "{ %bool(False) }",
"tp_empty_string": "",
},
"throttle_protection": {
"enable": disable_value,
"sleep_per_download_s": {
"min": 0.01,
"max": 0.01,
},
"sleep_per_subscription_s": {
"min": 0.02,
"max": 0.02,
},
},
}
subscription = Subscription.from_dict(
config=config,
preset_name=subscription_name,
preset_dict=preset_dict,
)
with mock_download_collection_entries(
is_youtube_channel=False, num_urls=1, is_extracted_audio=False
), assert_logs(
logger=throttle_protection_logger,
expected_message="Sleeping between downloads for %0.2f seconds",
log_level="debug",
expected_occurrences=0,
):
_ = subscription.download(dry_run=False)

View file

@ -1,5 +1,6 @@
import copy import copy
import pytest
from unit.script.conftest import single_variable_output from unit.script.conftest import single_variable_output
from ytdl_sub.utils.script import ScriptUtils from ytdl_sub.utils.script import ScriptUtils
@ -31,3 +32,22 @@ class TestScriptUtils:
output = single_variable_output(ScriptUtils.to_script(json_dict)) output = single_variable_output(ScriptUtils.to_script(json_dict))
assert output == expected_output assert output == expected_output
@pytest.mark.parametrize(
"input_str, expected_output",
[
("", False),
("true", True),
("false", False),
("[ ]", False),
("{ }", False),
("True", True),
("False", False),
("lol not False", True),
("0", False),
("-1", True),
("1", True),
],
)
def test_bool_formatter_output(self, input_str: str, expected_output: bool):
assert ScriptUtils.bool_formatter_output(input_str) == expected_output