Merge branch 'master' into prebuilt_presets
This commit is contained in:
commit
89f13e59d0
31 changed files with 473 additions and 103 deletions
|
|
@ -22,6 +22,15 @@ Extracts audio from a video file.
|
||||||
opus, vorbis, wav, and best to grab the best possible format at runtime.
|
opus, vorbis, wav, and best to grab the best possible format at runtime.
|
||||||
|
|
||||||
|
|
||||||
|
``enable``
|
||||||
|
|
||||||
|
:expected type: Optional[OverridesFormatter]
|
||||||
|
:description:
|
||||||
|
Can typically be left undefined to always default to enable. For preset convenience,
|
||||||
|
this field can be set using an override variable to easily toggle whether this plugin
|
||||||
|
is enabled or not via Boolean.
|
||||||
|
|
||||||
|
|
||||||
``quality``
|
``quality``
|
||||||
|
|
||||||
:expected type: Float
|
:expected type: Float
|
||||||
|
|
@ -76,6 +85,15 @@ chapters and remove specific ones. Can also remove chapters using regex.
|
||||||
Defaults to True. Embed chapters into the file.
|
Defaults to True. Embed chapters into the file.
|
||||||
|
|
||||||
|
|
||||||
|
``enable``
|
||||||
|
|
||||||
|
:expected type: Optional[OverridesFormatter]
|
||||||
|
:description:
|
||||||
|
Can typically be left undefined to always default to enable. For preset convenience,
|
||||||
|
this field can be set using an override variable to easily toggle whether this plugin
|
||||||
|
is enabled or not via Boolean.
|
||||||
|
|
||||||
|
|
||||||
``force_key_frames``
|
``force_key_frames``
|
||||||
|
|
||||||
:expected type: Optional[Boolean]
|
:expected type: Optional[Boolean]
|
||||||
|
|
@ -148,6 +166,15 @@ granularity possible.
|
||||||
Only download videos before this datetime.
|
Only download videos before this datetime.
|
||||||
|
|
||||||
|
|
||||||
|
``enable``
|
||||||
|
|
||||||
|
:expected type: Optional[OverridesFormatter]
|
||||||
|
:description:
|
||||||
|
Can typically be left undefined to always default to enable. For preset convenience,
|
||||||
|
this field can be set using an override variable to easily toggle whether this plugin
|
||||||
|
is enabled or not via Boolean.
|
||||||
|
|
||||||
|
|
||||||
----------------------------------------------------------------------------------------------------
|
----------------------------------------------------------------------------------------------------
|
||||||
|
|
||||||
download
|
download
|
||||||
|
|
@ -259,6 +286,15 @@ Also supports custom ffmpeg conversions:
|
||||||
with ``ffmpeg_post_process_args``. Defaults to ``yt-dlp``.
|
with ``ffmpeg_post_process_args``. Defaults to ``yt-dlp``.
|
||||||
|
|
||||||
|
|
||||||
|
``enable``
|
||||||
|
|
||||||
|
:expected type: Optional[OverridesFormatter]
|
||||||
|
:description:
|
||||||
|
Can typically be left undefined to always default to enable. For preset convenience,
|
||||||
|
this field can be set using an override variable to easily toggle whether this plugin
|
||||||
|
is enabled or not via Boolean.
|
||||||
|
|
||||||
|
|
||||||
``ffmpeg_post_process_args``
|
``ffmpeg_post_process_args``
|
||||||
|
|
||||||
:expected type: Optional[OverridesFormatter]
|
:expected type: Optional[OverridesFormatter]
|
||||||
|
|
@ -395,6 +431,15 @@ with a ``.nfo`` extension. You can add any values into the NFO.
|
||||||
episode: "{upload_month}{upload_day_padded}"
|
episode: "{upload_month}{upload_day_padded}"
|
||||||
kodi_safe: False
|
kodi_safe: False
|
||||||
|
|
||||||
|
``enable``
|
||||||
|
|
||||||
|
:expected type: Optional[OverridesFormatter]
|
||||||
|
:description:
|
||||||
|
Can typically be left undefined to always default to enable. For preset convenience,
|
||||||
|
this field can be set using an override variable to easily toggle whether this plugin
|
||||||
|
is enabled or not via Boolean.
|
||||||
|
|
||||||
|
|
||||||
``kodi_safe``
|
``kodi_safe``
|
||||||
|
|
||||||
:expected type: Optional[Boolean]
|
:expected type: Optional[Boolean]
|
||||||
|
|
@ -484,6 +529,15 @@ Usage:
|
||||||
# optional
|
# optional
|
||||||
kodi_safe: False
|
kodi_safe: False
|
||||||
|
|
||||||
|
``enable``
|
||||||
|
|
||||||
|
:expected type: Optional[OverridesFormatter]
|
||||||
|
:description:
|
||||||
|
Can typically be left undefined to always default to enable. For preset convenience,
|
||||||
|
this field can be set using an override variable to easily toggle whether this plugin
|
||||||
|
is enabled or not via Boolean.
|
||||||
|
|
||||||
|
|
||||||
``kodi_safe``
|
``kodi_safe``
|
||||||
|
|
||||||
:expected type: Optional[Boolean]
|
:expected type: Optional[Boolean]
|
||||||
|
|
@ -800,6 +854,15 @@ and using ``title_and_description`` can regex match/exclude from either ``title`
|
||||||
- "{upload_month}"
|
- "{upload_month}"
|
||||||
- "{upload_day}"
|
- "{upload_day}"
|
||||||
|
|
||||||
|
``enable``
|
||||||
|
|
||||||
|
:expected type: Optional[OverridesFormatter]
|
||||||
|
:description:
|
||||||
|
Can typically be left undefined to always default to enable. For preset convenience,
|
||||||
|
this field can be set using an override variable to easily toggle whether this plugin
|
||||||
|
is enabled or not via Boolean.
|
||||||
|
|
||||||
|
|
||||||
``skip_if_match_fails``
|
``skip_if_match_fails``
|
||||||
|
|
||||||
:expected type: Optional[Boolean]
|
:expected type: Optional[Boolean]
|
||||||
|
|
@ -880,6 +943,15 @@ It will set the respective language to the correct subtitle file.
|
||||||
webm files can only embed "vtt" subtitle types.
|
webm files can only embed "vtt" subtitle types.
|
||||||
|
|
||||||
|
|
||||||
|
``enable``
|
||||||
|
|
||||||
|
:expected type: Optional[OverridesFormatter]
|
||||||
|
:description:
|
||||||
|
Can typically be left undefined to always default to enable. For preset convenience,
|
||||||
|
this field can be set using an override variable to easily toggle whether this plugin
|
||||||
|
is enabled or not via Boolean.
|
||||||
|
|
||||||
|
|
||||||
``languages``
|
``languages``
|
||||||
|
|
||||||
:expected type: Optional[List[String]]
|
:expected type: Optional[List[String]]
|
||||||
|
|
@ -931,6 +1003,15 @@ scripted.
|
||||||
max: 36
|
max: 36
|
||||||
subscription_download_probability: 1.0
|
subscription_download_probability: 1.0
|
||||||
|
|
||||||
|
``enable``
|
||||||
|
|
||||||
|
:expected type: Optional[OverridesFormatter]
|
||||||
|
:description:
|
||||||
|
Can typically be left undefined to always default to enable. For preset convenience,
|
||||||
|
this field can be set using an override variable to easily toggle whether this plugin
|
||||||
|
is enabled or not via Boolean.
|
||||||
|
|
||||||
|
|
||||||
``max_downloads_per_subscription``
|
``max_downloads_per_subscription``
|
||||||
|
|
||||||
:expected type: Optional[Range]
|
:expected type: Optional[Range]
|
||||||
|
|
|
||||||
|
|
@ -107,15 +107,10 @@ def _download_subscriptions_from_yaml_files(
|
||||||
subscriptions += Subscription.from_file_path(
|
subscriptions += Subscription.from_file_path(
|
||||||
config=config,
|
config=config,
|
||||||
subscription_path=path,
|
subscription_path=path,
|
||||||
|
subscription_matches=subscription_matches,
|
||||||
subscription_override_dict=subscription_override_dict,
|
subscription_override_dict=subscription_override_dict,
|
||||||
)
|
)
|
||||||
|
|
||||||
if subscriptions and subscription_matches:
|
|
||||||
logger.info("Filtering subscriptions by name based on --match arguments")
|
|
||||||
subscriptions = [
|
|
||||||
sub for sub in subscriptions if any(match in sub.name for match in subscription_matches)
|
|
||||||
]
|
|
||||||
|
|
||||||
for subscription in subscriptions:
|
for subscription in subscriptions:
|
||||||
with subscription.exception_handling():
|
with subscription.exception_handling():
|
||||||
logger.info(
|
logger.info(
|
||||||
|
|
|
||||||
|
|
@ -1,5 +1,6 @@
|
||||||
from abc import ABC
|
from abc import ABC
|
||||||
from abc import abstractmethod
|
from abc import abstractmethod
|
||||||
|
from functools import cached_property
|
||||||
from typing import Dict
|
from typing import Dict
|
||||||
from typing import Generic
|
from typing import Generic
|
||||||
from typing import List
|
from typing import List
|
||||||
|
|
@ -8,9 +9,11 @@ from typing import Tuple
|
||||||
from typing import Type
|
from typing import Type
|
||||||
|
|
||||||
from ytdl_sub.config.overrides import Overrides
|
from ytdl_sub.config.overrides import Overrides
|
||||||
|
from ytdl_sub.config.validators.options import ToggleableOptionsDictValidator
|
||||||
from ytdl_sub.config.validators.options import TOptionsValidator
|
from ytdl_sub.config.validators.options import TOptionsValidator
|
||||||
from ytdl_sub.entries.entry import Entry
|
from ytdl_sub.entries.entry import Entry
|
||||||
from ytdl_sub.utils.file_handler import FileMetadata
|
from ytdl_sub.utils.file_handler import FileMetadata
|
||||||
|
from ytdl_sub.utils.script import ScriptUtils
|
||||||
from ytdl_sub.ytdl_additions.enhanced_download_archive import DownloadArchiver
|
from ytdl_sub.ytdl_additions.enhanced_download_archive import DownloadArchiver
|
||||||
from ytdl_sub.ytdl_additions.enhanced_download_archive import EnhancedDownloadArchive
|
from ytdl_sub.ytdl_additions.enhanced_download_archive import EnhancedDownloadArchive
|
||||||
|
|
||||||
|
|
@ -40,6 +43,17 @@ class Plugin(BasePlugin[TOptionsValidator], Generic[TOptionsValidator], ABC):
|
||||||
Class to define the new plugin functionality
|
Class to define the new plugin functionality
|
||||||
"""
|
"""
|
||||||
|
|
||||||
|
@cached_property
|
||||||
|
def is_enabled(self) -> bool:
|
||||||
|
"""
|
||||||
|
Returns True if enabled, False if disabled.
|
||||||
|
"""
|
||||||
|
if isinstance(self.plugin_options, ToggleableOptionsDictValidator):
|
||||||
|
return ScriptUtils.bool_formatter_output(
|
||||||
|
self.overrides.apply_formatter(self.plugin_options.enable)
|
||||||
|
)
|
||||||
|
return True
|
||||||
|
|
||||||
def ytdl_options_match_filters(self) -> Tuple[List[str], List[str]]:
|
def ytdl_options_match_filters(self) -> Tuple[List[str], List[str]]:
|
||||||
"""
|
"""
|
||||||
Returns
|
Returns
|
||||||
|
|
|
||||||
|
|
@ -6,6 +6,7 @@ from typing import TypeVar
|
||||||
from ytdl_sub.config.plugin.plugin_operation import PluginOperation
|
from ytdl_sub.config.plugin.plugin_operation import PluginOperation
|
||||||
from ytdl_sub.utils.exceptions import ValidationException
|
from ytdl_sub.utils.exceptions import ValidationException
|
||||||
from ytdl_sub.validators.strict_dict_validator import StrictDictValidator
|
from ytdl_sub.validators.strict_dict_validator import StrictDictValidator
|
||||||
|
from ytdl_sub.validators.string_formatter_validators import OverridesBooleanFormatterValidator
|
||||||
from ytdl_sub.validators.validators import Validator
|
from ytdl_sub.validators.validators import Validator
|
||||||
|
|
||||||
# pylint: disable=no-self-use
|
# pylint: disable=no-self-use
|
||||||
|
|
@ -57,3 +58,28 @@ TOptionsValidator = TypeVar("TOptionsValidator", bound=OptionsValidator)
|
||||||
|
|
||||||
class OptionsDictValidator(StrictDictValidator, OptionsValidator, ABC):
|
class OptionsDictValidator(StrictDictValidator, OptionsValidator, ABC):
|
||||||
pass
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
class ToggleableOptionsDictValidator(OptionsDictValidator):
|
||||||
|
_optional_keys = {"enable"}
|
||||||
|
|
||||||
|
def __init__(self, name, value):
|
||||||
|
assert (
|
||||||
|
"enable" in self._optional_keys
|
||||||
|
), f"{self.__class__.__name__} does not have enable as an optional field"
|
||||||
|
super().__init__(name, value)
|
||||||
|
|
||||||
|
self._enable = self._validate_key(
|
||||||
|
key="enable", validator=OverridesBooleanFormatterValidator, default="True"
|
||||||
|
)
|
||||||
|
|
||||||
|
@property
|
||||||
|
def enable(self) -> OverridesBooleanFormatterValidator:
|
||||||
|
"""
|
||||||
|
:expected type: Optional[OverridesFormatter]
|
||||||
|
:description:
|
||||||
|
Can typically be left undefined to always default to enable. For preset convenience,
|
||||||
|
this field can be set using an override variable to easily toggle whether this plugin
|
||||||
|
is enabled or not via Boolean.
|
||||||
|
"""
|
||||||
|
return self._enable
|
||||||
|
|
|
||||||
|
|
@ -13,9 +13,10 @@ from ytdl_sub.config.plugin.preset_plugins import PresetPlugins
|
||||||
from ytdl_sub.config.preset_options import OutputOptions
|
from ytdl_sub.config.preset_options import OutputOptions
|
||||||
from ytdl_sub.config.validators.options import OptionsValidator
|
from ytdl_sub.config.validators.options import OptionsValidator
|
||||||
from ytdl_sub.downloaders.url.validators import MultiUrlValidator
|
from ytdl_sub.downloaders.url.validators import MultiUrlValidator
|
||||||
from ytdl_sub.entries.script.variable_definitions import VARIABLE_SCRIPTS
|
|
||||||
from ytdl_sub.entries.variables.override_variables import SubscriptionVariables
|
from ytdl_sub.entries.variables.override_variables import SubscriptionVariables
|
||||||
from ytdl_sub.script.script import Script
|
from ytdl_sub.script.script import Script
|
||||||
|
from ytdl_sub.utils.scriptable import BASE_SCRIPT
|
||||||
|
from ytdl_sub.validators.string_formatter_validators import to_variable_dependency_format_string
|
||||||
from ytdl_sub.validators.string_formatter_validators import validate_formatters
|
from ytdl_sub.validators.string_formatter_validators import validate_formatters
|
||||||
|
|
||||||
|
|
||||||
|
|
@ -32,10 +33,10 @@ def _add_dummy_overrides(overrides: Overrides) -> Dict[str, str]:
|
||||||
# Have the dummy override variable contain all variable deps that it uses in the string
|
# Have the dummy override variable contain all variable deps that it uses in the string
|
||||||
dummy_overrides: Dict[str, str] = {}
|
dummy_overrides: Dict[str, str] = {}
|
||||||
for override_name in _override_variables(overrides):
|
for override_name in _override_variables(overrides):
|
||||||
dummy_overrides[override_name] = ""
|
|
||||||
# pylint: disable=protected-access
|
# pylint: disable=protected-access
|
||||||
for variable_dependency in overrides.script._variables[override_name].variables:
|
dummy_overrides[override_name] = to_variable_dependency_format_string(
|
||||||
dummy_overrides[override_name] += f"{{ {variable_dependency.name } }}"
|
script=overrides.script, parsed_format_string=overrides.script._variables[override_name]
|
||||||
|
)
|
||||||
# pylint: enable=protected-access
|
# pylint: enable=protected-access
|
||||||
return dummy_overrides
|
return dummy_overrides
|
||||||
|
|
||||||
|
|
@ -72,8 +73,15 @@ def _override_variables(overrides: Overrides) -> Set[str]:
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
def _entry_variables() -> Set[str]:
|
_DUMMY_ENTRY_VARIABLES: Dict[str, str] = {
|
||||||
return set(list(VARIABLE_SCRIPTS.keys()))
|
name: to_variable_dependency_format_string(
|
||||||
|
# pylint: disable=protected-access
|
||||||
|
script=BASE_SCRIPT,
|
||||||
|
parsed_format_string=BASE_SCRIPT._variables[name]
|
||||||
|
# pylint: enable=protected-access
|
||||||
|
)
|
||||||
|
for name in BASE_SCRIPT.variable_names
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
class VariableValidation:
|
class VariableValidation:
|
||||||
|
|
@ -97,12 +105,12 @@ class VariableValidation:
|
||||||
"""
|
"""
|
||||||
Do some gymnastics to initialize the Overrides script.
|
Do some gymnastics to initialize the Overrides script.
|
||||||
"""
|
"""
|
||||||
entry_variables = _entry_variables()
|
|
||||||
override_variables = _override_variables(overrides)
|
override_variables = _override_variables(overrides)
|
||||||
|
|
||||||
# Set resolved variables as all entry + override variables
|
# Set resolved variables as all entry + override variables
|
||||||
# at this point to generate every possible added/modified variable
|
# at this point to generate every possible added/modified variable
|
||||||
self.resolved_variables = entry_variables | override_variables
|
self.resolved_variables = set(_DUMMY_ENTRY_VARIABLES.keys()) | override_variables
|
||||||
|
plugin_variables: Set[str] = set()
|
||||||
|
|
||||||
for (
|
for (
|
||||||
plugin_options,
|
plugin_options,
|
||||||
|
|
@ -125,6 +133,7 @@ class VariableValidation:
|
||||||
# Set unresolved as variables that are added but do not exist as
|
# Set unresolved as variables that are added but do not exist as
|
||||||
# entry/override variables since they are created at run-time
|
# entry/override variables since they are created at run-time
|
||||||
self.unresolved_variables |= added_variables | modified_variables
|
self.unresolved_variables |= added_variables | modified_variables
|
||||||
|
plugin_variables |= added_variables | modified_variables
|
||||||
|
|
||||||
# Then update resolved variables to reflect that
|
# Then update resolved variables to reflect that
|
||||||
self.resolved_variables -= self.unresolved_variables
|
self.resolved_variables -= self.unresolved_variables
|
||||||
|
|
@ -137,10 +146,11 @@ class VariableValidation:
|
||||||
)
|
)
|
||||||
|
|
||||||
# copy the script and mock entry variables
|
# copy the script and mock entry variables
|
||||||
self.script = copy.deepcopy(overrides.script).add(_add_dummy_variables(entry_variables))
|
self.script = copy.deepcopy(overrides.script)
|
||||||
self.script.add(
|
self.script.add(
|
||||||
variables=_add_dummy_overrides(overrides=overrides),
|
variables=_add_dummy_overrides(overrides=overrides)
|
||||||
unresolvable=self.unresolved_variables,
|
| _add_dummy_variables(variables=plugin_variables)
|
||||||
|
| _DUMMY_ENTRY_VARIABLES
|
||||||
)
|
)
|
||||||
|
|
||||||
return self
|
return self
|
||||||
|
|
@ -158,7 +168,6 @@ class VariableValidation:
|
||||||
|
|
||||||
resolved_variables = added_variables | modified_variables
|
resolved_variables = added_variables | modified_variables
|
||||||
|
|
||||||
self.script.add(_add_dummy_variables(resolved_variables))
|
|
||||||
self.resolved_variables |= resolved_variables
|
self.resolved_variables |= resolved_variables
|
||||||
self.unresolved_variables -= resolved_variables
|
self.unresolved_variables -= resolved_variables
|
||||||
|
|
||||||
|
|
@ -177,13 +186,10 @@ class VariableValidation:
|
||||||
):
|
):
|
||||||
self._add_variables(PluginOperation.MODIFY_ENTRY_METADATA, options=plugin_options)
|
self._add_variables(PluginOperation.MODIFY_ENTRY_METADATA, options=plugin_options)
|
||||||
|
|
||||||
self._update_script()
|
|
||||||
for plugin_options in PluginMapping.order_options_by(
|
for plugin_options in PluginMapping.order_options_by(
|
||||||
self.plugins.zipped(), PluginOperation.MODIFY_ENTRY
|
self.plugins.zipped(), PluginOperation.MODIFY_ENTRY
|
||||||
):
|
):
|
||||||
added = self._add_variables(PluginOperation.MODIFY_ENTRY, options=plugin_options)
|
self._add_variables(PluginOperation.MODIFY_ENTRY, options=plugin_options)
|
||||||
if added:
|
|
||||||
self._update_script()
|
|
||||||
|
|
||||||
# Validate that any formatter in the plugin options can resolve
|
# Validate that any formatter in the plugin options can resolve
|
||||||
validate_formatters(
|
validate_formatters(
|
||||||
|
|
|
||||||
|
|
@ -102,7 +102,7 @@ class YTDLP:
|
||||||
)
|
)
|
||||||
|
|
||||||
if is_downloaded and is_thumbnail_downloaded:
|
if is_downloaded and is_thumbnail_downloaded:
|
||||||
return entry_dict
|
return entry_dict or {} # in-case yt-dlp returns None
|
||||||
|
|
||||||
# Always add check_formats
|
# Always add check_formats
|
||||||
# See https://github.com/yt-dlp/yt-dlp/issues/502
|
# See https://github.com/yt-dlp/yt-dlp/issues/502
|
||||||
|
|
@ -216,14 +216,11 @@ class YTDLP:
|
||||||
**kwargs
|
**kwargs
|
||||||
arguments passed directory to YoutubeDL extract_info
|
arguments passed directory to YoutubeDL extract_info
|
||||||
"""
|
"""
|
||||||
parent_dict: Dict = {}
|
|
||||||
try:
|
try:
|
||||||
with cls._listen_and_log_downloaded_info_json(
|
with cls._listen_and_log_downloaded_info_json(
|
||||||
working_directory=working_directory, log_prefix=log_prefix_on_info_json_dl
|
working_directory=working_directory, log_prefix=log_prefix_on_info_json_dl
|
||||||
):
|
):
|
||||||
parent_dict = cls.extract_info(
|
cls.extract_info(ytdl_options_overrides=ytdl_options_overrides, **kwargs)
|
||||||
ytdl_options_overrides=ytdl_options_overrides, **kwargs
|
|
||||||
)
|
|
||||||
except RejectedVideoReached:
|
except RejectedVideoReached:
|
||||||
cls.logger.debug(
|
cls.logger.debug(
|
||||||
"RejectedVideoReached, stopping additional downloads "
|
"RejectedVideoReached, stopping additional downloads "
|
||||||
|
|
@ -237,19 +234,34 @@ class YTDLP:
|
||||||
except MaxDownloadsReached:
|
except MaxDownloadsReached:
|
||||||
cls.logger.info("MaxDownloadsReached, stopping additional downloads.")
|
cls.logger.info("MaxDownloadsReached, stopping additional downloads.")
|
||||||
|
|
||||||
# For YouTube playlists in particular, channel metadata is not fetched. Attempt to get
|
parent_dicts: List[Dict] = []
|
||||||
# channel metadata via grabbing uploader_url info json a max of 3 times
|
entry_dicts = cls._get_entry_dicts_from_info_json_files(working_directory=working_directory)
|
||||||
current_iter = 0
|
entry_ids = {entry_dict.get("id") for entry_dict in entry_dicts}
|
||||||
url = kwargs.get("url")
|
|
||||||
uploader_url = parent_dict.get("uploader_url")
|
|
||||||
while current_iter < 3 and uploader_url and url != uploader_url:
|
|
||||||
cls.logger.debug("Attempting to get parent metadata from URL %s", uploader_url)
|
|
||||||
parent_dict = cls.extract_info(
|
|
||||||
ytdl_options_overrides=ytdl_options_overrides | {"playlist_items": "0:0"},
|
|
||||||
url=uploader_url,
|
|
||||||
)
|
|
||||||
current_iter += 1
|
|
||||||
url = uploader_url
|
|
||||||
uploader_url = parent_dict.get("uploader_url")
|
|
||||||
|
|
||||||
return cls._get_entry_dicts_from_info_json_files(working_directory=working_directory)
|
# Try to get additional uploader (source) metadata that yt-dlp does not fetch
|
||||||
|
# in a single request
|
||||||
|
for entry_dict in entry_dicts:
|
||||||
|
if not (uploader_id := entry_dict.get("uploader_id")):
|
||||||
|
continue
|
||||||
|
|
||||||
|
if uploader_id in entry_ids or not (uploader_url := entry_dict.get("uploader_url")):
|
||||||
|
continue
|
||||||
|
|
||||||
|
cls.logger.debug("Attempting to get parent metadata from URL %s", uploader_url)
|
||||||
|
try:
|
||||||
|
parent_dict = cls.extract_info(
|
||||||
|
ytdl_options_overrides=ytdl_options_overrides | {"playlist_items": "0:0"},
|
||||||
|
url=uploader_url,
|
||||||
|
)
|
||||||
|
except Exception: # pylint: disable=broad-except
|
||||||
|
# Do not try this uploader_id again
|
||||||
|
entry_ids.add(uploader_id)
|
||||||
|
break
|
||||||
|
|
||||||
|
if isinstance(parent_dict, dict):
|
||||||
|
parent_id = parent_dict.get("id")
|
||||||
|
parent_dicts.append(parent_dict)
|
||||||
|
entry_ids |= {uploader_id, parent_id}
|
||||||
|
cls.logger.debug("Adding parent metadata with ids [%s, %s]", uploader_id, parent_id)
|
||||||
|
|
||||||
|
return entry_dicts + parent_dicts
|
||||||
|
|
|
||||||
|
|
@ -6,7 +6,7 @@ from typing import Set
|
||||||
|
|
||||||
from ytdl_sub.config.plugin.plugin import Plugin
|
from ytdl_sub.config.plugin.plugin import Plugin
|
||||||
from ytdl_sub.config.plugin.plugin_operation import PluginOperation
|
from ytdl_sub.config.plugin.plugin_operation import PluginOperation
|
||||||
from ytdl_sub.config.validators.options import OptionsDictValidator
|
from ytdl_sub.config.validators.options import ToggleableOptionsDictValidator
|
||||||
from ytdl_sub.downloaders.ytdl_options_builder import YTDLOptionsBuilder
|
from ytdl_sub.downloaders.ytdl_options_builder import YTDLOptionsBuilder
|
||||||
from ytdl_sub.entries.entry import Entry
|
from ytdl_sub.entries.entry import Entry
|
||||||
from ytdl_sub.entries.script.variable_definitions import VARIABLES
|
from ytdl_sub.entries.script.variable_definitions import VARIABLES
|
||||||
|
|
@ -21,7 +21,7 @@ from ytdl_sub.validators.validators import FloatValidator
|
||||||
v: VariableDefinitions = VARIABLES
|
v: VariableDefinitions = VARIABLES
|
||||||
|
|
||||||
|
|
||||||
class AudioExtractOptions(OptionsDictValidator):
|
class AudioExtractOptions(ToggleableOptionsDictValidator):
|
||||||
"""
|
"""
|
||||||
Extracts audio from a video file.
|
Extracts audio from a video file.
|
||||||
|
|
||||||
|
|
@ -35,7 +35,7 @@ class AudioExtractOptions(OptionsDictValidator):
|
||||||
"""
|
"""
|
||||||
|
|
||||||
_required_keys = {"codec"}
|
_required_keys = {"codec"}
|
||||||
_optional_keys = {"quality"}
|
_optional_keys = {"enable", "quality"}
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def partial_validate(cls, name: str, value: Any) -> None:
|
def partial_validate(cls, name: str, value: Any) -> None:
|
||||||
|
|
|
||||||
|
|
@ -7,7 +7,7 @@ from typing import Set
|
||||||
|
|
||||||
from ytdl_sub.config.plugin.plugin import Plugin
|
from ytdl_sub.config.plugin.plugin import Plugin
|
||||||
from ytdl_sub.config.plugin.plugin_operation import PluginOperation
|
from ytdl_sub.config.plugin.plugin_operation import PluginOperation
|
||||||
from ytdl_sub.config.validators.options import OptionsDictValidator
|
from ytdl_sub.config.validators.options import ToggleableOptionsDictValidator
|
||||||
from ytdl_sub.downloaders.ytdl_options_builder import YTDLOptionsBuilder
|
from ytdl_sub.downloaders.ytdl_options_builder import YTDLOptionsBuilder
|
||||||
from ytdl_sub.entries.entry import Entry
|
from ytdl_sub.entries.entry import Entry
|
||||||
from ytdl_sub.entries.entry import ytdl_sub_chapters_from_comments
|
from ytdl_sub.entries.entry import ytdl_sub_chapters_from_comments
|
||||||
|
|
@ -59,7 +59,7 @@ class SponsorBlockCategoryListValidator(ListValidator[SponsorBlockCategoriesVali
|
||||||
_inner_list_type = SponsorBlockCategoriesValidator
|
_inner_list_type = SponsorBlockCategoriesValidator
|
||||||
|
|
||||||
|
|
||||||
class ChaptersOptions(OptionsDictValidator):
|
class ChaptersOptions(ToggleableOptionsDictValidator):
|
||||||
"""
|
"""
|
||||||
Embeds chapters to video files if they are present. Additional options to add SponsorBlock
|
Embeds chapters to video files if they are present. Additional options to add SponsorBlock
|
||||||
chapters and remove specific ones. Can also remove chapters using regex.
|
chapters and remove specific ones. Can also remove chapters using regex.
|
||||||
|
|
@ -90,6 +90,7 @@ class ChaptersOptions(OptionsDictValidator):
|
||||||
"""
|
"""
|
||||||
|
|
||||||
_optional_keys = {
|
_optional_keys = {
|
||||||
|
"enable",
|
||||||
"embed_chapters",
|
"embed_chapters",
|
||||||
"allow_chapters_from_comments",
|
"allow_chapters_from_comments",
|
||||||
"sponsorblock_categories",
|
"sponsorblock_categories",
|
||||||
|
|
|
||||||
|
|
@ -3,12 +3,12 @@ from typing import Optional
|
||||||
from typing import Tuple
|
from typing import Tuple
|
||||||
|
|
||||||
from ytdl_sub.config.plugin.plugin import Plugin
|
from ytdl_sub.config.plugin.plugin import Plugin
|
||||||
from ytdl_sub.config.validators.options import OptionsDictValidator
|
from ytdl_sub.config.validators.options import ToggleableOptionsDictValidator
|
||||||
from ytdl_sub.utils.datetime import to_date_str
|
from ytdl_sub.utils.datetime import to_date_str
|
||||||
from ytdl_sub.validators.string_datetime import StringDatetimeValidator
|
from ytdl_sub.validators.string_datetime import StringDatetimeValidator
|
||||||
|
|
||||||
|
|
||||||
class DateRangeOptions(OptionsDictValidator):
|
class DateRangeOptions(ToggleableOptionsDictValidator):
|
||||||
"""
|
"""
|
||||||
Only download files uploaded within the specified date range.
|
Only download files uploaded within the specified date range.
|
||||||
Dates must adhere to a yt-dlp datetime. From their docs:
|
Dates must adhere to a yt-dlp datetime. From their docs:
|
||||||
|
|
@ -31,7 +31,7 @@ class DateRangeOptions(OptionsDictValidator):
|
||||||
after: "today-2weeks"
|
after: "today-2weeks"
|
||||||
"""
|
"""
|
||||||
|
|
||||||
_optional_keys = {"before", "after"}
|
_optional_keys = {"enable", "before", "after"}
|
||||||
|
|
||||||
def __init__(self, name, value):
|
def __init__(self, name, value):
|
||||||
super().__init__(name, value)
|
super().__init__(name, value)
|
||||||
|
|
|
||||||
|
|
@ -7,7 +7,7 @@ from typing import Set
|
||||||
from ytdl_sub.config.overrides import Overrides
|
from ytdl_sub.config.overrides import Overrides
|
||||||
from ytdl_sub.config.plugin.plugin import Plugin
|
from ytdl_sub.config.plugin.plugin import Plugin
|
||||||
from ytdl_sub.config.plugin.plugin_operation import PluginOperation
|
from ytdl_sub.config.plugin.plugin_operation import PluginOperation
|
||||||
from ytdl_sub.config.validators.options import OptionsDictValidator
|
from ytdl_sub.config.validators.options import ToggleableOptionsDictValidator
|
||||||
from ytdl_sub.entries.entry import Entry
|
from ytdl_sub.entries.entry import Entry
|
||||||
from ytdl_sub.entries.script.variable_definitions import VARIABLES
|
from ytdl_sub.entries.script.variable_definitions import VARIABLES
|
||||||
from ytdl_sub.entries.script.variable_definitions import VariableDefinitions
|
from ytdl_sub.entries.script.variable_definitions import VariableDefinitions
|
||||||
|
|
@ -28,7 +28,7 @@ class FileConvertWithValidator(StringSelectValidator):
|
||||||
_select_values = {"yt-dlp", "ffmpeg"}
|
_select_values = {"yt-dlp", "ffmpeg"}
|
||||||
|
|
||||||
|
|
||||||
class FileConvertOptions(OptionsDictValidator):
|
class FileConvertOptions(ToggleableOptionsDictValidator):
|
||||||
"""
|
"""
|
||||||
Converts video files from one extension to another.
|
Converts video files from one extension to another.
|
||||||
|
|
||||||
|
|
@ -56,7 +56,7 @@ class FileConvertOptions(OptionsDictValidator):
|
||||||
"""
|
"""
|
||||||
|
|
||||||
_required_keys = {"convert_to"}
|
_required_keys = {"convert_to"}
|
||||||
_optional_keys = {"convert_with", "ffmpeg_post_process_args"}
|
_optional_keys = {"enable", "convert_with", "ffmpeg_post_process_args"}
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def partial_validate(cls, name: str, value: Any) -> None:
|
def partial_validate(cls, name: str, value: Any) -> None:
|
||||||
|
|
|
||||||
|
|
@ -1,4 +1,3 @@
|
||||||
import json
|
|
||||||
from typing import Dict
|
from typing import Dict
|
||||||
from typing import Optional
|
from typing import Optional
|
||||||
|
|
||||||
|
|
@ -8,6 +7,7 @@ from ytdl_sub.config.validators.options import OptionsValidator
|
||||||
from ytdl_sub.entries.entry import Entry
|
from ytdl_sub.entries.entry import Entry
|
||||||
from ytdl_sub.utils.exceptions import StringFormattingException
|
from ytdl_sub.utils.exceptions import StringFormattingException
|
||||||
from ytdl_sub.utils.logger import Logger
|
from ytdl_sub.utils.logger import Logger
|
||||||
|
from ytdl_sub.utils.script import ScriptUtils
|
||||||
from ytdl_sub.validators.string_formatter_validators import ListFormatterValidator
|
from ytdl_sub.validators.string_formatter_validators import ListFormatterValidator
|
||||||
from ytdl_sub.ytdl_additions.enhanced_download_archive import EnhancedDownloadArchive
|
from ytdl_sub.ytdl_additions.enhanced_download_archive import EnhancedDownloadArchive
|
||||||
|
|
||||||
|
|
@ -53,7 +53,9 @@ class FilterExcludePlugin(Plugin[FilterExcludeOptions]):
|
||||||
return entry
|
return entry
|
||||||
|
|
||||||
for formatter in self.plugin_options.list:
|
for formatter in self.plugin_options.list:
|
||||||
out = json.loads(self.overrides.apply_formatter(formatter=formatter, entry=entry))
|
out = ScriptUtils.bool_formatter_output(
|
||||||
|
self.overrides.apply_formatter(formatter=formatter, entry=entry)
|
||||||
|
)
|
||||||
if bool(out):
|
if bool(out):
|
||||||
logger.info(
|
logger.info(
|
||||||
"Filtering '%s' from the filter %s evaluating to True",
|
"Filtering '%s' from the filter %s evaluating to True",
|
||||||
|
|
|
||||||
|
|
@ -1,4 +1,3 @@
|
||||||
import json
|
|
||||||
from typing import Dict
|
from typing import Dict
|
||||||
from typing import Optional
|
from typing import Optional
|
||||||
|
|
||||||
|
|
@ -8,6 +7,7 @@ from ytdl_sub.config.validators.options import OptionsValidator
|
||||||
from ytdl_sub.entries.entry import Entry
|
from ytdl_sub.entries.entry import Entry
|
||||||
from ytdl_sub.utils.exceptions import StringFormattingException
|
from ytdl_sub.utils.exceptions import StringFormattingException
|
||||||
from ytdl_sub.utils.logger import Logger
|
from ytdl_sub.utils.logger import Logger
|
||||||
|
from ytdl_sub.utils.script import ScriptUtils
|
||||||
from ytdl_sub.validators.string_formatter_validators import ListFormatterValidator
|
from ytdl_sub.validators.string_formatter_validators import ListFormatterValidator
|
||||||
from ytdl_sub.ytdl_additions.enhanced_download_archive import EnhancedDownloadArchive
|
from ytdl_sub.ytdl_additions.enhanced_download_archive import EnhancedDownloadArchive
|
||||||
|
|
||||||
|
|
@ -61,7 +61,9 @@ class FilterIncludePlugin(Plugin[FilterIncludeOptions]):
|
||||||
return entry
|
return entry
|
||||||
|
|
||||||
for formatter in self.plugin_options.list:
|
for formatter in self.plugin_options.list:
|
||||||
out = json.loads(self.overrides.apply_formatter(formatter=formatter, entry=entry))
|
out = ScriptUtils.bool_formatter_output(
|
||||||
|
self.overrides.apply_formatter(formatter=formatter, entry=entry)
|
||||||
|
)
|
||||||
if not bool(out):
|
if not bool(out):
|
||||||
logger.info(
|
logger.info(
|
||||||
"Filtering '%s' from the filter %s evaluating to False",
|
"Filtering '%s' from the filter %s evaluating to False",
|
||||||
|
|
|
||||||
|
|
@ -4,7 +4,7 @@ from typing import List
|
||||||
from typing import Tuple
|
from typing import Tuple
|
||||||
|
|
||||||
from ytdl_sub.config.plugin.plugin import Plugin
|
from ytdl_sub.config.plugin.plugin import Plugin
|
||||||
from ytdl_sub.config.validators.options import OptionsDictValidator
|
from ytdl_sub.config.validators.options import ToggleableOptionsDictValidator
|
||||||
from ytdl_sub.utils.logger import Logger
|
from ytdl_sub.utils.logger import Logger
|
||||||
from ytdl_sub.validators.validators import StringListValidator
|
from ytdl_sub.validators.validators import StringListValidator
|
||||||
|
|
||||||
|
|
@ -56,7 +56,7 @@ def combine_filters(filters: List[str], to_combine: List[str]) -> List[str]:
|
||||||
return output_filters
|
return output_filters
|
||||||
|
|
||||||
|
|
||||||
class MatchFiltersOptions(OptionsDictValidator):
|
class MatchFiltersOptions(ToggleableOptionsDictValidator):
|
||||||
"""
|
"""
|
||||||
Set ``--match-filters`` to pass into yt-dlp to filter entries from being downloaded.
|
Set ``--match-filters`` to pass into yt-dlp to filter entries from being downloaded.
|
||||||
Uses the same syntax as yt-dlp. An entry will be downloaded if any one of the filters are met.
|
Uses the same syntax as yt-dlp. An entry will be downloaded if any one of the filters are met.
|
||||||
|
|
@ -74,7 +74,7 @@ class MatchFiltersOptions(OptionsDictValidator):
|
||||||
# - "availability=?public"
|
# - "availability=?public"
|
||||||
"""
|
"""
|
||||||
|
|
||||||
_optional_keys = {"filters"}
|
_optional_keys = {"enable", "filters"}
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def partial_validate(cls, name: str, value: Any) -> None:
|
def partial_validate(cls, name: str, value: Any) -> None:
|
||||||
|
|
|
||||||
|
|
@ -8,7 +8,7 @@ from typing import List
|
||||||
from typing import Optional
|
from typing import Optional
|
||||||
|
|
||||||
from ytdl_sub.config.plugin.plugin import Plugin
|
from ytdl_sub.config.plugin.plugin import Plugin
|
||||||
from ytdl_sub.config.validators.options import OptionsDictValidator
|
from ytdl_sub.config.validators.options import ToggleableOptionsDictValidator
|
||||||
from ytdl_sub.entries.entry import Entry
|
from ytdl_sub.entries.entry import Entry
|
||||||
from ytdl_sub.utils.file_handler import FileHandler
|
from ytdl_sub.utils.file_handler import FileHandler
|
||||||
from ytdl_sub.utils.file_handler import FileMetadata
|
from ytdl_sub.utils.file_handler import FileMetadata
|
||||||
|
|
@ -23,13 +23,13 @@ from ytdl_sub.validators.string_formatter_validators import StringFormatterValid
|
||||||
from ytdl_sub.validators.validators import BoolValidator
|
from ytdl_sub.validators.validators import BoolValidator
|
||||||
|
|
||||||
|
|
||||||
class SharedNfoTagsOptions(OptionsDictValidator):
|
class SharedNfoTagsOptions(ToggleableOptionsDictValidator):
|
||||||
"""
|
"""
|
||||||
Shared code between NFO tags and Ouptut Directory NFO Tags
|
Shared code between NFO tags and Ouptut Directory NFO Tags
|
||||||
"""
|
"""
|
||||||
|
|
||||||
_required_keys = {"nfo_name", "nfo_root", "tags"}
|
_required_keys = {"nfo_name", "nfo_root", "tags"}
|
||||||
_optional_keys = {"kodi_safe"}
|
_optional_keys = {"enable", "kodi_safe"}
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def partial_validate(cls, name: str, value: Any) -> None:
|
def partial_validate(cls, name: str, value: Any) -> None:
|
||||||
|
|
|
||||||
|
|
@ -32,7 +32,7 @@ class OutputDirectoryNfoTagsOptions(SharedNfoTagsOptions):
|
||||||
# Hack to make it so collection named seasons do not error
|
# Hack to make it so collection named seasons do not error
|
||||||
# when adding output_directory_nfo info for plex
|
# when adding output_directory_nfo info for plex
|
||||||
_required_keys = set()
|
_required_keys = set()
|
||||||
_optional_keys = {"kodi_safe", "nfo_name", "nfo_root", "tags"}
|
_optional_keys = {"enable", "kodi_safe", "nfo_name", "nfo_root", "tags"}
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def nfo_root(self) -> StringFormatterValidator:
|
def nfo_root(self) -> StringFormatterValidator:
|
||||||
|
|
|
||||||
|
|
@ -8,7 +8,7 @@ from typing import Set
|
||||||
from ytdl_sub.config.overrides import Overrides
|
from ytdl_sub.config.overrides import Overrides
|
||||||
from ytdl_sub.config.plugin.plugin import Plugin
|
from ytdl_sub.config.plugin.plugin import Plugin
|
||||||
from ytdl_sub.config.plugin.plugin_operation import PluginOperation
|
from ytdl_sub.config.plugin.plugin_operation import PluginOperation
|
||||||
from ytdl_sub.config.validators.options import OptionsDictValidator
|
from ytdl_sub.config.validators.options import ToggleableOptionsDictValidator
|
||||||
from ytdl_sub.entries.entry import Entry
|
from ytdl_sub.entries.entry import Entry
|
||||||
from ytdl_sub.script.parser import parse
|
from ytdl_sub.script.parser import parse
|
||||||
from ytdl_sub.script.utils.exceptions import RuntimeException
|
from ytdl_sub.script.utils.exceptions import RuntimeException
|
||||||
|
|
@ -120,7 +120,7 @@ class FromSourceVariablesRegex(DictValidator):
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
class RegexOptions(OptionsDictValidator):
|
class RegexOptions(ToggleableOptionsDictValidator):
|
||||||
r"""
|
r"""
|
||||||
.. attention::
|
.. attention::
|
||||||
|
|
||||||
|
|
@ -218,7 +218,7 @@ class RegexOptions(OptionsDictValidator):
|
||||||
"""
|
"""
|
||||||
|
|
||||||
_required_keys = {"from"}
|
_required_keys = {"from"}
|
||||||
_optional_keys = {"skip_if_match_fails"}
|
_optional_keys = {"enable", "skip_if_match_fails"}
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def partial_validate(cls, name: str, value: Any) -> None:
|
def partial_validate(cls, name: str, value: Any) -> None:
|
||||||
|
|
|
||||||
|
|
@ -6,7 +6,7 @@ from typing import Set
|
||||||
|
|
||||||
from ytdl_sub.config.plugin.plugin import Plugin
|
from ytdl_sub.config.plugin.plugin import Plugin
|
||||||
from ytdl_sub.config.plugin.plugin_operation import PluginOperation
|
from ytdl_sub.config.plugin.plugin_operation import PluginOperation
|
||||||
from ytdl_sub.config.validators.options import OptionsDictValidator
|
from ytdl_sub.config.validators.options import ToggleableOptionsDictValidator
|
||||||
from ytdl_sub.downloaders.ytdl_options_builder import YTDLOptionsBuilder
|
from ytdl_sub.downloaders.ytdl_options_builder import YTDLOptionsBuilder
|
||||||
from ytdl_sub.entries.entry import Entry
|
from ytdl_sub.entries.entry import Entry
|
||||||
from ytdl_sub.entries.script.variable_definitions import VARIABLES
|
from ytdl_sub.entries.script.variable_definitions import VARIABLES
|
||||||
|
|
@ -31,7 +31,7 @@ class SubtitlesTypeValidator(StringSelectValidator):
|
||||||
_select_values = SUBTITLE_EXTENSIONS
|
_select_values = SUBTITLE_EXTENSIONS
|
||||||
|
|
||||||
|
|
||||||
class SubtitleOptions(OptionsDictValidator):
|
class SubtitleOptions(ToggleableOptionsDictValidator):
|
||||||
"""
|
"""
|
||||||
Defines how to download and store subtitles. Using this plugin creates two new variables:
|
Defines how to download and store subtitles. Using this plugin creates two new variables:
|
||||||
``lang`` and ``subtitles_ext``. ``lang`` is dynamic since you can download multiple subtitles.
|
``lang`` and ``subtitles_ext``. ``lang`` is dynamic since you can download multiple subtitles.
|
||||||
|
|
@ -52,6 +52,7 @@ class SubtitleOptions(OptionsDictValidator):
|
||||||
"""
|
"""
|
||||||
|
|
||||||
_optional_keys = {
|
_optional_keys = {
|
||||||
|
"enable",
|
||||||
"subtitles_name",
|
"subtitles_name",
|
||||||
"subtitles_type",
|
"subtitles_type",
|
||||||
"embed_subtitles",
|
"embed_subtitles",
|
||||||
|
|
|
||||||
|
|
@ -6,7 +6,7 @@ from typing import Tuple
|
||||||
|
|
||||||
from ytdl_sub.config.overrides import Overrides
|
from ytdl_sub.config.overrides import Overrides
|
||||||
from ytdl_sub.config.plugin.plugin import Plugin
|
from ytdl_sub.config.plugin.plugin import Plugin
|
||||||
from ytdl_sub.config.validators.options import OptionsDictValidator
|
from ytdl_sub.config.validators.options import ToggleableOptionsDictValidator
|
||||||
from ytdl_sub.entries.entry import Entry
|
from ytdl_sub.entries.entry import Entry
|
||||||
from ytdl_sub.utils.file_handler import FileMetadata
|
from ytdl_sub.utils.file_handler import FileMetadata
|
||||||
from ytdl_sub.utils.logger import Logger
|
from ytdl_sub.utils.logger import Logger
|
||||||
|
|
@ -59,7 +59,7 @@ class RandomizedRangeValidator(StrictDictValidator):
|
||||||
return int(self.randomized_float())
|
return int(self.randomized_float())
|
||||||
|
|
||||||
|
|
||||||
class ThrottleProtectionOptions(OptionsDictValidator):
|
class ThrottleProtectionOptions(ToggleableOptionsDictValidator):
|
||||||
"""
|
"""
|
||||||
Provides options to make ytdl-sub look more 'human-like' to protect from throttling. For
|
Provides options to make ytdl-sub look more 'human-like' to protect from throttling. For
|
||||||
range-based values, a random number will be chosen within the range to avoid sleeps looking
|
range-based values, a random number will be chosen within the range to avoid sleeps looking
|
||||||
|
|
@ -85,6 +85,7 @@ class ThrottleProtectionOptions(OptionsDictValidator):
|
||||||
"""
|
"""
|
||||||
|
|
||||||
_optional_keys = {
|
_optional_keys = {
|
||||||
|
"enable",
|
||||||
"sleep_per_download_s",
|
"sleep_per_download_s",
|
||||||
"sleep_per_subscription_s",
|
"sleep_per_subscription_s",
|
||||||
"max_downloads_per_subscription",
|
"max_downloads_per_subscription",
|
||||||
|
|
|
||||||
|
|
@ -3,7 +3,7 @@ from typing import Any
|
||||||
from typing import Dict
|
from typing import Dict
|
||||||
|
|
||||||
from ytdl_sub.config.plugin.plugin import Plugin
|
from ytdl_sub.config.plugin.plugin import Plugin
|
||||||
from ytdl_sub.config.validators.options import OptionsDictValidator
|
from ytdl_sub.config.validators.options import ToggleableOptionsDictValidator
|
||||||
from ytdl_sub.entries.entry import Entry
|
from ytdl_sub.entries.entry import Entry
|
||||||
from ytdl_sub.utils.ffmpeg import add_ffmpeg_metadata_key_values
|
from ytdl_sub.utils.ffmpeg import add_ffmpeg_metadata_key_values
|
||||||
from ytdl_sub.utils.file_handler import FileMetadata
|
from ytdl_sub.utils.file_handler import FileMetadata
|
||||||
|
|
@ -13,7 +13,7 @@ from ytdl_sub.validators.string_formatter_validators import DictFormatterValidat
|
||||||
logger = Logger.get("video-tags")
|
logger = Logger.get("video-tags")
|
||||||
|
|
||||||
|
|
||||||
class VideoTagsOptions(OptionsDictValidator):
|
class VideoTagsOptions(ToggleableOptionsDictValidator):
|
||||||
"""
|
"""
|
||||||
Adds tags to every downloaded video file using ffmpeg ``-metadata key=value`` args.
|
Adds tags to every downloaded video file using ffmpeg ``-metadata key=value`` args.
|
||||||
|
|
||||||
|
|
@ -27,7 +27,7 @@ class VideoTagsOptions(OptionsDictValidator):
|
||||||
description: "{description}"
|
description: "{description}"
|
||||||
"""
|
"""
|
||||||
|
|
||||||
_optional_keys = {"tags"}
|
_optional_keys = {"enable", "tags"}
|
||||||
_allow_extra_keys = True
|
_allow_extra_keys = True
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
|
|
|
||||||
|
|
@ -248,6 +248,52 @@ class Script:
|
||||||
for variable_name, resolved in resolved_variables.items():
|
for variable_name, resolved in resolved_variables.items():
|
||||||
self._variables[variable_name] = SyntaxTree(ast=[resolved])
|
self._variables[variable_name] = SyntaxTree(ast=[resolved])
|
||||||
|
|
||||||
|
def _recursive_get_unresolved_output_filter_variables(
|
||||||
|
self, current_var: SyntaxTree, subset_to_resolve: Set[str], unresolvable: Set[Variable]
|
||||||
|
) -> Set[str]:
|
||||||
|
for var_dep in current_var.variables:
|
||||||
|
if var_dep in unresolvable:
|
||||||
|
raise ScriptVariableNotResolved(
|
||||||
|
f"Output filter variable contains the variable {var_dep} "
|
||||||
|
f"which is set as unresolvable"
|
||||||
|
)
|
||||||
|
subset_to_resolve.add(var_dep.name)
|
||||||
|
subset_to_resolve |= self._recursive_get_unresolved_output_filter_variables(
|
||||||
|
current_var=self._variables[var_dep.name],
|
||||||
|
subset_to_resolve=subset_to_resolve,
|
||||||
|
unresolvable=unresolvable,
|
||||||
|
)
|
||||||
|
|
||||||
|
return subset_to_resolve
|
||||||
|
|
||||||
|
def _get_unresolved_output_filter(
|
||||||
|
self,
|
||||||
|
unresolved: Dict[Variable, SyntaxTree],
|
||||||
|
output_filter: Set[str],
|
||||||
|
unresolvable: Set[Variable],
|
||||||
|
) -> Dict[Variable, SyntaxTree]:
|
||||||
|
"""
|
||||||
|
When an output filter is applied, only a subset of variables that the filter
|
||||||
|
depends on need to be resolved.
|
||||||
|
"""
|
||||||
|
subset_to_resolve: Set[str] = set()
|
||||||
|
|
||||||
|
for output_filter_variable in output_filter:
|
||||||
|
subset_to_resolve.add(output_filter_variable)
|
||||||
|
|
||||||
|
if output_filter_variable not in self._variables:
|
||||||
|
raise ScriptVariableNotResolved(
|
||||||
|
"Tried to specify an output filter variable that does not exist"
|
||||||
|
)
|
||||||
|
|
||||||
|
subset_to_resolve |= self._recursive_get_unresolved_output_filter_variables(
|
||||||
|
current_var=self._variables[output_filter_variable],
|
||||||
|
subset_to_resolve=subset_to_resolve,
|
||||||
|
unresolvable=unresolvable,
|
||||||
|
)
|
||||||
|
|
||||||
|
return {var: syntax for var, syntax in unresolved.items() if var.name in subset_to_resolve}
|
||||||
|
|
||||||
def _resolve(
|
def _resolve(
|
||||||
self,
|
self,
|
||||||
pre_resolved: Optional[Dict[str, Resolvable]] = None,
|
pre_resolved: Optional[Dict[str, Resolvable]] = None,
|
||||||
|
|
@ -288,6 +334,13 @@ class Script:
|
||||||
if Variable(name) not in unresolved_filter
|
if Variable(name) not in unresolved_filter
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if output_filter:
|
||||||
|
unresolved = self._get_unresolved_output_filter(
|
||||||
|
unresolved=unresolved,
|
||||||
|
output_filter=output_filter,
|
||||||
|
unresolvable=unresolvable,
|
||||||
|
)
|
||||||
|
|
||||||
while unresolved:
|
while unresolved:
|
||||||
unresolved_count: int = len(unresolved)
|
unresolved_count: int = len(unresolved)
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -75,6 +75,7 @@ class Subscription(SubscriptionDownload):
|
||||||
cls,
|
cls,
|
||||||
config: ConfigFile,
|
config: ConfigFile,
|
||||||
subscription_path: str | Path,
|
subscription_path: str | Path,
|
||||||
|
subscription_matches: Optional[List[str]] = None,
|
||||||
subscription_override_dict: Optional[Dict] = None,
|
subscription_override_dict: Optional[Dict] = None,
|
||||||
) -> List["Subscription"]:
|
) -> List["Subscription"]:
|
||||||
"""
|
"""
|
||||||
|
|
@ -86,6 +87,8 @@ class Subscription(SubscriptionDownload):
|
||||||
Validated instance of the config
|
Validated instance of the config
|
||||||
subscription_path:
|
subscription_path:
|
||||||
File path to the subscription yaml file
|
File path to the subscription yaml file
|
||||||
|
subscription_matches:
|
||||||
|
Optional list, only output subscriptions that match one or more of these values
|
||||||
subscription_override_dict:
|
subscription_override_dict:
|
||||||
Optional dict containing overrides to every subscription
|
Optional dict containing overrides to every subscription
|
||||||
|
|
||||||
|
|
@ -99,16 +102,16 @@ class Subscription(SubscriptionDownload):
|
||||||
If subscription file is misconfigured
|
If subscription file is misconfigured
|
||||||
"""
|
"""
|
||||||
subscriptions: List["Subscription"] = []
|
subscriptions: List["Subscription"] = []
|
||||||
subscription_dict = load_yaml(file_path=subscription_path)
|
subscription_object = load_yaml(file_path=subscription_path)
|
||||||
|
|
||||||
has_file_preset = FILE_PRESET_APPLY_KEY in subscription_dict
|
has_file_preset = FILE_PRESET_APPLY_KEY in subscription_object
|
||||||
|
|
||||||
# If a file preset is present...
|
# If a file preset is present...
|
||||||
if has_file_preset:
|
if has_file_preset:
|
||||||
# Validate it (make sure it is a dict)
|
# Validate it (make sure it is a dict)
|
||||||
file_preset = LiteralDictValidator(
|
file_preset = LiteralDictValidator(
|
||||||
name=f"{subscription_path}.{FILE_PRESET_APPLY_KEY}",
|
name=f"{subscription_path}.{FILE_PRESET_APPLY_KEY}",
|
||||||
value=subscription_dict[FILE_PRESET_APPLY_KEY],
|
value=subscription_object[FILE_PRESET_APPLY_KEY],
|
||||||
)
|
)
|
||||||
|
|
||||||
# Deep copy the config and add this file preset to its preset list
|
# Deep copy the config and add this file preset to its preset list
|
||||||
|
|
@ -116,7 +119,9 @@ class Subscription(SubscriptionDownload):
|
||||||
config.presets.dict[FILE_PRESET_APPLY_KEY] = file_preset.dict
|
config.presets.dict[FILE_PRESET_APPLY_KEY] = file_preset.dict
|
||||||
|
|
||||||
subscriptions_dict: Dict[str, Any] = {
|
subscriptions_dict: Dict[str, Any] = {
|
||||||
key: obj for key, obj in subscription_dict.items() if key not in [FILE_PRESET_APPLY_KEY]
|
key: obj
|
||||||
|
for key, obj in subscription_object.items()
|
||||||
|
if key not in [FILE_PRESET_APPLY_KEY]
|
||||||
}
|
}
|
||||||
|
|
||||||
subscriptions_dicts = SubscriptionValidator(
|
subscriptions_dicts = SubscriptionValidator(
|
||||||
|
|
@ -129,7 +134,16 @@ class Subscription(SubscriptionDownload):
|
||||||
global_presets_to_apply=[FILE_PRESET_APPLY_KEY] if has_file_preset else []
|
global_presets_to_apply=[FILE_PRESET_APPLY_KEY] if has_file_preset else []
|
||||||
)
|
)
|
||||||
|
|
||||||
for subscription_key, subscription_object in subscriptions_dicts.items():
|
if subscriptions_dicts and subscription_matches:
|
||||||
|
logger.info("Filtering subscriptions by name based on --match arguments")
|
||||||
|
subscriptions_dicts = {
|
||||||
|
subscription_name: subscription_object
|
||||||
|
for subscription_name, subscription_object in subscriptions_dicts.items()
|
||||||
|
if any(match in subscription_name for match in subscription_matches)
|
||||||
|
}
|
||||||
|
|
||||||
|
for subscription_name, subscription_object in subscriptions_dicts.items():
|
||||||
|
|
||||||
# Hard-override subscriptions here
|
# Hard-override subscriptions here
|
||||||
mergedeep.merge(
|
mergedeep.merge(
|
||||||
subscription_object,
|
subscription_object,
|
||||||
|
|
@ -140,7 +154,7 @@ class Subscription(SubscriptionDownload):
|
||||||
subscriptions.append(
|
subscriptions.append(
|
||||||
cls.from_dict(
|
cls.from_dict(
|
||||||
config=config,
|
config=config,
|
||||||
preset_name=subscription_key,
|
preset_name=subscription_name,
|
||||||
preset_dict=subscription_object,
|
preset_dict=subscription_object,
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
|
|
|
||||||
|
|
@ -190,7 +190,7 @@ class SubscriptionDownload(BaseSubscription, ABC):
|
||||||
-------
|
-------
|
||||||
List of plugins defined in the subscription, initialized and ready to use.
|
List of plugins defined in the subscription, initialized and ready to use.
|
||||||
"""
|
"""
|
||||||
return [
|
plugins = [
|
||||||
plugin_type(
|
plugin_type(
|
||||||
options=plugin_options,
|
options=plugin_options,
|
||||||
overrides=self.overrides,
|
overrides=self.overrides,
|
||||||
|
|
@ -198,6 +198,7 @@ class SubscriptionDownload(BaseSubscription, ABC):
|
||||||
)
|
)
|
||||||
for plugin_type, plugin_options in self.plugins.zipped()
|
for plugin_type, plugin_options in self.plugins.zipped()
|
||||||
]
|
]
|
||||||
|
return [plugin for plugin in plugins if plugin.is_enabled]
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def _cleanup_entry_files(cls, entry: Entry):
|
def _cleanup_entry_files(cls, entry: Entry):
|
||||||
|
|
|
||||||
|
|
@ -95,10 +95,12 @@ class SubscriptionYTDLOptions:
|
||||||
if self._preset.output_options.maintain_download_archive:
|
if self._preset.output_options.maintain_download_archive:
|
||||||
ytdl_options["download_archive"] = self._enhanced_download_archive.working_file_path
|
ytdl_options["download_archive"] = self._enhanced_download_archive.working_file_path
|
||||||
if self._preset.output_options.keep_max_files:
|
if self._preset.output_options.keep_max_files:
|
||||||
# yt-dlp has a weird bug with max_downloads=1, set to 2 for safe measure
|
keep_max_files = int(
|
||||||
ytdl_options["max_downloads"] = max(
|
self._overrides.apply_formatter(self._preset.output_options.keep_max_files)
|
||||||
int(self._overrides.apply_formatter(self._preset.output_options.keep_max_files)), 2
|
|
||||||
)
|
)
|
||||||
|
if keep_max_files > 0:
|
||||||
|
# yt-dlp has a weird bug with max_downloads=1, set to 2 for safe measure
|
||||||
|
ytdl_options["max_downloads"] = max(keep_max_files, 2)
|
||||||
|
|
||||||
return ytdl_options
|
return ytdl_options
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -38,3 +38,15 @@ class ScriptUtils:
|
||||||
out = f"{{%from_json('''{dumped_json}''')}}"
|
out = f"{{%from_json('''{dumped_json}''')}}"
|
||||||
|
|
||||||
return out
|
return out
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def bool_formatter_output(cls, output: str) -> bool:
|
||||||
|
"""
|
||||||
|
Translate formatter output to a boolean
|
||||||
|
"""
|
||||||
|
if not output or output.lower() == "false":
|
||||||
|
return False
|
||||||
|
try:
|
||||||
|
return bool(json.loads(output))
|
||||||
|
except Exception: # pylint: disable=broad-except
|
||||||
|
return True
|
||||||
|
|
|
||||||
|
|
@ -14,10 +14,8 @@ from ytdl_sub.script.utils.exceptions import RuntimeException
|
||||||
from ytdl_sub.utils.exceptions import StringFormattingException
|
from ytdl_sub.utils.exceptions import StringFormattingException
|
||||||
from ytdl_sub.utils.script import ScriptUtils
|
from ytdl_sub.utils.script import ScriptUtils
|
||||||
|
|
||||||
_BASE_SCRIPT: Script = Script(
|
BASE_SCRIPT: Script = Script(
|
||||||
ScriptUtils.add_sanitized_variables(
|
dict(ScriptUtils.add_sanitized_variables(VARIABLE_SCRIPTS), **CUSTOM_FUNCTION_SCRIPTS)
|
||||||
dict(copy.deepcopy(VARIABLE_SCRIPTS), **copy.deepcopy(CUSTOM_FUNCTION_SCRIPTS))
|
|
||||||
)
|
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
|
@ -37,7 +35,7 @@ class Scriptable(ABC):
|
||||||
"""
|
"""
|
||||||
Initializes with base values
|
Initializes with base values
|
||||||
"""
|
"""
|
||||||
self._script = copy.deepcopy(_BASE_SCRIPT)
|
self._script = copy.deepcopy(BASE_SCRIPT)
|
||||||
self._unresolvable = copy.deepcopy(UNRESOLVED_VARIABLES)
|
self._unresolvable = copy.deepcopy(UNRESOLVED_VARIABLES)
|
||||||
|
|
||||||
@property
|
@property
|
||||||
|
|
|
||||||
|
|
@ -6,8 +6,10 @@ from typing import final
|
||||||
from ytdl_sub.entries.script.variable_definitions import VARIABLES
|
from ytdl_sub.entries.script.variable_definitions import VARIABLES
|
||||||
from ytdl_sub.script.parser import parse
|
from ytdl_sub.script.parser import parse
|
||||||
from ytdl_sub.script.script import Script
|
from ytdl_sub.script.script import Script
|
||||||
|
from ytdl_sub.script.types.syntax_tree import SyntaxTree
|
||||||
|
from ytdl_sub.script.utils.exceptions import RuntimeException
|
||||||
|
from ytdl_sub.script.utils.exceptions import ScriptVariableNotResolved
|
||||||
from ytdl_sub.script.utils.exceptions import UserException
|
from ytdl_sub.script.utils.exceptions import UserException
|
||||||
from ytdl_sub.script.utils.exceptions import VariableDoesNotExist
|
|
||||||
from ytdl_sub.utils.exceptions import StringFormattingVariableNotFoundException
|
from ytdl_sub.utils.exceptions import StringFormattingVariableNotFoundException
|
||||||
from ytdl_sub.validators.validators import DictValidator
|
from ytdl_sub.validators.validators import DictValidator
|
||||||
from ytdl_sub.validators.validators import ListValidator
|
from ytdl_sub.validators.validators import ListValidator
|
||||||
|
|
@ -90,7 +92,7 @@ class OverridesStringFormatterValidator(StringFormatterValidator):
|
||||||
# pylint: enable=line-too-long
|
# pylint: enable=line-too-long
|
||||||
|
|
||||||
|
|
||||||
class OverridesIntegerFormatterValidator(StringFormatterValidator):
|
class OverridesIntegerFormatterValidator(OverridesStringFormatterValidator):
|
||||||
_expected_value_type_name = "integer"
|
_expected_value_type_name = "integer"
|
||||||
|
|
||||||
def post_process(self, resolved: str) -> str:
|
def post_process(self, resolved: str) -> str:
|
||||||
|
|
@ -104,6 +106,10 @@ class OverridesIntegerFormatterValidator(StringFormatterValidator):
|
||||||
return resolved
|
return resolved
|
||||||
|
|
||||||
|
|
||||||
|
class OverridesBooleanFormatterValidator(OverridesStringFormatterValidator):
|
||||||
|
_expected_value_type_name = "boolean"
|
||||||
|
|
||||||
|
|
||||||
class ListFormatterValidator(ListValidator[StringFormatterValidator]):
|
class ListFormatterValidator(ListValidator[StringFormatterValidator]):
|
||||||
_inner_list_type = StringFormatterValidator
|
_inner_list_type = StringFormatterValidator
|
||||||
|
|
||||||
|
|
@ -142,21 +148,67 @@ class OverridesDictFormatterValidator(DictFormatterValidator):
|
||||||
_key_validator = OverridesStringFormatterValidator
|
_key_validator = OverridesStringFormatterValidator
|
||||||
|
|
||||||
|
|
||||||
|
def to_variable_dependency_format_string(script: Script, parsed_format_string: SyntaxTree) -> str:
|
||||||
|
"""
|
||||||
|
Create a dummy format string that contains all variable deps as a string.
|
||||||
|
"""
|
||||||
|
dummy_format_string = ""
|
||||||
|
for var in parsed_format_string.variables:
|
||||||
|
dummy_format_string += f"{{ {var.name} }}"
|
||||||
|
# pylint: disable=protected-access
|
||||||
|
for variable_dependency in script._variables[var.name].variables:
|
||||||
|
dummy_format_string += f"{{ {variable_dependency.name} }}"
|
||||||
|
# pylint: enable=protected-access
|
||||||
|
return dummy_format_string
|
||||||
|
|
||||||
|
|
||||||
def _validate_formatter(
|
def _validate_formatter(
|
||||||
mock_script: Script,
|
mock_script: Script,
|
||||||
unresolved_variables: Set[str],
|
unresolved_variables: Set[str],
|
||||||
formatter_validator: Union[StringFormatterValidator, OverridesStringFormatterValidator],
|
formatter_validator: Union[StringFormatterValidator, OverridesStringFormatterValidator],
|
||||||
) -> None:
|
) -> None:
|
||||||
try:
|
is_static_formatter = False
|
||||||
unresolvable = unresolved_variables
|
unresolvable = unresolved_variables
|
||||||
if isinstance(formatter_validator, OverridesStringFormatterValidator):
|
if isinstance(formatter_validator, OverridesStringFormatterValidator):
|
||||||
unresolvable = unresolved_variables.union({VARIABLES.entry_metadata.variable_name})
|
is_static_formatter = True
|
||||||
|
unresolvable = unresolved_variables.union({VARIABLES.entry_metadata.variable_name})
|
||||||
|
|
||||||
|
parsed = parse(
|
||||||
|
text=formatter_validator.format_string,
|
||||||
|
)
|
||||||
|
variable_names = {var.name for var in parsed.variables}
|
||||||
|
custom_function_names = {f"%{func.name}" for func in parsed.custom_functions}
|
||||||
|
|
||||||
|
if not variable_names.issubset(mock_script.variable_names):
|
||||||
|
raise StringFormattingVariableNotFoundException(
|
||||||
|
"contains the following variables that do not exist: "
|
||||||
|
f"{', '.join(sorted(variable_names - mock_script.variable_names))}"
|
||||||
|
)
|
||||||
|
if not custom_function_names.issubset(mock_script.function_names):
|
||||||
|
raise StringFormattingVariableNotFoundException(
|
||||||
|
"contains the following custom functions that do not exist: "
|
||||||
|
f"{', '.join(sorted(custom_function_names - mock_script.function_names))}"
|
||||||
|
)
|
||||||
|
if unresolved := variable_names.intersection(unresolvable):
|
||||||
|
raise StringFormattingVariableNotFoundException(
|
||||||
|
"contains the following variables that are unresolved when executing this "
|
||||||
|
f"formatter: {', '.join(sorted(unresolved))}"
|
||||||
|
)
|
||||||
|
try:
|
||||||
mock_script.resolve_once(
|
mock_script.resolve_once(
|
||||||
{"tmp_var": formatter_validator.format_string},
|
{
|
||||||
|
"tmp_var": to_variable_dependency_format_string(
|
||||||
|
script=mock_script, parsed_format_string=parsed
|
||||||
|
)
|
||||||
|
},
|
||||||
unresolvable=unresolvable,
|
unresolvable=unresolvable,
|
||||||
)
|
)
|
||||||
except VariableDoesNotExist as exc:
|
except RuntimeException as exc:
|
||||||
|
if isinstance(exc, ScriptVariableNotResolved) and is_static_formatter:
|
||||||
|
raise StringFormattingVariableNotFoundException(
|
||||||
|
"static formatters must contain variables that have no dependency to "
|
||||||
|
"entry variables"
|
||||||
|
) from exc
|
||||||
raise StringFormattingVariableNotFoundException(exc) from exc
|
raise StringFormattingVariableNotFoundException(exc) from exc
|
||||||
|
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -113,7 +113,7 @@ def assert_logs(
|
||||||
for call_args in patched_debug.call_args_list:
|
for call_args in patched_debug.call_args_list:
|
||||||
occurrences += int(expected_message in call_args.args[0])
|
occurrences += int(expected_message in call_args.args[0])
|
||||||
|
|
||||||
if expected_occurrences:
|
if expected_occurrences is not None:
|
||||||
assert (
|
assert (
|
||||||
occurrences == expected_occurrences
|
occurrences == expected_occurrences
|
||||||
), f"{expected_message} was expected {expected_occurrences} times, got {occurrences}"
|
), f"{expected_message} was expected {expected_occurrences} times, got {occurrences}"
|
||||||
|
|
|
||||||
|
|
@ -30,7 +30,7 @@
|
||||||
"Project ⧸ Zombie/Season 2011/s2011.e063001 - Project Zombie |Fin|.nfo": "54ea4a48116aa98480a79495036c25e9",
|
"Project ⧸ Zombie/Season 2011/s2011.e063001 - Project Zombie |Fin|.nfo": "54ea4a48116aa98480a79495036c25e9",
|
||||||
"Project ⧸ Zombie/Season 2011/s2011.e112101 - Skyrim 'Ultra HD w⧸Mods' [PC]-thumb.jpg": "1718599d5189c65f7d8cf6acfa5ea851",
|
"Project ⧸ Zombie/Season 2011/s2011.e112101 - Skyrim 'Ultra HD w⧸Mods' [PC]-thumb.jpg": "1718599d5189c65f7d8cf6acfa5ea851",
|
||||||
"Project ⧸ Zombie/Season 2011/s2011.e112101 - Skyrim 'Ultra HD w⧸Mods' [PC].info.json": "INFO_JSON",
|
"Project ⧸ Zombie/Season 2011/s2011.e112101 - Skyrim 'Ultra HD w⧸Mods' [PC].info.json": "INFO_JSON",
|
||||||
"Project ⧸ Zombie/Season 2011/s2011.e112101 - Skyrim 'Ultra HD w⧸Mods' [PC].mp4": "2a18cc4baa198edecdc5154082f73eff",
|
"Project ⧸ Zombie/Season 2011/s2011.e112101 - Skyrim 'Ultra HD w⧸Mods' [PC].mp4": "554d46311112228c22ca9532007803cd",
|
||||||
"Project ⧸ Zombie/Season 2011/s2011.e112101 - Skyrim 'Ultra HD w⧸Mods' [PC].nfo": "04f6aad56d85b1f65b81b6b8000f6479",
|
"Project ⧸ Zombie/Season 2011/s2011.e112101 - Skyrim 'Ultra HD w⧸Mods' [PC].nfo": "04f6aad56d85b1f65b81b6b8000f6479",
|
||||||
"Project ⧸ Zombie/Season 2012/s2012.e012301 - Project Zombie |Map Trailer|-thumb.jpg": "54ebe9df801b278fdd17b21afa8373a6",
|
"Project ⧸ Zombie/Season 2012/s2012.e012301 - Project Zombie |Map Trailer|-thumb.jpg": "54ebe9df801b278fdd17b21afa8373a6",
|
||||||
"Project ⧸ Zombie/Season 2012/s2012.e012301 - Project Zombie |Map Trailer|.info.json": "INFO_JSON",
|
"Project ⧸ Zombie/Season 2012/s2012.e012301 - Project Zombie |Map Trailer|.info.json": "INFO_JSON",
|
||||||
|
|
|
||||||
|
|
@ -129,7 +129,7 @@ class TestPreset:
|
||||||
):
|
):
|
||||||
with pytest.raises(
|
with pytest.raises(
|
||||||
StringFormattingVariableNotFoundException,
|
StringFormattingVariableNotFoundException,
|
||||||
match="Variable dne_var does not exist.",
|
match="contains the following variables that do not exist: dne_var",
|
||||||
):
|
):
|
||||||
_ = Preset(
|
_ = Preset(
|
||||||
config=config_file,
|
config=config_file,
|
||||||
|
|
@ -145,7 +145,7 @@ class TestPreset:
|
||||||
):
|
):
|
||||||
with pytest.raises(
|
with pytest.raises(
|
||||||
StringFormattingVariableNotFoundException,
|
StringFormattingVariableNotFoundException,
|
||||||
match="Variable dne_var does not exist",
|
match="contains the following variables that do not exist: dne_var",
|
||||||
):
|
):
|
||||||
_ = Preset(
|
_ = Preset(
|
||||||
config=config_file,
|
config=config_file,
|
||||||
|
|
@ -161,7 +161,7 @@ class TestPreset:
|
||||||
):
|
):
|
||||||
with pytest.raises(
|
with pytest.raises(
|
||||||
StringFormattingVariableNotFoundException,
|
StringFormattingVariableNotFoundException,
|
||||||
match="Variable dne_var does not exist",
|
match="contains the following variables that do not exist: dne_var",
|
||||||
):
|
):
|
||||||
_ = Preset(
|
_ = Preset(
|
||||||
config=config_file,
|
config=config_file,
|
||||||
|
|
@ -182,7 +182,7 @@ class TestPreset:
|
||||||
):
|
):
|
||||||
with pytest.raises(
|
with pytest.raises(
|
||||||
StringFormattingVariableNotFoundException,
|
StringFormattingVariableNotFoundException,
|
||||||
match="Variable dne_var does not exist",
|
match="contains the following variables that do not exist: dne_var",
|
||||||
):
|
):
|
||||||
_ = Preset(
|
_ = Preset(
|
||||||
config=config_file,
|
config=config_file,
|
||||||
|
|
@ -198,6 +198,26 @@ class TestPreset:
|
||||||
},
|
},
|
||||||
)
|
)
|
||||||
|
|
||||||
|
def test_preset_error__dict_override_variable_not_static(
|
||||||
|
self, config_file, output_options, youtube_video
|
||||||
|
):
|
||||||
|
with pytest.raises(
|
||||||
|
StringFormattingVariableNotFoundException,
|
||||||
|
match="static formatters must contain variables that "
|
||||||
|
"have no dependency to entry variables",
|
||||||
|
):
|
||||||
|
_ = Preset(
|
||||||
|
config=config_file,
|
||||||
|
name="test",
|
||||||
|
value={
|
||||||
|
"download": youtube_video,
|
||||||
|
"output_options": {
|
||||||
|
"output_directory": "{title}",
|
||||||
|
"file_name": "{uid}",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
def test_preset_with_multi_url__contains_empty_url(self, config_file, output_options):
|
def test_preset_with_multi_url__contains_empty_url(self, config_file, output_options):
|
||||||
_ = Preset(
|
_ = Preset(
|
||||||
config=config_file,
|
config=config_file,
|
||||||
|
|
|
||||||
|
|
@ -1,3 +1,4 @@
|
||||||
|
import pytest
|
||||||
from conftest import assert_logs
|
from conftest import assert_logs
|
||||||
|
|
||||||
from ytdl_sub.plugins.throttle_protection import logger as throttle_protection_logger
|
from ytdl_sub.plugins.throttle_protection import logger as throttle_protection_logger
|
||||||
|
|
@ -57,3 +58,59 @@ class TestThrottleProtectionPlugin:
|
||||||
expected_occurrences=1,
|
expected_occurrences=1,
|
||||||
):
|
):
|
||||||
_ = subscription.download(dry_run=False)
|
_ = subscription.download(dry_run=False)
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
"disable_value",
|
||||||
|
[
|
||||||
|
"",
|
||||||
|
False,
|
||||||
|
"{tp_bool_string}",
|
||||||
|
"{tp_empty_string}",
|
||||||
|
],
|
||||||
|
)
|
||||||
|
def test_disabled(
|
||||||
|
self,
|
||||||
|
config,
|
||||||
|
subscription_name,
|
||||||
|
output_directory,
|
||||||
|
mock_download_collection_entries,
|
||||||
|
disable_value,
|
||||||
|
):
|
||||||
|
preset_dict = {
|
||||||
|
"preset": [
|
||||||
|
"Kodi Music Videos",
|
||||||
|
],
|
||||||
|
"overrides": {
|
||||||
|
"url": "https://your.name.here",
|
||||||
|
"music_video_directory": output_directory,
|
||||||
|
"tp_bool_string": "{ %bool(False) }",
|
||||||
|
"tp_empty_string": "",
|
||||||
|
},
|
||||||
|
"throttle_protection": {
|
||||||
|
"enable": disable_value,
|
||||||
|
"sleep_per_download_s": {
|
||||||
|
"min": 0.01,
|
||||||
|
"max": 0.01,
|
||||||
|
},
|
||||||
|
"sleep_per_subscription_s": {
|
||||||
|
"min": 0.02,
|
||||||
|
"max": 0.02,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
subscription = Subscription.from_dict(
|
||||||
|
config=config,
|
||||||
|
preset_name=subscription_name,
|
||||||
|
preset_dict=preset_dict,
|
||||||
|
)
|
||||||
|
|
||||||
|
with mock_download_collection_entries(
|
||||||
|
is_youtube_channel=False, num_urls=1, is_extracted_audio=False
|
||||||
|
), assert_logs(
|
||||||
|
logger=throttle_protection_logger,
|
||||||
|
expected_message="Sleeping between downloads for %0.2f seconds",
|
||||||
|
log_level="debug",
|
||||||
|
expected_occurrences=0,
|
||||||
|
):
|
||||||
|
_ = subscription.download(dry_run=False)
|
||||||
|
|
|
||||||
|
|
@ -1,5 +1,6 @@
|
||||||
import copy
|
import copy
|
||||||
|
|
||||||
|
import pytest
|
||||||
from unit.script.conftest import single_variable_output
|
from unit.script.conftest import single_variable_output
|
||||||
|
|
||||||
from ytdl_sub.utils.script import ScriptUtils
|
from ytdl_sub.utils.script import ScriptUtils
|
||||||
|
|
@ -31,3 +32,22 @@ class TestScriptUtils:
|
||||||
|
|
||||||
output = single_variable_output(ScriptUtils.to_script(json_dict))
|
output = single_variable_output(ScriptUtils.to_script(json_dict))
|
||||||
assert output == expected_output
|
assert output == expected_output
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
"input_str, expected_output",
|
||||||
|
[
|
||||||
|
("", False),
|
||||||
|
("true", True),
|
||||||
|
("false", False),
|
||||||
|
("[ ]", False),
|
||||||
|
("{ }", False),
|
||||||
|
("True", True),
|
||||||
|
("False", False),
|
||||||
|
("lol not False", True),
|
||||||
|
("0", False),
|
||||||
|
("-1", True),
|
||||||
|
("1", True),
|
||||||
|
],
|
||||||
|
)
|
||||||
|
def test_bool_formatter_output(self, input_str: str, expected_output: bool):
|
||||||
|
assert ScriptUtils.bool_formatter_output(input_str) == expected_output
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue