danvics/ytdl-sub
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Merge branch 'master' into prebuilt_presets

Browse source
This commit is contained in:
Qualis Svagtlys 2024-01-11 07:22:58 -06:00
commit 89f13e59d0
31 changed files with 473 additions and 103 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

81
docs/source/config_reference/plugins.rst
View file

@ -22,6 +22,15 @@ Extracts audio from a video file.
opus, vorbis, wav, and best to grab the best possible format at runtime.
``enable``
:expected type: Optional[OverridesFormatter]
:description:
Can typically be left undefined to always default to enable. For preset convenience,
this field can be set using an override variable to easily toggle whether this plugin
is enabled or not via Boolean.
``quality``
:expected type: Float
@ -76,6 +85,15 @@ chapters and remove specific ones. Can also remove chapters using regex.
Defaults to True. Embed chapters into the file.
``enable``
:expected type: Optional[OverridesFormatter]
:description:
Can typically be left undefined to always default to enable. For preset convenience,
this field can be set using an override variable to easily toggle whether this plugin
is enabled or not via Boolean.
``force_key_frames``
:expected type: Optional[Boolean]
@ -148,6 +166,15 @@ granularity possible.
Only download videos before this datetime.
``enable``
:expected type: Optional[OverridesFormatter]
:description:
Can typically be left undefined to always default to enable. For preset convenience,
this field can be set using an override variable to easily toggle whether this plugin
is enabled or not via Boolean.
----------------------------------------------------------------------------------------------------
download
@ -259,6 +286,15 @@ Also supports custom ffmpeg conversions:
with ``ffmpeg_post_process_args``. Defaults to ``yt-dlp``.
``enable``
:expected type: Optional[OverridesFormatter]
:description:
Can typically be left undefined to always default to enable. For preset convenience,
this field can be set using an override variable to easily toggle whether this plugin
is enabled or not via Boolean.
``ffmpeg_post_process_args``
:expected type: Optional[OverridesFormatter]
@ -395,6 +431,15 @@ with a ``.nfo`` extension. You can add any values into the NFO.
episode: "{upload_month}{upload_day_padded}"
kodi_safe: False
``enable``
:expected type: Optional[OverridesFormatter]
:description:
Can typically be left undefined to always default to enable. For preset convenience,
this field can be set using an override variable to easily toggle whether this plugin
is enabled or not via Boolean.
``kodi_safe``
:expected type: Optional[Boolean]
@ -484,6 +529,15 @@ Usage:
# optional
kodi_safe: False
``enable``
:expected type: Optional[OverridesFormatter]
:description:
Can typically be left undefined to always default to enable. For preset convenience,
this field can be set using an override variable to easily toggle whether this plugin
is enabled or not via Boolean.
``kodi_safe``
:expected type: Optional[Boolean]
@ -800,6 +854,15 @@ and using ``title_and_description`` can regex match/exclude from either ``title`
- "{upload_month}"
- "{upload_day}"
``enable``
:expected type: Optional[OverridesFormatter]
:description:
Can typically be left undefined to always default to enable. For preset convenience,
this field can be set using an override variable to easily toggle whether this plugin
is enabled or not via Boolean.
``skip_if_match_fails``
:expected type: Optional[Boolean]
@ -880,6 +943,15 @@ It will set the respective language to the correct subtitle file.
webm files can only embed "vtt" subtitle types.
``enable``
:expected type: Optional[OverridesFormatter]
:description:
Can typically be left undefined to always default to enable. For preset convenience,
this field can be set using an override variable to easily toggle whether this plugin
is enabled or not via Boolean.
``languages``
:expected type: Optional[List[String]]
@ -931,6 +1003,15 @@ scripted.
max: 36
subscription_download_probability: 1.0
``enable``
:expected type: Optional[OverridesFormatter]
:description:
Can typically be left undefined to always default to enable. For preset convenience,
this field can be set using an override variable to easily toggle whether this plugin
is enabled or not via Boolean.
``max_downloads_per_subscription``
:expected type: Optional[Range]

7
src/ytdl_sub/cli/entrypoint.py
View file

@ -107,15 +107,10 @@ def _download_subscriptions_from_yaml_files(
subscriptions += Subscription.from_file_path(
config=config,
subscription_path=path,
subscription_matches=subscription_matches,
subscription_override_dict=subscription_override_dict,
)
if subscriptions and subscription_matches:
logger.info("Filtering subscriptions by name based on --match arguments")
subscriptions = [
sub for sub in subscriptions if any(match in sub.name for match in subscription_matches)
]
for subscription in subscriptions:
with subscription.exception_handling():
logger.info(

14
src/ytdl_sub/config/plugin/plugin.py
View file

@ -1,5 +1,6 @@
from abc import ABC
from abc import abstractmethod
from functools import cached_property
from typing import Dict
from typing import Generic
from typing import List
@ -8,9 +9,11 @@ from typing import Tuple
from typing import Type
from ytdl_sub.config.overrides import Overrides
from ytdl_sub.config.validators.options import ToggleableOptionsDictValidator
from ytdl_sub.config.validators.options import TOptionsValidator
from ytdl_sub.entries.entry import Entry
from ytdl_sub.utils.file_handler import FileMetadata
from ytdl_sub.utils.script import ScriptUtils
from ytdl_sub.ytdl_additions.enhanced_download_archive import DownloadArchiver
from ytdl_sub.ytdl_additions.enhanced_download_archive import EnhancedDownloadArchive
@ -40,6 +43,17 @@ class Plugin(BasePlugin[TOptionsValidator], Generic[TOptionsValidator], ABC):
Class to define the new plugin functionality
"""
@cached_property
def is_enabled(self) -> bool:
"""
Returns True if enabled, False if disabled.
"""
if isinstance(self.plugin_options, ToggleableOptionsDictValidator):
return ScriptUtils.bool_formatter_output(
self.overrides.apply_formatter(self.plugin_options.enable)
)
return True
def ytdl_options_match_filters(self) -> Tuple[List[str], List[str]]:
"""
Returns

26
src/ytdl_sub/config/validators/options.py
View file

@ -6,6 +6,7 @@ from typing import TypeVar
from ytdl_sub.config.plugin.plugin_operation import PluginOperation
from ytdl_sub.utils.exceptions import ValidationException
from ytdl_sub.validators.strict_dict_validator import StrictDictValidator
from ytdl_sub.validators.string_formatter_validators import OverridesBooleanFormatterValidator
from ytdl_sub.validators.validators import Validator
# pylint: disable=no-self-use
@ -57,3 +58,28 @@ TOptionsValidator = TypeVar("TOptionsValidator", bound=OptionsValidator)
class OptionsDictValidator(StrictDictValidator, OptionsValidator, ABC):
pass
class ToggleableOptionsDictValidator(OptionsDictValidator):
_optional_keys = {"enable"}
def __init__(self, name, value):
assert (
"enable" in self._optional_keys
), f"{self.__class__.__name__} does not have enable as an optional field"
super().__init__(name, value)
self._enable = self._validate_key(
key="enable", validator=OverridesBooleanFormatterValidator, default="True"
)
@property
def enable(self) -> OverridesBooleanFormatterValidator:
"""
:expected type: Optional[OverridesFormatter]
:description:
Can typically be left undefined to always default to enable. For preset convenience,
this field can be set using an override variable to easily toggle whether this plugin
is enabled or not via Boolean.
"""
return self._enable

38
src/ytdl_sub/config/validators/variable_validation.py
View file

@ -13,9 +13,10 @@ from ytdl_sub.config.plugin.preset_plugins import PresetPlugins
from ytdl_sub.config.preset_options import OutputOptions
from ytdl_sub.config.validators.options import OptionsValidator
from ytdl_sub.downloaders.url.validators import MultiUrlValidator
from ytdl_sub.entries.script.variable_definitions import VARIABLE_SCRIPTS
from ytdl_sub.entries.variables.override_variables import SubscriptionVariables
from ytdl_sub.script.script import Script
from ytdl_sub.utils.scriptable import BASE_SCRIPT
from ytdl_sub.validators.string_formatter_validators import to_variable_dependency_format_string
from ytdl_sub.validators.string_formatter_validators import validate_formatters
@ -32,10 +33,10 @@ def _add_dummy_overrides(overrides: Overrides) -> Dict[str, str]:
# Have the dummy override variable contain all variable deps that it uses in the string
dummy_overrides: Dict[str, str] = {}
for override_name in _override_variables(overrides):
dummy_overrides[override_name] = ""
# pylint: disable=protected-access
for variable_dependency in overrides.script._variables[override_name].variables:
dummy_overrides[override_name] += f"{{ {variable_dependency.name } }}"
dummy_overrides[override_name] = to_variable_dependency_format_string(
script=overrides.script, parsed_format_string=overrides.script._variables[override_name]
)
# pylint: enable=protected-access
return dummy_overrides
@ -72,8 +73,15 @@ def _override_variables(overrides: Overrides) -> Set[str]:
}
def _entry_variables() -> Set[str]:
return set(list(VARIABLE_SCRIPTS.keys()))
_DUMMY_ENTRY_VARIABLES: Dict[str, str] = {
name: to_variable_dependency_format_string(
# pylint: disable=protected-access
script=BASE_SCRIPT,
parsed_format_string=BASE_SCRIPT._variables[name]
# pylint: enable=protected-access
)
for name in BASE_SCRIPT.variable_names
}
class VariableValidation:
@ -97,12 +105,12 @@ class VariableValidation:
"""
Do some gymnastics to initialize the Overrides script.
"""
entry_variables = _entry_variables()
override_variables = _override_variables(overrides)
# Set resolved variables as all entry + override variables
# at this point to generate every possible added/modified variable
self.resolved_variables = entry_variables | override_variables
self.resolved_variables = set(_DUMMY_ENTRY_VARIABLES.keys()) | override_variables
plugin_variables: Set[str] = set()
for (
plugin_options,
@ -125,6 +133,7 @@ class VariableValidation:
# Set unresolved as variables that are added but do not exist as
# entry/override variables since they are created at run-time
self.unresolved_variables |= added_variables | modified_variables
plugin_variables |= added_variables | modified_variables
# Then update resolved variables to reflect that
self.resolved_variables -= self.unresolved_variables
@ -137,10 +146,11 @@ class VariableValidation:
)
# copy the script and mock entry variables
self.script = copy.deepcopy(overrides.script).add(_add_dummy_variables(entry_variables))
self.script = copy.deepcopy(overrides.script)
self.script.add(
variables=_add_dummy_overrides(overrides=overrides),
unresolvable=self.unresolved_variables,
variables=_add_dummy_overrides(overrides=overrides)
| _add_dummy_variables(variables=plugin_variables)
| _DUMMY_ENTRY_VARIABLES
)
return self
@ -158,7 +168,6 @@ class VariableValidation:
resolved_variables = added_variables | modified_variables
self.script.add(_add_dummy_variables(resolved_variables))
self.resolved_variables |= resolved_variables
self.unresolved_variables -= resolved_variables
@ -177,13 +186,10 @@ class VariableValidation:
):
self._add_variables(PluginOperation.MODIFY_ENTRY_METADATA, options=plugin_options)
self._update_script()
for plugin_options in PluginMapping.order_options_by(
self.plugins.zipped(), PluginOperation.MODIFY_ENTRY
):
added = self._add_variables(PluginOperation.MODIFY_ENTRY, options=plugin_options)
if added:
self._update_script()
self._add_variables(PluginOperation.MODIFY_ENTRY, options=plugin_options)
# Validate that any formatter in the plugin options can resolve
validate_formatters(

52
src/ytdl_sub/downloaders/ytdlp.py
View file

@ -102,7 +102,7 @@ class YTDLP:
)
if is_downloaded and is_thumbnail_downloaded:
return entry_dict
return entry_dict or {} # in-case yt-dlp returns None
# Always add check_formats
# See https://github.com/yt-dlp/yt-dlp/issues/502
@ -216,14 +216,11 @@ class YTDLP:
**kwargs
arguments passed directory to YoutubeDL extract_info
"""
parent_dict: Dict = {}
try:
with cls._listen_and_log_downloaded_info_json(
working_directory=working_directory, log_prefix=log_prefix_on_info_json_dl
):
parent_dict = cls.extract_info(
ytdl_options_overrides=ytdl_options_overrides, **kwargs
)
cls.extract_info(ytdl_options_overrides=ytdl_options_overrides, **kwargs)
except RejectedVideoReached:
cls.logger.debug(
"RejectedVideoReached, stopping additional downloads "
@ -237,19 +234,34 @@ class YTDLP:
except MaxDownloadsReached:
cls.logger.info("MaxDownloadsReached, stopping additional downloads.")
# For YouTube playlists in particular, channel metadata is not fetched. Attempt to get
# channel metadata via grabbing uploader_url info json a max of 3 times
current_iter = 0
url = kwargs.get("url")
uploader_url = parent_dict.get("uploader_url")
while current_iter < 3 and uploader_url and url != uploader_url:
cls.logger.debug("Attempting to get parent metadata from URL %s", uploader_url)
parent_dict = cls.extract_info(
ytdl_options_overrides=ytdl_options_overrides | {"playlist_items": "0:0"},
url=uploader_url,
)
current_iter += 1
url = uploader_url
uploader_url = parent_dict.get("uploader_url")
parent_dicts: List[Dict] = []
entry_dicts = cls._get_entry_dicts_from_info_json_files(working_directory=working_directory)
entry_ids = {entry_dict.get("id") for entry_dict in entry_dicts}
return cls._get_entry_dicts_from_info_json_files(working_directory=working_directory)
# Try to get additional uploader (source) metadata that yt-dlp does not fetch
# in a single request
for entry_dict in entry_dicts:
if not (uploader_id := entry_dict.get("uploader_id")):
continue
if uploader_id in entry_ids or not (uploader_url := entry_dict.get("uploader_url")):
continue
cls.logger.debug("Attempting to get parent metadata from URL %s", uploader_url)
try:
parent_dict = cls.extract_info(
ytdl_options_overrides=ytdl_options_overrides | {"playlist_items": "0:0"},
url=uploader_url,
)
except Exception: # pylint: disable=broad-except
# Do not try this uploader_id again
entry_ids.add(uploader_id)
break
if isinstance(parent_dict, dict):
parent_id = parent_dict.get("id")
parent_dicts.append(parent_dict)
entry_ids |= {uploader_id, parent_id}
cls.logger.debug("Adding parent metadata with ids [%s, %s]", uploader_id, parent_id)
return entry_dicts + parent_dicts

6
src/ytdl_sub/plugins/audio_extract.py
View file

@ -6,7 +6,7 @@ from typing import Set
from ytdl_sub.config.plugin.plugin import Plugin
from ytdl_sub.config.plugin.plugin_operation import PluginOperation
from ytdl_sub.config.validators.options import OptionsDictValidator
from ytdl_sub.config.validators.options import ToggleableOptionsDictValidator
from ytdl_sub.downloaders.ytdl_options_builder import YTDLOptionsBuilder
from ytdl_sub.entries.entry import Entry
from ytdl_sub.entries.script.variable_definitions import VARIABLES
@ -21,7 +21,7 @@ from ytdl_sub.validators.validators import FloatValidator
v: VariableDefinitions = VARIABLES
class AudioExtractOptions(OptionsDictValidator):
class AudioExtractOptions(ToggleableOptionsDictValidator):
"""
Extracts audio from a video file.
@ -35,7 +35,7 @@ class AudioExtractOptions(OptionsDictValidator):
"""
_required_keys = {"codec"}
_optional_keys = {"quality"}
_optional_keys = {"enable", "quality"}
@classmethod
def partial_validate(cls, name: str, value: Any) -> None:

5
src/ytdl_sub/plugins/chapters.py
View file

@ -7,7 +7,7 @@ from typing import Set
from ytdl_sub.config.plugin.plugin import Plugin
from ytdl_sub.config.plugin.plugin_operation import PluginOperation
from ytdl_sub.config.validators.options import OptionsDictValidator
from ytdl_sub.config.validators.options import ToggleableOptionsDictValidator
from ytdl_sub.downloaders.ytdl_options_builder import YTDLOptionsBuilder
from ytdl_sub.entries.entry import Entry
from ytdl_sub.entries.entry import ytdl_sub_chapters_from_comments
@ -59,7 +59,7 @@ class SponsorBlockCategoryListValidator(ListValidator[SponsorBlockCategoriesVali
_inner_list_type = SponsorBlockCategoriesValidator
class ChaptersOptions(OptionsDictValidator):
class ChaptersOptions(ToggleableOptionsDictValidator):
"""
Embeds chapters to video files if they are present. Additional options to add SponsorBlock
chapters and remove specific ones. Can also remove chapters using regex.
@ -90,6 +90,7 @@ class ChaptersOptions(OptionsDictValidator):
"""
_optional_keys = {
"enable",
"embed_chapters",
"allow_chapters_from_comments",
"sponsorblock_categories",

6
src/ytdl_sub/plugins/date_range.py
View file

@ -3,12 +3,12 @@ from typing import Optional
from typing import Tuple
from ytdl_sub.config.plugin.plugin import Plugin
from ytdl_sub.config.validators.options import OptionsDictValidator
from ytdl_sub.config.validators.options import ToggleableOptionsDictValidator
from ytdl_sub.utils.datetime import to_date_str
from ytdl_sub.validators.string_datetime import StringDatetimeValidator
class DateRangeOptions(OptionsDictValidator):
class DateRangeOptions(ToggleableOptionsDictValidator):
"""
Only download files uploaded within the specified date range.
Dates must adhere to a yt-dlp datetime. From their docs:
@ -31,7 +31,7 @@ class DateRangeOptions(OptionsDictValidator):
after: "today-2weeks"
"""
_optional_keys = {"before", "after"}
_optional_keys = {"enable", "before", "after"}
def __init__(self, name, value):
super().__init__(name, value)

6
src/ytdl_sub/plugins/file_convert.py
View file

@ -7,7 +7,7 @@ from typing import Set
from ytdl_sub.config.overrides import Overrides
from ytdl_sub.config.plugin.plugin import Plugin
from ytdl_sub.config.plugin.plugin_operation import PluginOperation
from ytdl_sub.config.validators.options import OptionsDictValidator
from ytdl_sub.config.validators.options import ToggleableOptionsDictValidator
from ytdl_sub.entries.entry import Entry
from ytdl_sub.entries.script.variable_definitions import VARIABLES
from ytdl_sub.entries.script.variable_definitions import VariableDefinitions
@ -28,7 +28,7 @@ class FileConvertWithValidator(StringSelectValidator):
_select_values = {"yt-dlp", "ffmpeg"}
class FileConvertOptions(OptionsDictValidator):
class FileConvertOptions(ToggleableOptionsDictValidator):
"""
Converts video files from one extension to another.
@ -56,7 +56,7 @@ class FileConvertOptions(OptionsDictValidator):
"""
_required_keys = {"convert_to"}
_optional_keys = {"convert_with", "ffmpeg_post_process_args"}
_optional_keys = {"enable", "convert_with", "ffmpeg_post_process_args"}
@classmethod
def partial_validate(cls, name: str, value: Any) -> None:

6
src/ytdl_sub/plugins/filter_exclude.py
View file

@ -1,4 +1,3 @@
import json
from typing import Dict
from typing import Optional
@ -8,6 +7,7 @@ from ytdl_sub.config.validators.options import OptionsValidator
from ytdl_sub.entries.entry import Entry
from ytdl_sub.utils.exceptions import StringFormattingException
from ytdl_sub.utils.logger import Logger
from ytdl_sub.utils.script import ScriptUtils
from ytdl_sub.validators.string_formatter_validators import ListFormatterValidator
from ytdl_sub.ytdl_additions.enhanced_download_archive import EnhancedDownloadArchive
@ -53,7 +53,9 @@ class FilterExcludePlugin(Plugin[FilterExcludeOptions]):
return entry
for formatter in self.plugin_options.list:
out = json.loads(self.overrides.apply_formatter(formatter=formatter, entry=entry))
out = ScriptUtils.bool_formatter_output(
self.overrides.apply_formatter(formatter=formatter, entry=entry)
)
if bool(out):
logger.info(
"Filtering '%s' from the filter %s evaluating to True",

6
src/ytdl_sub/plugins/filter_include.py
View file

@ -1,4 +1,3 @@
import json
from typing import Dict
from typing import Optional
@ -8,6 +7,7 @@ from ytdl_sub.config.validators.options import OptionsValidator
from ytdl_sub.entries.entry import Entry
from ytdl_sub.utils.exceptions import StringFormattingException
from ytdl_sub.utils.logger import Logger
from ytdl_sub.utils.script import ScriptUtils
from ytdl_sub.validators.string_formatter_validators import ListFormatterValidator
from ytdl_sub.ytdl_additions.enhanced_download_archive import EnhancedDownloadArchive
@ -61,7 +61,9 @@ class FilterIncludePlugin(Plugin[FilterIncludeOptions]):
return entry
for formatter in self.plugin_options.list:
out = json.loads(self.overrides.apply_formatter(formatter=formatter, entry=entry))
out = ScriptUtils.bool_formatter_output(
self.overrides.apply_formatter(formatter=formatter, entry=entry)
)
if not bool(out):
logger.info(
"Filtering '%s' from the filter %s evaluating to False",

6
src/ytdl_sub/plugins/match_filters.py
View file

@ -4,7 +4,7 @@ from typing import List
from typing import Tuple
from ytdl_sub.config.plugin.plugin import Plugin
from ytdl_sub.config.validators.options import OptionsDictValidator
from ytdl_sub.config.validators.options import ToggleableOptionsDictValidator
from ytdl_sub.utils.logger import Logger
from ytdl_sub.validators.validators import StringListValidator
@ -56,7 +56,7 @@ def combine_filters(filters: List[str], to_combine: List[str]) -> List[str]:
return output_filters
class MatchFiltersOptions(OptionsDictValidator):
class MatchFiltersOptions(ToggleableOptionsDictValidator):
"""
Set ``--match-filters`` to pass into yt-dlp to filter entries from being downloaded.
Uses the same syntax as yt-dlp. An entry will be downloaded if any one of the filters are met.
@ -74,7 +74,7 @@ class MatchFiltersOptions(OptionsDictValidator):
# - "availability=?public"
"""
_optional_keys = {"filters"}
_optional_keys = {"enable", "filters"}
@classmethod
def partial_validate(cls, name: str, value: Any) -> None:

6
src/ytdl_sub/plugins/nfo_tags.py
View file

@ -8,7 +8,7 @@ from typing import List
from typing import Optional
from ytdl_sub.config.plugin.plugin import Plugin
from ytdl_sub.config.validators.options import OptionsDictValidator
from ytdl_sub.config.validators.options import ToggleableOptionsDictValidator
from ytdl_sub.entries.entry import Entry
from ytdl_sub.utils.file_handler import FileHandler
from ytdl_sub.utils.file_handler import FileMetadata
@ -23,13 +23,13 @@ from ytdl_sub.validators.string_formatter_validators import StringFormatterValid
from ytdl_sub.validators.validators import BoolValidator
class SharedNfoTagsOptions(OptionsDictValidator):
class SharedNfoTagsOptions(ToggleableOptionsDictValidator):
"""
Shared code between NFO tags and Ouptut Directory NFO Tags
"""
_required_keys = {"nfo_name", "nfo_root", "tags"}
_optional_keys = {"kodi_safe"}
_optional_keys = {"enable", "kodi_safe"}
@classmethod
def partial_validate(cls, name: str, value: Any) -> None:

2
src/ytdl_sub/plugins/output_directory_nfo_tags.py
View file

@ -32,7 +32,7 @@ class OutputDirectoryNfoTagsOptions(SharedNfoTagsOptions):
# Hack to make it so collection named seasons do not error
# when adding output_directory_nfo info for plex
_required_keys = set()
_optional_keys = {"kodi_safe", "nfo_name", "nfo_root", "tags"}
_optional_keys = {"enable", "kodi_safe", "nfo_name", "nfo_root", "tags"}
@property
def nfo_root(self) -> StringFormatterValidator:

6
src/ytdl_sub/plugins/regex.py
View file

@ -8,7 +8,7 @@ from typing import Set
from ytdl_sub.config.overrides import Overrides
from ytdl_sub.config.plugin.plugin import Plugin
from ytdl_sub.config.plugin.plugin_operation import PluginOperation
from ytdl_sub.config.validators.options import OptionsDictValidator
from ytdl_sub.config.validators.options import ToggleableOptionsDictValidator
from ytdl_sub.entries.entry import Entry
from ytdl_sub.script.parser import parse
from ytdl_sub.script.utils.exceptions import RuntimeException
@ -120,7 +120,7 @@ class FromSourceVariablesRegex(DictValidator):
}
class RegexOptions(OptionsDictValidator):
class RegexOptions(ToggleableOptionsDictValidator):
r"""
.. attention::
@ -218,7 +218,7 @@ class RegexOptions(OptionsDictValidator):
"""
_required_keys = {"from"}
_optional_keys = {"skip_if_match_fails"}
_optional_keys = {"enable", "skip_if_match_fails"}
@classmethod
def partial_validate(cls, name: str, value: Any) -> None:

5
src/ytdl_sub/plugins/subtitles.py
View file

@ -6,7 +6,7 @@ from typing import Set
from ytdl_sub.config.plugin.plugin import Plugin
from ytdl_sub.config.plugin.plugin_operation import PluginOperation
from ytdl_sub.config.validators.options import OptionsDictValidator
from ytdl_sub.config.validators.options import ToggleableOptionsDictValidator
from ytdl_sub.downloaders.ytdl_options_builder import YTDLOptionsBuilder
from ytdl_sub.entries.entry import Entry
from ytdl_sub.entries.script.variable_definitions import VARIABLES
@ -31,7 +31,7 @@ class SubtitlesTypeValidator(StringSelectValidator):
_select_values = SUBTITLE_EXTENSIONS
class SubtitleOptions(OptionsDictValidator):
class SubtitleOptions(ToggleableOptionsDictValidator):
"""
Defines how to download and store subtitles. Using this plugin creates two new variables:
``lang`` and ``subtitles_ext``. ``lang`` is dynamic since you can download multiple subtitles.
@ -52,6 +52,7 @@ class SubtitleOptions(OptionsDictValidator):
"""
_optional_keys = {
"enable",
"subtitles_name",
"subtitles_type",
"embed_subtitles",

5
src/ytdl_sub/plugins/throttle_protection.py
View file

@ -6,7 +6,7 @@ from typing import Tuple
from ytdl_sub.config.overrides import Overrides
from ytdl_sub.config.plugin.plugin import Plugin
from ytdl_sub.config.validators.options import OptionsDictValidator
from ytdl_sub.config.validators.options import ToggleableOptionsDictValidator
from ytdl_sub.entries.entry import Entry
from ytdl_sub.utils.file_handler import FileMetadata
from ytdl_sub.utils.logger import Logger
@ -59,7 +59,7 @@ class RandomizedRangeValidator(StrictDictValidator):
return int(self.randomized_float())
class ThrottleProtectionOptions(OptionsDictValidator):
class ThrottleProtectionOptions(ToggleableOptionsDictValidator):
"""
Provides options to make ytdl-sub look more 'human-like' to protect from throttling. For
range-based values, a random number will be chosen within the range to avoid sleeps looking
@ -85,6 +85,7 @@ class ThrottleProtectionOptions(OptionsDictValidator):
"""
_optional_keys = {
"enable",
"sleep_per_download_s",
"sleep_per_subscription_s",
"max_downloads_per_subscription",

6
src/ytdl_sub/plugins/video_tags.py
View file

@ -3,7 +3,7 @@ from typing import Any
from typing import Dict
from ytdl_sub.config.plugin.plugin import Plugin
from ytdl_sub.config.validators.options import OptionsDictValidator
from ytdl_sub.config.validators.options import ToggleableOptionsDictValidator
from ytdl_sub.entries.entry import Entry
from ytdl_sub.utils.ffmpeg import add_ffmpeg_metadata_key_values
from ytdl_sub.utils.file_handler import FileMetadata
@ -13,7 +13,7 @@ from ytdl_sub.validators.string_formatter_validators import DictFormatterValidat
logger = Logger.get("video-tags")
class VideoTagsOptions(OptionsDictValidator):
class VideoTagsOptions(ToggleableOptionsDictValidator):
"""
Adds tags to every downloaded video file using ffmpeg ``-metadata key=value`` args.
@ -27,7 +27,7 @@ class VideoTagsOptions(OptionsDictValidator):
description: "{description}"
"""
_optional_keys = {"tags"}
_optional_keys = {"enable", "tags"}
_allow_extra_keys = True
@classmethod

53
src/ytdl_sub/script/script.py
View file

@ -248,6 +248,52 @@ class Script:
for variable_name, resolved in resolved_variables.items():
self._variables[variable_name] = SyntaxTree(ast=[resolved])
def _recursive_get_unresolved_output_filter_variables(
self, current_var: SyntaxTree, subset_to_resolve: Set[str], unresolvable: Set[Variable]
) -> Set[str]:
for var_dep in current_var.variables:
if var_dep in unresolvable:
raise ScriptVariableNotResolved(
f"Output filter variable contains the variable {var_dep} "
f"which is set as unresolvable"
)
subset_to_resolve.add(var_dep.name)
subset_to_resolve |= self._recursive_get_unresolved_output_filter_variables(
current_var=self._variables[var_dep.name],
subset_to_resolve=subset_to_resolve,
unresolvable=unresolvable,
)
return subset_to_resolve
def _get_unresolved_output_filter(
self,
unresolved: Dict[Variable, SyntaxTree],
output_filter: Set[str],
unresolvable: Set[Variable],
) -> Dict[Variable, SyntaxTree]:
"""
When an output filter is applied, only a subset of variables that the filter
depends on need to be resolved.
"""
subset_to_resolve: Set[str] = set()
for output_filter_variable in output_filter:
subset_to_resolve.add(output_filter_variable)
if output_filter_variable not in self._variables:
raise ScriptVariableNotResolved(
"Tried to specify an output filter variable that does not exist"
)
subset_to_resolve |= self._recursive_get_unresolved_output_filter_variables(
current_var=self._variables[output_filter_variable],
subset_to_resolve=subset_to_resolve,
unresolvable=unresolvable,
)
return {var: syntax for var, syntax in unresolved.items() if var.name in subset_to_resolve}
def _resolve(
self,
pre_resolved: Optional[Dict[str, Resolvable]] = None,
@ -288,6 +334,13 @@ class Script:
if Variable(name) not in unresolved_filter
}
if output_filter:
unresolved = self._get_unresolved_output_filter(
unresolved=unresolved,
output_filter=output_filter,
unresolvable=unresolvable,
)
while unresolved:
unresolved_count: int = len(unresolved)

26
src/ytdl_sub/subscriptions/subscription.py
View file

@ -75,6 +75,7 @@ class Subscription(SubscriptionDownload):
cls,
config: ConfigFile,
subscription_path: str | Path,
subscription_matches: Optional[List[str]] = None,
subscription_override_dict: Optional[Dict] = None,
) -> List["Subscription"]:
"""
@ -86,6 +87,8 @@ class Subscription(SubscriptionDownload):
Validated instance of the config
subscription_path:
File path to the subscription yaml file
subscription_matches:
Optional list, only output subscriptions that match one or more of these values
subscription_override_dict:
Optional dict containing overrides to every subscription
@ -99,16 +102,16 @@ class Subscription(SubscriptionDownload):
If subscription file is misconfigured
"""
subscriptions: List["Subscription"] = []
subscription_dict = load_yaml(file_path=subscription_path)
subscription_object = load_yaml(file_path=subscription_path)
has_file_preset = FILE_PRESET_APPLY_KEY in subscription_dict
has_file_preset = FILE_PRESET_APPLY_KEY in subscription_object
# If a file preset is present...
if has_file_preset:
# Validate it (make sure it is a dict)
file_preset = LiteralDictValidator(
name=f"{subscription_path}.{FILE_PRESET_APPLY_KEY}",
value=subscription_dict[FILE_PRESET_APPLY_KEY],
value=subscription_object[FILE_PRESET_APPLY_KEY],
)
# Deep copy the config and add this file preset to its preset list
@ -116,7 +119,9 @@ class Subscription(SubscriptionDownload):
config.presets.dict[FILE_PRESET_APPLY_KEY] = file_preset.dict
subscriptions_dict: Dict[str, Any] = {
key: obj for key, obj in subscription_dict.items() if key not in [FILE_PRESET_APPLY_KEY]
key: obj
for key, obj in subscription_object.items()
if key not in [FILE_PRESET_APPLY_KEY]
}
subscriptions_dicts = SubscriptionValidator(
@ -129,7 +134,16 @@ class Subscription(SubscriptionDownload):
global_presets_to_apply=[FILE_PRESET_APPLY_KEY] if has_file_preset else []
)
for subscription_key, subscription_object in subscriptions_dicts.items():
if subscriptions_dicts and subscription_matches:
logger.info("Filtering subscriptions by name based on --match arguments")
subscriptions_dicts = {
subscription_name: subscription_object
for subscription_name, subscription_object in subscriptions_dicts.items()
if any(match in subscription_name for match in subscription_matches)
}
for subscription_name, subscription_object in subscriptions_dicts.items():
# Hard-override subscriptions here
mergedeep.merge(
subscription_object,
@ -140,7 +154,7 @@ class Subscription(SubscriptionDownload):
subscriptions.append(
cls.from_dict(
config=config,
preset_name=subscription_key,
preset_name=subscription_name,
preset_dict=subscription_object,
)
)

3
src/ytdl_sub/subscriptions/subscription_download.py
View file

@ -190,7 +190,7 @@ class SubscriptionDownload(BaseSubscription, ABC):
-------
List of plugins defined in the subscription, initialized and ready to use.
"""
return [
plugins = [
plugin_type(
options=plugin_options,
overrides=self.overrides,
@ -198,6 +198,7 @@ class SubscriptionDownload(BaseSubscription, ABC):
)
for plugin_type, plugin_options in self.plugins.zipped()
]
return [plugin for plugin in plugins if plugin.is_enabled]
@classmethod
def _cleanup_entry_files(cls, entry: Entry):

8
src/ytdl_sub/subscriptions/subscription_ytdl_options.py
View file

@ -95,10 +95,12 @@ class SubscriptionYTDLOptions:
if self._preset.output_options.maintain_download_archive:
ytdl_options["download_archive"] = self._enhanced_download_archive.working_file_path
if self._preset.output_options.keep_max_files:
# yt-dlp has a weird bug with max_downloads=1, set to 2 for safe measure
ytdl_options["max_downloads"] = max(
int(self._overrides.apply_formatter(self._preset.output_options.keep_max_files)), 2
keep_max_files = int(
self._overrides.apply_formatter(self._preset.output_options.keep_max_files)
)
if keep_max_files > 0:
# yt-dlp has a weird bug with max_downloads=1, set to 2 for safe measure
ytdl_options["max_downloads"] = max(keep_max_files, 2)
return ytdl_options

12
src/ytdl_sub/utils/script.py
View file

@ -38,3 +38,15 @@ class ScriptUtils:
out = f"{{%from_json('''{dumped_json}''')}}"
return out
@classmethod
def bool_formatter_output(cls, output: str) -> bool:
"""
Translate formatter output to a boolean
"""
if not output or output.lower() == "false":
return False
try:
return bool(json.loads(output))
except Exception: # pylint: disable=broad-except
return True

8
src/ytdl_sub/utils/scriptable.py
View file

@ -14,10 +14,8 @@ from ytdl_sub.script.utils.exceptions import RuntimeException
from ytdl_sub.utils.exceptions import StringFormattingException
from ytdl_sub.utils.script import ScriptUtils
_BASE_SCRIPT: Script = Script(
ScriptUtils.add_sanitized_variables(
dict(copy.deepcopy(VARIABLE_SCRIPTS), **copy.deepcopy(CUSTOM_FUNCTION_SCRIPTS))
)
BASE_SCRIPT: Script = Script(
dict(ScriptUtils.add_sanitized_variables(VARIABLE_SCRIPTS), **CUSTOM_FUNCTION_SCRIPTS)
)
@ -37,7 +35,7 @@ class Scriptable(ABC):
"""
Initializes with base values
"""
self._script = copy.deepcopy(_BASE_SCRIPT)
self._script = copy.deepcopy(BASE_SCRIPT)
self._unresolvable = copy.deepcopy(UNRESOLVED_VARIABLES)
@property

68
src/ytdl_sub/validators/string_formatter_validators.py
View file

@ -6,8 +6,10 @@ from typing import final
from ytdl_sub.entries.script.variable_definitions import VARIABLES
from ytdl_sub.script.parser import parse
from ytdl_sub.script.script import Script
from ytdl_sub.script.types.syntax_tree import SyntaxTree
from ytdl_sub.script.utils.exceptions import RuntimeException
from ytdl_sub.script.utils.exceptions import ScriptVariableNotResolved
from ytdl_sub.script.utils.exceptions import UserException
from ytdl_sub.script.utils.exceptions import VariableDoesNotExist
from ytdl_sub.utils.exceptions import StringFormattingVariableNotFoundException
from ytdl_sub.validators.validators import DictValidator
from ytdl_sub.validators.validators import ListValidator
@ -90,7 +92,7 @@ class OverridesStringFormatterValidator(StringFormatterValidator):
# pylint: enable=line-too-long
class OverridesIntegerFormatterValidator(StringFormatterValidator):
class OverridesIntegerFormatterValidator(OverridesStringFormatterValidator):
_expected_value_type_name = "integer"
def post_process(self, resolved: str) -> str:
@ -104,6 +106,10 @@ class OverridesIntegerFormatterValidator(StringFormatterValidator):
return resolved
class OverridesBooleanFormatterValidator(OverridesStringFormatterValidator):
_expected_value_type_name = "boolean"
class ListFormatterValidator(ListValidator[StringFormatterValidator]):
_inner_list_type = StringFormatterValidator
@ -142,21 +148,67 @@ class OverridesDictFormatterValidator(DictFormatterValidator):
_key_validator = OverridesStringFormatterValidator
def to_variable_dependency_format_string(script: Script, parsed_format_string: SyntaxTree) -> str:
"""
Create a dummy format string that contains all variable deps as a string.
"""
dummy_format_string = ""
for var in parsed_format_string.variables:
dummy_format_string += f"{{ {var.name} }}"
# pylint: disable=protected-access
for variable_dependency in script._variables[var.name].variables:
dummy_format_string += f"{{ {variable_dependency.name} }}"
# pylint: enable=protected-access
return dummy_format_string
def _validate_formatter(
mock_script: Script,
unresolved_variables: Set[str],
formatter_validator: Union[StringFormatterValidator, OverridesStringFormatterValidator],
) -> None:
try:
unresolvable = unresolved_variables
if isinstance(formatter_validator, OverridesStringFormatterValidator):
unresolvable = unresolved_variables.union({VARIABLES.entry_metadata.variable_name})
is_static_formatter = False
unresolvable = unresolved_variables
if isinstance(formatter_validator, OverridesStringFormatterValidator):
is_static_formatter = True
unresolvable = unresolved_variables.union({VARIABLES.entry_metadata.variable_name})
parsed = parse(
text=formatter_validator.format_string,
)
variable_names = {var.name for var in parsed.variables}
custom_function_names = {f"%{func.name}" for func in parsed.custom_functions}
if not variable_names.issubset(mock_script.variable_names):
raise StringFormattingVariableNotFoundException(
"contains the following variables that do not exist: "
f"{', '.join(sorted(variable_names - mock_script.variable_names))}"
)
if not custom_function_names.issubset(mock_script.function_names):
raise StringFormattingVariableNotFoundException(
"contains the following custom functions that do not exist: "
f"{', '.join(sorted(custom_function_names - mock_script.function_names))}"
)
if unresolved := variable_names.intersection(unresolvable):
raise StringFormattingVariableNotFoundException(
"contains the following variables that are unresolved when executing this "
f"formatter: {', '.join(sorted(unresolved))}"
)
try:
mock_script.resolve_once(
{"tmp_var": formatter_validator.format_string},
{
"tmp_var": to_variable_dependency_format_string(
script=mock_script, parsed_format_string=parsed
)
},
unresolvable=unresolvable,
)
except VariableDoesNotExist as exc:
except RuntimeException as exc:
if isinstance(exc, ScriptVariableNotResolved) and is_static_formatter:
raise StringFormattingVariableNotFoundException(
"static formatters must contain variables that have no dependency to "
"entry variables"
) from exc
raise StringFormattingVariableNotFoundException(exc) from exc

2
tests/conftest.py
View file

@ -113,7 +113,7 @@ def assert_logs(
for call_args in patched_debug.call_args_list:
occurrences += int(expected_message in call_args.args[0])
if expected_occurrences:
if expected_occurrences is not None:
assert (
occurrences == expected_occurrences
), f"{expected_message} was expected {expected_occurrences} times, got {occurrences}"

2
tests/resources/expected_downloads_summaries/youtube/test_channel_full.json
View file

@ -30,7 +30,7 @@
"Project ⧸ Zombie/Season 2011/s2011.e063001 - Project Zombie ｜Fin｜.nfo": "54ea4a48116aa98480a79495036c25e9",
"Project ⧸ Zombie/Season 2011/s2011.e112101 - Skyrim 'Ultra HD w⧸Mods' [PC]-thumb.jpg": "1718599d5189c65f7d8cf6acfa5ea851",
"Project ⧸ Zombie/Season 2011/s2011.e112101 - Skyrim 'Ultra HD w⧸Mods' [PC].info.json": "INFO_JSON",
"Project ⧸ Zombie/Season 2011/s2011.e112101 - Skyrim 'Ultra HD w⧸Mods' [PC].mp4": "2a18cc4baa198edecdc5154082f73eff",
"Project ⧸ Zombie/Season 2011/s2011.e112101 - Skyrim 'Ultra HD w⧸Mods' [PC].mp4": "554d46311112228c22ca9532007803cd",
"Project ⧸ Zombie/Season 2011/s2011.e112101 - Skyrim 'Ultra HD w⧸Mods' [PC].nfo": "04f6aad56d85b1f65b81b6b8000f6479",
"Project ⧸ Zombie/Season 2012/s2012.e012301 - Project Zombie ｜Map Trailer｜-thumb.jpg": "54ebe9df801b278fdd17b21afa8373a6",
"Project ⧸ Zombie/Season 2012/s2012.e012301 - Project Zombie ｜Map Trailer｜.info.json": "INFO_JSON",

28
tests/unit/config/test_preset.py
View file

@ -129,7 +129,7 @@ class TestPreset:
):
with pytest.raises(
StringFormattingVariableNotFoundException,
match="Variable dne_var does not exist.",
match="contains the following variables that do not exist: dne_var",
):
_ = Preset(
config=config_file,
@ -145,7 +145,7 @@ class TestPreset:
):
with pytest.raises(
StringFormattingVariableNotFoundException,
match="Variable dne_var does not exist",
match="contains the following variables that do not exist: dne_var",
):
_ = Preset(
config=config_file,
@ -161,7 +161,7 @@ class TestPreset:
):
with pytest.raises(
StringFormattingVariableNotFoundException,
match="Variable dne_var does not exist",
match="contains the following variables that do not exist: dne_var",
):
_ = Preset(
config=config_file,
@ -182,7 +182,7 @@ class TestPreset:
):
with pytest.raises(
StringFormattingVariableNotFoundException,
match="Variable dne_var does not exist",
match="contains the following variables that do not exist: dne_var",
):
_ = Preset(
config=config_file,
@ -198,6 +198,26 @@ class TestPreset:
},
)
def test_preset_error__dict_override_variable_not_static(
self, config_file, output_options, youtube_video
):
with pytest.raises(
StringFormattingVariableNotFoundException,
match="static formatters must contain variables that "
"have no dependency to entry variables",
):
_ = Preset(
config=config_file,
name="test",
value={
"download": youtube_video,
"output_options": {
"output_directory": "{title}",
"file_name": "{uid}",
},
},
)
def test_preset_with_multi_url__contains_empty_url(self, config_file, output_options):
_ = Preset(
config=config_file,

57
tests/unit/plugins/test_throttle_protection.py
View file

@ -1,3 +1,4 @@
import pytest
from conftest import assert_logs
from ytdl_sub.plugins.throttle_protection import logger as throttle_protection_logger
@ -57,3 +58,59 @@ class TestThrottleProtectionPlugin:
expected_occurrences=1,
):
_ = subscription.download(dry_run=False)
@pytest.mark.parametrize(
"disable_value",
[
"",
False,
"{tp_bool_string}",
"{tp_empty_string}",
],
)
def test_disabled(
self,
config,
subscription_name,
output_directory,
mock_download_collection_entries,
disable_value,
):
preset_dict = {
"preset": [
"Kodi Music Videos",
],
"overrides": {
"url": "https://your.name.here",
"music_video_directory": output_directory,
"tp_bool_string": "{ %bool(False) }",
"tp_empty_string": "",
},
"throttle_protection": {
"enable": disable_value,
"sleep_per_download_s": {
"min": 0.01,
"max": 0.01,
},
"sleep_per_subscription_s": {
"min": 0.02,
"max": 0.02,
},
},
}
subscription = Subscription.from_dict(
config=config,
preset_name=subscription_name,
preset_dict=preset_dict,
)
with mock_download_collection_entries(
is_youtube_channel=False, num_urls=1, is_extracted_audio=False
), assert_logs(
logger=throttle_protection_logger,
expected_message="Sleeping between downloads for %0.2f seconds",
log_level="debug",
expected_occurrences=0,
):
_ = subscription.download(dry_run=False)

20
tests/unit/utils/test_script_utils.py
View file

@ -1,5 +1,6 @@
import copy
import pytest
from unit.script.conftest import single_variable_output
from ytdl_sub.utils.script import ScriptUtils
@ -31,3 +32,22 @@ class TestScriptUtils:
output = single_variable_output(ScriptUtils.to_script(json_dict))
assert output == expected_output
@pytest.mark.parametrize(
"input_str, expected_output",
[
("", False),
("true", True),
("false", False),
("[ ]", False),
("{ }", False),
("True", True),
("False", False),
("lol not False", True),
("0", False),
("-1", True),
("1", True),
],
)
def test_bool_formatter_output(self, input_str: str, expected_output: bool):
assert ScriptUtils.bool_formatter_output(input_str) == expected_output