Merge branch 'master' into j/music-extras

This commit is contained in:
Jesse Bannon 2024-01-18 22:42:50 -08:00
commit 4a07b72293
88 changed files with 1389 additions and 770 deletions

View file

@ -22,6 +22,15 @@ Extracts audio from a video file.
opus, vorbis, wav, and best to grab the best possible format at runtime. opus, vorbis, wav, and best to grab the best possible format at runtime.
``enable``
:expected type: Optional[OverridesFormatter]
:description:
Can typically be left undefined to always default to enable. For preset convenience,
this field can be set using an override variable to easily toggle whether this plugin
is enabled or not via Boolean.
``quality`` ``quality``
:expected type: Float :expected type: Float
@ -76,6 +85,15 @@ chapters and remove specific ones. Can also remove chapters using regex.
Defaults to True. Embed chapters into the file. Defaults to True. Embed chapters into the file.
``enable``
:expected type: Optional[OverridesFormatter]
:description:
Can typically be left undefined to always default to enable. For preset convenience,
this field can be set using an override variable to easily toggle whether this plugin
is enabled or not via Boolean.
``force_key_frames`` ``force_key_frames``
:expected type: Optional[Boolean] :expected type: Optional[Boolean]
@ -148,6 +166,23 @@ granularity possible.
Only download videos before this datetime. Only download videos before this datetime.
``breaks``
:expected type: Optional[OverridesFormatter]
:description:
Toggle to enable breaking subsequent metadata downloads if an entry's upload date
is out of range. Defaults to True.
``enable``
:expected type: Optional[OverridesFormatter]
:description:
Can typically be left undefined to always default to enable. For preset convenience,
this field can be set using an override variable to easily toggle whether this plugin
is enabled or not via Boolean.
---------------------------------------------------------------------------------------------------- ----------------------------------------------------------------------------------------------------
download download
@ -259,6 +294,15 @@ Also supports custom ffmpeg conversions:
with ``ffmpeg_post_process_args``. Defaults to ``yt-dlp``. with ``ffmpeg_post_process_args``. Defaults to ``yt-dlp``.
``enable``
:expected type: Optional[OverridesFormatter]
:description:
Can typically be left undefined to always default to enable. For preset convenience,
this field can be set using an override variable to easily toggle whether this plugin
is enabled or not via Boolean.
``ffmpeg_post_process_args`` ``ffmpeg_post_process_args``
:expected type: Optional[OverridesFormatter] :expected type: Optional[OverridesFormatter]
@ -395,6 +439,15 @@ with a ``.nfo`` extension. You can add any values into the NFO.
episode: "{upload_month}{upload_day_padded}" episode: "{upload_month}{upload_day_padded}"
kodi_safe: False kodi_safe: False
``enable``
:expected type: Optional[OverridesFormatter]
:description:
Can typically be left undefined to always default to enable. For preset convenience,
this field can be set using an override variable to easily toggle whether this plugin
is enabled or not via Boolean.
``kodi_safe`` ``kodi_safe``
:expected type: Optional[Boolean] :expected type: Optional[Boolean]
@ -484,6 +537,15 @@ Usage:
# optional # optional
kodi_safe: False kodi_safe: False
``enable``
:expected type: Optional[OverridesFormatter]
:description:
Can typically be left undefined to always default to enable. For preset convenience,
this field can be set using an override variable to easily toggle whether this plugin
is enabled or not via Boolean.
``kodi_safe`` ``kodi_safe``
:expected type: Optional[Boolean] :expected type: Optional[Boolean]
@ -800,6 +862,15 @@ and using ``title_and_description`` can regex match/exclude from either ``title`
- "{upload_month}" - "{upload_month}"
- "{upload_day}" - "{upload_day}"
``enable``
:expected type: Optional[OverridesFormatter]
:description:
Can typically be left undefined to always default to enable. For preset convenience,
this field can be set using an override variable to easily toggle whether this plugin
is enabled or not via Boolean.
``skip_if_match_fails`` ``skip_if_match_fails``
:expected type: Optional[Boolean] :expected type: Optional[Boolean]
@ -880,6 +951,15 @@ It will set the respective language to the correct subtitle file.
webm files can only embed "vtt" subtitle types. webm files can only embed "vtt" subtitle types.
``enable``
:expected type: Optional[OverridesFormatter]
:description:
Can typically be left undefined to always default to enable. For preset convenience,
this field can be set using an override variable to easily toggle whether this plugin
is enabled or not via Boolean.
``languages`` ``languages``
:expected type: Optional[List[String]] :expected type: Optional[List[String]]
@ -931,6 +1011,15 @@ scripted.
max: 36 max: 36
subscription_download_probability: 1.0 subscription_download_probability: 1.0
``enable``
:expected type: Optional[OverridesFormatter]
:description:
Can typically be left undefined to always default to enable. For preset convenience,
this field can be set using an override variable to easily toggle whether this plugin
is enabled or not via Boolean.
``max_downloads_per_subscription`` ``max_downloads_per_subscription``
:expected type: Optional[Range] :expected type: Optional[Range]
@ -998,9 +1087,6 @@ for more details.
# Stop downloading additional metadata/videos if it # Stop downloading additional metadata/videos if it
# exists in your download archive # exists in your download archive
break_on_existing: True break_on_existing: True
# Stop downloading additional metadata/videos if it
# is out of your date range
break_on_reject: True
# Path to your YouTube cookies file to download 18+ restricted content # Path to your YouTube cookies file to download 18+ restricted content
cookiefile: "/path/to/cookies/file.txt" cookiefile: "/path/to/cookies/file.txt"
# Only download this number of videos/audio # Only download this number of videos/audio

View file

@ -38,10 +38,11 @@ array_apply_fixed
array_at array_at
~~~~~~~~ ~~~~~~~~
:spec: ``array_at(array: Array, idx: Integer) -> AnyArgument`` :spec: ``array_at(array: Array, idx: Integer, default: Optional[AnyArgument]) -> AnyArgument``
:description: :description:
Return the element in the Array at index ``idx``. Return the element in the Array at index ``idx``. If ``idx`` exceeds the array length,
either return ``default`` if provided or throw an error.
array_contains array_contains
~~~~~~~~~~~~~~ ~~~~~~~~~~~~~~
@ -225,6 +226,27 @@ xor
Conditional Functions Conditional Functions
--------------------- ---------------------
elif
~~~~
:spec: ``elif(if_elif_else: AnyArgument, ...) -> AnyArgument``
:description:
Conditional ``if`` statement that is capable of doing else-ifs (``elif``) via
adjacent arguments. It is expected for there to be an odd number of arguments >= 3 to
supply at least one conditional and an else.
:usage:
.. code-block:: python
%elif(
condition1,
return1,
condition2,
return2,
...
else_return
)
if if
~~ ~~
:spec: ``if(condition: Boolean, true: ReturnableArgumentA, false: ReturnableArgumentB) -> Union[ReturnableArgumentA, ReturnableArgumentB]`` :spec: ``if(condition: Boolean, true: ReturnableArgumentA, false: ReturnableArgumentB) -> Union[ReturnableArgumentA, ReturnableArgumentB]``
@ -543,6 +565,13 @@ slice
:description: :description:
Returns the slice of the Array. Returns the slice of the Array.
split
~~~~~
:spec: ``split(string: String, sep: String, max_split: Optional[Integer]) -> Array``
:description:
Splits the input string into multiple strings.
string string
~~~~~~ ~~~~~~
:spec: ``string(value: AnyArgument) -> String`` :spec: ``string(value: AnyArgument) -> String``

View file

@ -1,3 +1,3 @@
sphinx-book-theme==1.0.1 sphinx-book-theme==1.1.0
sphinx-copybutton==0.5.2 sphinx-copybutton==0.5.2
sphinx-design==0.5.0 sphinx-design==0.5.0

View file

@ -25,6 +25,8 @@ General options must be specified before the command (i.e. ``sub``).
path to store the transaction log output of all files added, modified, deleted path to store the transaction log output of all files added, modified, deleted
-st, --suppress-transaction-log -st, --suppress-transaction-log
do not output transaction logs to console or file do not output transaction logs to console or file
-m MATCH [MATCH ...], --match MATCH [MATCH ...]
match subscription names to one or more substrings, and only run those subscriptions
Sub Options Sub Options
----------- -----------
@ -37,6 +39,14 @@ Download all subscriptions specified in each ``SUBPATH``.
``SUBPATH`` is one or more paths to subscription files, uses ``subscriptions.yaml`` if not provided. ``SUBPATH`` is one or more paths to subscription files, uses ``subscriptions.yaml`` if not provided.
It will use the config specified by ``--config``, or ``config.yaml`` if not provided. It will use the config specified by ``--config``, or ``config.yaml`` if not provided.
.. code-block:: text
:caption: Additional Options
-u, --update-with-info-json
update all subscriptions with the current config using info.json files
-o DL_OVERRIDE, --dl-override DL_OVERRIDE
override all subscription config values using `dl` syntax, i.e. --dl-override='--ytdl_options.max_downloads 3'
Download Options Download Options
----------------- -----------------
Download a single subscription in the form of CLI arguments. Download a single subscription in the form of CLI arguments.
@ -67,7 +77,7 @@ Using the command:
--overrides.url: "https://www.youtube.com/channel/UCuAXFkgsw1L7xaCfnd5JJOw" --overrides.url: "https://www.youtube.com/channel/UCuAXFkgsw1L7xaCfnd5JJOw"
See how to shorten commands using See how to shorten commands using
`download aliases <https://ytdl-sub.readthedocs.io/en/latest/config.html#ytdl_sub.config.config_validator.ConfigOptions.dl_aliases>`_. `download aliases <https://ytdl-sub.readthedocs.io/en/latest/config_reference/config_yaml.html#ytdl_sub.config.config_validator.ConfigOptions.dl_aliases>`_.
View Options View Options
----------------- -----------------
@ -76,6 +86,7 @@ View Options
ytdl-sub view [-sc] [URL] ytdl-sub view [-sc] [URL]
.. code-block:: text .. code-block:: text
:caption: Additional Options
-sc, --split-chapters -sc, --split-chapters
View source variables after splitting by chapters View source variables after splitting by chapters

View file

@ -50,8 +50,9 @@ lint =
isort==5.10.1 isort==5.10.1
pylint==2.13.5 pylint==2.13.5
docs = docs =
sphinx==4.5.0 sphinx==7.2.6
sphinx-rtd-theme==1.0.0 sphinx-rtd-theme==2.0.0
sphinx-book-theme==1.1.0
build = build =
build build
twine twine

View file

@ -3,6 +3,7 @@ import os
import sys import sys
from datetime import datetime from datetime import datetime
from pathlib import Path from pathlib import Path
from typing import Dict
from typing import List from typing import List
from typing import Optional from typing import Optional
@ -67,7 +68,12 @@ def _maybe_write_subscription_log_file(
def _download_subscriptions_from_yaml_files( def _download_subscriptions_from_yaml_files(
config: ConfigFile, subscription_paths: List[str], update_with_info_json: bool, dry_run: bool config: ConfigFile,
subscription_paths: List[str],
subscription_matches: List[str],
subscription_override_dict: Dict,
update_with_info_json: bool,
dry_run: bool,
) -> List[Subscription]: ) -> List[Subscription]:
""" """
Downloads all subscriptions from one or many subscription yaml files. Downloads all subscriptions from one or many subscription yaml files.
@ -78,6 +84,8 @@ def _download_subscriptions_from_yaml_files(
Configuration file Configuration file
subscription_paths subscription_paths
Path to subscription files to download Path to subscription files to download
subscription_matches
Optional list of substrings to match subscription names to (only run if matched)
update_with_info_json update_with_info_json
Whether to actually download or update using existing info json Whether to actually download or update using existing info json
dry_run dry_run
@ -96,7 +104,12 @@ def _download_subscriptions_from_yaml_files(
# Load all the subscriptions first to perform all validation before downloading # Load all the subscriptions first to perform all validation before downloading
for path in subscription_paths: for path in subscription_paths:
subscriptions += Subscription.from_file_path(config=config, subscription_path=path) subscriptions += Subscription.from_file_path(
config=config,
subscription_path=path,
subscription_matches=subscription_matches,
subscription_override_dict=subscription_override_dict,
)
for subscription in subscriptions: for subscription in subscriptions:
with subscription.exception_handling(): with subscription.exception_handling():
@ -119,7 +132,7 @@ def _download_subscriptions_from_yaml_files(
exception=subscription.exception, exception=subscription.exception,
) )
Logger.cleanup(cleanup_error_log=False) Logger.cleanup(has_error=False)
gc.collect() # Garbage collect after each subscription download gc.collect() # Garbage collect after each subscription download
return subscriptions return subscriptions
@ -221,10 +234,18 @@ def main() -> List[Subscription]:
"full backup before usage. You have been warned!", "full backup before usage. You have been warned!",
) )
subscription_override_dict = {}
if args.dl_override:
subscription_override_dict = DownloadArgsParser.from_dl_override(
override=args.dl_override, config=config
).to_subscription_dict()
logger.info("Validating subscriptions...") logger.info("Validating subscriptions...")
subscriptions = _download_subscriptions_from_yaml_files( subscriptions = _download_subscriptions_from_yaml_files(
config=config, config=config,
subscription_paths=args.subscription_paths, subscription_paths=args.subscription_paths,
subscription_matches=args.match,
subscription_override_dict=subscription_override_dict,
update_with_info_json=args.update_with_info_json, update_with_info_json=args.update_with_info_json,
dry_run=args.dry_run, dry_run=args.dry_run,
) )

View file

@ -9,6 +9,7 @@ from typing import Tuple
from mergedeep import mergedeep from mergedeep import mergedeep
from ytdl_sub.cli.parsers.main import MainArguments from ytdl_sub.cli.parsers.main import MainArguments
from ytdl_sub.config.config_file import ConfigFile
from ytdl_sub.config.config_validator import ConfigOptions from ytdl_sub.config.config_validator import ConfigOptions
from ytdl_sub.utils.exceptions import InvalidDlArguments from ytdl_sub.utils.exceptions import InvalidDlArguments
@ -247,3 +248,12 @@ class DownloadArgsParser:
""" """
hash_string = str(sorted(self._unknown_arguments)) hash_string = str(sorted(self._unknown_arguments))
return hashlib.sha256(hash_string.encode()).hexdigest()[-8:] return hashlib.sha256(hash_string.encode()).hexdigest()[-8:]
@classmethod
def from_dl_override(cls, override: str, config: ConfigFile) -> "DownloadArgsParser":
"""
Create a DownloadArgsParser from a sub --override argument value
"""
return DownloadArgsParser(
extra_arguments=override.split(), config_options=config.config_options
)

View file

@ -40,6 +40,10 @@ class MainArguments:
long="--suppress-transaction-log", long="--suppress-transaction-log",
is_positional=True, is_positional=True,
) )
MATCH = CLIArgument(
short="-m",
long="--match",
)
@classmethod @classmethod
def all(cls) -> List[CLIArgument]: def all(cls) -> List[CLIArgument]:
@ -54,6 +58,7 @@ class MainArguments:
cls.LOG_LEVEL, cls.LOG_LEVEL,
cls.TRANSACTION_LOG, cls.TRANSACTION_LOG,
cls.SUPPRESS_TRANSACTION_LOG, cls.SUPPRESS_TRANSACTION_LOG,
cls.MATCH,
] ]
@classmethod @classmethod
@ -124,6 +129,16 @@ def _add_shared_arguments(arg_parser: argparse.ArgumentParser, suppress_defaults
help="do not output transaction logs to console or file", help="do not output transaction logs to console or file",
default=argparse.SUPPRESS if suppress_defaults else False, default=argparse.SUPPRESS if suppress_defaults else False,
) )
arg_parser.add_argument(
MainArguments.MATCH.short,
MainArguments.MATCH.long,
dest="match",
nargs="+",
action="extend",
type=str,
help="match subscription names to one or more substrings, and only run those subscriptions",
default=argparse.SUPPRESS if suppress_defaults else [],
)
################################################################################################### ###################################################################################################
@ -142,6 +157,10 @@ class SubArguments:
short="-u", short="-u",
long="--update-with-info-json", long="--update-with-info-json",
) )
OVERRIDE = CLIArgument(
short="-o",
long="--dl-override",
)
subscription_parser = subparsers.add_parser("sub") subscription_parser = subparsers.add_parser("sub")
@ -160,6 +179,13 @@ subscription_parser.add_argument(
help="update all subscriptions with the current config using info.json files", help="update all subscriptions with the current config using info.json files",
default=False, default=False,
) )
subscription_parser.add_argument(
SubArguments.OVERRIDE.short,
SubArguments.OVERRIDE.long,
type=str,
help="override all subscription config values using `dl` syntax, "
"i.e. --dl-override='--ytdl_options.max_downloads 3'",
)
################################################################################################### ###################################################################################################
# DOWNLOAD PARSER # DOWNLOAD PARSER

View file

@ -1,5 +1,6 @@
from abc import ABC from abc import ABC
from abc import abstractmethod from abc import abstractmethod
from functools import cached_property
from typing import Dict from typing import Dict
from typing import Generic from typing import Generic
from typing import List from typing import List
@ -8,9 +9,11 @@ from typing import Tuple
from typing import Type from typing import Type
from ytdl_sub.config.overrides import Overrides from ytdl_sub.config.overrides import Overrides
from ytdl_sub.config.validators.options import ToggleableOptionsDictValidator
from ytdl_sub.config.validators.options import TOptionsValidator from ytdl_sub.config.validators.options import TOptionsValidator
from ytdl_sub.entries.entry import Entry from ytdl_sub.entries.entry import Entry
from ytdl_sub.utils.file_handler import FileMetadata from ytdl_sub.utils.file_handler import FileMetadata
from ytdl_sub.utils.script import ScriptUtils
from ytdl_sub.ytdl_additions.enhanced_download_archive import DownloadArchiver from ytdl_sub.ytdl_additions.enhanced_download_archive import DownloadArchiver
from ytdl_sub.ytdl_additions.enhanced_download_archive import EnhancedDownloadArchive from ytdl_sub.ytdl_additions.enhanced_download_archive import EnhancedDownloadArchive
@ -40,6 +43,17 @@ class Plugin(BasePlugin[TOptionsValidator], Generic[TOptionsValidator], ABC):
Class to define the new plugin functionality Class to define the new plugin functionality
""" """
@cached_property
def is_enabled(self) -> bool:
"""
Returns True if enabled, False if disabled.
"""
if isinstance(self.plugin_options, ToggleableOptionsDictValidator):
return ScriptUtils.bool_formatter_output(
self.overrides.apply_formatter(self.plugin_options.enable)
)
return True
def ytdl_options_match_filters(self) -> Tuple[List[str], List[str]]: def ytdl_options_match_filters(self) -> Tuple[List[str], List[str]]:
""" """
Returns Returns

View file

@ -32,9 +32,6 @@ class YTDLOptions(LiteralDictValidator):
# Stop downloading additional metadata/videos if it # Stop downloading additional metadata/videos if it
# exists in your download archive # exists in your download archive
break_on_existing: True break_on_existing: True
# Stop downloading additional metadata/videos if it
# is out of your date range
break_on_reject: True
# Path to your YouTube cookies file to download 18+ restricted content # Path to your YouTube cookies file to download 18+ restricted content
cookiefile: "/path/to/cookies/file.txt" cookiefile: "/path/to/cookies/file.txt"
# Only download this number of videos/audio # Only download this number of videos/audio

View file

@ -6,6 +6,7 @@ from typing import TypeVar
from ytdl_sub.config.plugin.plugin_operation import PluginOperation from ytdl_sub.config.plugin.plugin_operation import PluginOperation
from ytdl_sub.utils.exceptions import ValidationException from ytdl_sub.utils.exceptions import ValidationException
from ytdl_sub.validators.strict_dict_validator import StrictDictValidator from ytdl_sub.validators.strict_dict_validator import StrictDictValidator
from ytdl_sub.validators.string_formatter_validators import OverridesBooleanFormatterValidator
from ytdl_sub.validators.validators import Validator from ytdl_sub.validators.validators import Validator
# pylint: disable=no-self-use # pylint: disable=no-self-use
@ -57,3 +58,28 @@ TOptionsValidator = TypeVar("TOptionsValidator", bound=OptionsValidator)
class OptionsDictValidator(StrictDictValidator, OptionsValidator, ABC): class OptionsDictValidator(StrictDictValidator, OptionsValidator, ABC):
pass pass
class ToggleableOptionsDictValidator(OptionsDictValidator):
_optional_keys = {"enable"}
def __init__(self, name, value):
assert (
"enable" in self._optional_keys
), f"{self.__class__.__name__} does not have enable as an optional field"
super().__init__(name, value)
self._enable = self._validate_key(
key="enable", validator=OverridesBooleanFormatterValidator, default="True"
)
@property
def enable(self) -> OverridesBooleanFormatterValidator:
"""
:expected type: Optional[OverridesFormatter]
:description:
Can typically be left undefined to always default to enable. For preset convenience,
this field can be set using an override variable to easily toggle whether this plugin
is enabled or not via Boolean.
"""
return self._enable

View file

@ -13,9 +13,10 @@ from ytdl_sub.config.plugin.preset_plugins import PresetPlugins
from ytdl_sub.config.preset_options import OutputOptions from ytdl_sub.config.preset_options import OutputOptions
from ytdl_sub.config.validators.options import OptionsValidator from ytdl_sub.config.validators.options import OptionsValidator
from ytdl_sub.downloaders.url.validators import MultiUrlValidator from ytdl_sub.downloaders.url.validators import MultiUrlValidator
from ytdl_sub.entries.script.variable_definitions import VARIABLE_SCRIPTS
from ytdl_sub.entries.variables.override_variables import SubscriptionVariables from ytdl_sub.entries.variables.override_variables import SubscriptionVariables
from ytdl_sub.script.script import Script from ytdl_sub.script.script import Script
from ytdl_sub.utils.scriptable import BASE_SCRIPT
from ytdl_sub.validators.string_formatter_validators import to_variable_dependency_format_string
from ytdl_sub.validators.string_formatter_validators import validate_formatters from ytdl_sub.validators.string_formatter_validators import validate_formatters
@ -32,10 +33,10 @@ def _add_dummy_overrides(overrides: Overrides) -> Dict[str, str]:
# Have the dummy override variable contain all variable deps that it uses in the string # Have the dummy override variable contain all variable deps that it uses in the string
dummy_overrides: Dict[str, str] = {} dummy_overrides: Dict[str, str] = {}
for override_name in _override_variables(overrides): for override_name in _override_variables(overrides):
dummy_overrides[override_name] = ""
# pylint: disable=protected-access # pylint: disable=protected-access
for variable_dependency in overrides.script._variables[override_name].variables: dummy_overrides[override_name] = to_variable_dependency_format_string(
dummy_overrides[override_name] += f"{{ {variable_dependency.name } }}" script=overrides.script, parsed_format_string=overrides.script._variables[override_name]
)
# pylint: enable=protected-access # pylint: enable=protected-access
return dummy_overrides return dummy_overrides
@ -72,8 +73,15 @@ def _override_variables(overrides: Overrides) -> Set[str]:
} }
def _entry_variables() -> Set[str]: _DUMMY_ENTRY_VARIABLES: Dict[str, str] = {
return set(list(VARIABLE_SCRIPTS.keys())) name: to_variable_dependency_format_string(
# pylint: disable=protected-access
script=BASE_SCRIPT,
parsed_format_string=BASE_SCRIPT._variables[name]
# pylint: enable=protected-access
)
for name in BASE_SCRIPT.variable_names
}
class VariableValidation: class VariableValidation:
@ -97,12 +105,12 @@ class VariableValidation:
""" """
Do some gymnastics to initialize the Overrides script. Do some gymnastics to initialize the Overrides script.
""" """
entry_variables = _entry_variables()
override_variables = _override_variables(overrides) override_variables = _override_variables(overrides)
# Set resolved variables as all entry + override variables # Set resolved variables as all entry + override variables
# at this point to generate every possible added/modified variable # at this point to generate every possible added/modified variable
self.resolved_variables = entry_variables | override_variables self.resolved_variables = set(_DUMMY_ENTRY_VARIABLES.keys()) | override_variables
plugin_variables: Set[str] = set()
for ( for (
plugin_options, plugin_options,
@ -125,6 +133,7 @@ class VariableValidation:
# Set unresolved as variables that are added but do not exist as # Set unresolved as variables that are added but do not exist as
# entry/override variables since they are created at run-time # entry/override variables since they are created at run-time
self.unresolved_variables |= added_variables | modified_variables self.unresolved_variables |= added_variables | modified_variables
plugin_variables |= added_variables | modified_variables
# Then update resolved variables to reflect that # Then update resolved variables to reflect that
self.resolved_variables -= self.unresolved_variables self.resolved_variables -= self.unresolved_variables
@ -137,10 +146,11 @@ class VariableValidation:
) )
# copy the script and mock entry variables # copy the script and mock entry variables
self.script = copy.deepcopy(overrides.script).add(_add_dummy_variables(entry_variables)) self.script = copy.deepcopy(overrides.script)
self.script.add( self.script.add(
variables=_add_dummy_overrides(overrides=overrides), variables=_add_dummy_overrides(overrides=overrides)
unresolvable=self.unresolved_variables, | _add_dummy_variables(variables=plugin_variables)
| _DUMMY_ENTRY_VARIABLES
) )
return self return self
@ -158,7 +168,6 @@ class VariableValidation:
resolved_variables = added_variables | modified_variables resolved_variables = added_variables | modified_variables
self.script.add(_add_dummy_variables(resolved_variables))
self.resolved_variables |= resolved_variables self.resolved_variables |= resolved_variables
self.unresolved_variables -= resolved_variables self.unresolved_variables -= resolved_variables
@ -177,13 +186,10 @@ class VariableValidation:
): ):
self._add_variables(PluginOperation.MODIFY_ENTRY_METADATA, options=plugin_options) self._add_variables(PluginOperation.MODIFY_ENTRY_METADATA, options=plugin_options)
self._update_script()
for plugin_options in PluginMapping.order_options_by( for plugin_options in PluginMapping.order_options_by(
self.plugins.zipped(), PluginOperation.MODIFY_ENTRY self.plugins.zipped(), PluginOperation.MODIFY_ENTRY
): ):
added = self._add_variables(PluginOperation.MODIFY_ENTRY, options=plugin_options) self._add_variables(PluginOperation.MODIFY_ENTRY, options=plugin_options)
if added:
self._update_script()
# Validate that any formatter in the plugin options can resolve # Validate that any formatter in the plugin options can resolve
validate_formatters( validate_formatters(

View file

@ -112,13 +112,6 @@ class UrlDownloaderThumbnailPlugin(SourcePluginExtension):
directory, run this function. This lets the downloader add any extra files directly to the directory, run this function. This lets the downloader add any extra files directly to the
output directory, for things like YT channel image, banner. output directory, for things like YT channel image, banner.
""" """
if playlist_metadata := entry.get(v.playlist_metadata, dict):
self._download_parent_thumbnails(
thumbnail_list_info=collection_url.playlist_thumbnails,
entry=entry,
parent=EntryParent(playlist_metadata, working_directory=self.working_directory),
)
if source_metadata := entry.get(v.source_metadata, dict): if source_metadata := entry.get(v.source_metadata, dict):
self._download_parent_thumbnails( self._download_parent_thumbnails(
thumbnail_list_info=collection_url.source_thumbnails, thumbnail_list_info=collection_url.source_thumbnails,
@ -126,6 +119,13 @@ class UrlDownloaderThumbnailPlugin(SourcePluginExtension):
parent=EntryParent(source_metadata, working_directory=self.working_directory), parent=EntryParent(source_metadata, working_directory=self.working_directory),
) )
if playlist_metadata := entry.get(v.playlist_metadata, dict):
self._download_parent_thumbnails(
thumbnail_list_info=collection_url.playlist_thumbnails,
entry=entry,
parent=EntryParent(playlist_metadata, working_directory=self.working_directory),
)
def modify_entry(self, entry: Entry) -> Optional[Entry]: def modify_entry(self, entry: Entry) -> Optional[Entry]:
""" """
Use the entry to download thumbnails (or move if LATEST_ENTRY). Use the entry to download thumbnails (or move if LATEST_ENTRY).

View file

@ -1,4 +1,3 @@
import copy
from typing import Any from typing import Any
from typing import Dict from typing import Dict
from typing import Optional from typing import Optional
@ -276,17 +275,11 @@ class MultiUrlValidator(OptionsValidator):
def __init__(self, name, value): def __init__(self, name, value):
super().__init__(name, value) super().__init__(name, value)
# Copy since we're popping things
value_copy = copy.deepcopy(value)
if isinstance(value, dict):
# Pop old required field in case it's still there
value_copy.pop("download_strategy", None)
# Deal with old multi-url download strategy # Deal with old multi-url download strategy
if isinstance(value, dict) and "urls" in value_copy: if isinstance(value, dict) and "urls" in value:
self._urls = UrlListValidator(name=name, value=value_copy["urls"]) self._urls = UrlListValidator(name=name, value=value["urls"])
else: else:
self._urls = UrlListValidator(name=name, value=value_copy) self._urls = UrlListValidator(name=name, value=value)
@property @property
def urls(self) -> UrlListValidator: def urls(self) -> UrlListValidator:

View file

@ -102,7 +102,7 @@ class YTDLP:
) )
if is_downloaded and is_thumbnail_downloaded: if is_downloaded and is_thumbnail_downloaded:
return entry_dict return entry_dict or {} # in-case yt-dlp returns None
# Always add check_formats # Always add check_formats
# See https://github.com/yt-dlp/yt-dlp/issues/502 # See https://github.com/yt-dlp/yt-dlp/issues/502
@ -220,11 +220,11 @@ class YTDLP:
with cls._listen_and_log_downloaded_info_json( with cls._listen_and_log_downloaded_info_json(
working_directory=working_directory, log_prefix=log_prefix_on_info_json_dl working_directory=working_directory, log_prefix=log_prefix_on_info_json_dl
): ):
_ = cls.extract_info(ytdl_options_overrides=ytdl_options_overrides, **kwargs) cls.extract_info(ytdl_options_overrides=ytdl_options_overrides, **kwargs)
except RejectedVideoReached: except RejectedVideoReached:
cls.logger.debug( cls.logger.debug(
"RejectedVideoReached, stopping additional downloads " "RejectedVideoReached, stopping additional downloads "
"(Can be disable by setting `ytdl_options.break_on_reject` to False)." "(Can be disable by setting `date_range.breaking` to False)."
) )
except ExistingVideoReached: except ExistingVideoReached:
cls.logger.debug( cls.logger.debug(
@ -234,4 +234,34 @@ class YTDLP:
except MaxDownloadsReached: except MaxDownloadsReached:
cls.logger.info("MaxDownloadsReached, stopping additional downloads.") cls.logger.info("MaxDownloadsReached, stopping additional downloads.")
return cls._get_entry_dicts_from_info_json_files(working_directory=working_directory) parent_dicts: List[Dict] = []
entry_dicts = cls._get_entry_dicts_from_info_json_files(working_directory=working_directory)
entry_ids = {entry_dict.get("id") for entry_dict in entry_dicts}
# Try to get additional uploader (source) metadata that yt-dlp does not fetch
# in a single request
for entry_dict in entry_dicts:
if not (uploader_id := entry_dict.get("uploader_id")):
continue
if uploader_id in entry_ids or not (uploader_url := entry_dict.get("uploader_url")):
continue
cls.logger.debug("Attempting to get parent metadata from URL %s", uploader_url)
try:
parent_dict = cls.extract_info(
ytdl_options_overrides=ytdl_options_overrides | {"playlist_items": "0:0"},
url=uploader_url,
)
except Exception: # pylint: disable=broad-except
# Do not try this uploader_id again
entry_ids.add(uploader_id)
break
if isinstance(parent_dict, dict):
parent_id = parent_dict.get("id")
parent_dicts.append(parent_dict)
entry_ids |= {uploader_id, parent_id}
cls.logger.debug("Adding parent metadata with ids [%s, %s]", uploader_id, parent_id)
return entry_dicts + parent_dicts

View file

@ -157,13 +157,16 @@ class EntryParent(BaseEntry):
def _uid_is_uploader_id(parent: "EntryParent"): def _uid_is_uploader_id(parent: "EntryParent"):
return parent.uid == parent.uploader_id return parent.uid == parent.uploader_id
top_level_parents = [ top_level_parents = [parent for parent in parents if parent.num_children() == 0]
parent for parent in parents if parent.num_children() == 0 and _url_matches(parent)
]
# If more than 1 parent exists, assume the uploader_id is the root parent # If more than 1 parent exists, assume the uploader_id is the root parent
if len(top_level_parents) > 1: if len(top_level_parents) > 1:
top_level_parents = [parent for parent in parents if _uid_is_uploader_id(parent)] top_level_parents = [
parent for parent in top_level_parents if _uid_is_uploader_id(parent)
]
if len(top_level_parents) > 1:
top_level_parents = [parent for parent in top_level_parents if _url_matches(parent)]
match len(top_level_parents): match len(top_level_parents):
case 0: case 0:

View file

@ -27,7 +27,7 @@ def main():
""" """
try: try:
return_code = _main() return_code = _main()
Logger.cleanup(cleanup_error_log=return_code == 0) Logger.cleanup(has_error=return_code != 0)
sys.exit(return_code) sys.exit(return_code)
except Exception as exc: # pylint: disable=broad-except except Exception as exc: # pylint: disable=broad-except
Logger.log_exception(exception=exc) Logger.log_exception(exception=exc)

View file

@ -6,7 +6,7 @@ from typing import Set
from ytdl_sub.config.plugin.plugin import Plugin from ytdl_sub.config.plugin.plugin import Plugin
from ytdl_sub.config.plugin.plugin_operation import PluginOperation from ytdl_sub.config.plugin.plugin_operation import PluginOperation
from ytdl_sub.config.validators.options import OptionsDictValidator from ytdl_sub.config.validators.options import ToggleableOptionsDictValidator
from ytdl_sub.downloaders.ytdl_options_builder import YTDLOptionsBuilder from ytdl_sub.downloaders.ytdl_options_builder import YTDLOptionsBuilder
from ytdl_sub.entries.entry import Entry from ytdl_sub.entries.entry import Entry
from ytdl_sub.entries.script.variable_definitions import VARIABLES from ytdl_sub.entries.script.variable_definitions import VARIABLES
@ -21,7 +21,7 @@ from ytdl_sub.validators.validators import FloatValidator
v: VariableDefinitions = VARIABLES v: VariableDefinitions = VARIABLES
class AudioExtractOptions(OptionsDictValidator): class AudioExtractOptions(ToggleableOptionsDictValidator):
""" """
Extracts audio from a video file. Extracts audio from a video file.
@ -35,7 +35,7 @@ class AudioExtractOptions(OptionsDictValidator):
""" """
_required_keys = {"codec"} _required_keys = {"codec"}
_optional_keys = {"quality"} _optional_keys = {"enable", "quality"}
@classmethod @classmethod
def partial_validate(cls, name: str, value: Any) -> None: def partial_validate(cls, name: str, value: Any) -> None:
@ -102,6 +102,8 @@ class AudioExtractPlugin(Plugin[AudioExtractOptions]):
return ytdl_options_builder.add( return ytdl_options_builder.add(
{ {
"postprocessors": [postprocessor_dict], "postprocessors": [postprocessor_dict],
"format": "bestaudio/best",
"keepvideo": False,
} }
).to_dict() ).to_dict()

View file

@ -7,7 +7,7 @@ from typing import Set
from ytdl_sub.config.plugin.plugin import Plugin from ytdl_sub.config.plugin.plugin import Plugin
from ytdl_sub.config.plugin.plugin_operation import PluginOperation from ytdl_sub.config.plugin.plugin_operation import PluginOperation
from ytdl_sub.config.validators.options import OptionsDictValidator from ytdl_sub.config.validators.options import ToggleableOptionsDictValidator
from ytdl_sub.downloaders.ytdl_options_builder import YTDLOptionsBuilder from ytdl_sub.downloaders.ytdl_options_builder import YTDLOptionsBuilder
from ytdl_sub.entries.entry import Entry from ytdl_sub.entries.entry import Entry
from ytdl_sub.entries.entry import ytdl_sub_chapters_from_comments from ytdl_sub.entries.entry import ytdl_sub_chapters_from_comments
@ -59,7 +59,7 @@ class SponsorBlockCategoryListValidator(ListValidator[SponsorBlockCategoriesVali
_inner_list_type = SponsorBlockCategoriesValidator _inner_list_type = SponsorBlockCategoriesValidator
class ChaptersOptions(OptionsDictValidator): class ChaptersOptions(ToggleableOptionsDictValidator):
""" """
Embeds chapters to video files if they are present. Additional options to add SponsorBlock Embeds chapters to video files if they are present. Additional options to add SponsorBlock
chapters and remove specific ones. Can also remove chapters using regex. chapters and remove specific ones. Can also remove chapters using regex.
@ -90,6 +90,7 @@ class ChaptersOptions(OptionsDictValidator):
""" """
_optional_keys = { _optional_keys = {
"enable",
"embed_chapters", "embed_chapters",
"allow_chapters_from_comments", "allow_chapters_from_comments",
"sponsorblock_categories", "sponsorblock_categories",

View file

@ -3,12 +3,14 @@ from typing import Optional
from typing import Tuple from typing import Tuple
from ytdl_sub.config.plugin.plugin import Plugin from ytdl_sub.config.plugin.plugin import Plugin
from ytdl_sub.config.validators.options import OptionsDictValidator from ytdl_sub.config.validators.options import ToggleableOptionsDictValidator
from ytdl_sub.utils.datetime import to_date_str from ytdl_sub.utils.datetime import to_date_str
from ytdl_sub.utils.script import ScriptUtils
from ytdl_sub.validators.string_datetime import StringDatetimeValidator from ytdl_sub.validators.string_datetime import StringDatetimeValidator
from ytdl_sub.validators.string_formatter_validators import OverridesBooleanFormatterValidator
class DateRangeOptions(OptionsDictValidator): class DateRangeOptions(ToggleableOptionsDictValidator):
""" """
Only download files uploaded within the specified date range. Only download files uploaded within the specified date range.
Dates must adhere to a yt-dlp datetime. From their docs: Dates must adhere to a yt-dlp datetime. From their docs:
@ -31,12 +33,15 @@ class DateRangeOptions(OptionsDictValidator):
after: "today-2weeks" after: "today-2weeks"
""" """
_optional_keys = {"before", "after"} _optional_keys = {"enable", "before", "after", "breaks"}
def __init__(self, name, value): def __init__(self, name, value):
super().__init__(name, value) super().__init__(name, value)
self._before = self._validate_key_if_present("before", StringDatetimeValidator) self._before = self._validate_key_if_present("before", StringDatetimeValidator)
self._after = self._validate_key_if_present("after", StringDatetimeValidator) self._after = self._validate_key_if_present("after", StringDatetimeValidator)
self._breaks = self._validate_key_if_present(
"breaks", OverridesBooleanFormatterValidator, default="True"
)
@property @property
def before(self) -> Optional[StringDatetimeValidator]: def before(self) -> Optional[StringDatetimeValidator]:
@ -56,6 +61,16 @@ class DateRangeOptions(OptionsDictValidator):
""" """
return self._after return self._after
@property
def breaks(self) -> OverridesBooleanFormatterValidator:
"""
:expected type: Optional[OverridesFormatter]
:description:
Toggle to enable breaking subsequent metadata downloads if an entry's upload date
is out of range. Defaults to True.
"""
return self._breaks
class DateRangePlugin(Plugin[DateRangeOptions]): class DateRangePlugin(Plugin[DateRangeOptions]):
plugin_options_type = DateRangeOptions plugin_options_type = DateRangeOptions
@ -79,6 +94,12 @@ class DateRangePlugin(Plugin[DateRangeOptions]):
after_str = to_date_str( after_str = to_date_str(
date_validator=self.plugin_options.after, overrides=self.overrides date_validator=self.plugin_options.after, overrides=self.overrides
) )
breaking_match_filters.append(f"upload_date >= {after_str}") after_filter = f"upload_date >= {after_str}"
if ScriptUtils.bool_formatter_output(
self.overrides.apply_formatter(self.plugin_options.breaks)
):
breaking_match_filters.append(after_filter)
else:
match_filters.append(after_filter)
return match_filters, breaking_match_filters return match_filters, breaking_match_filters

View file

@ -7,7 +7,7 @@ from typing import Set
from ytdl_sub.config.overrides import Overrides from ytdl_sub.config.overrides import Overrides
from ytdl_sub.config.plugin.plugin import Plugin from ytdl_sub.config.plugin.plugin import Plugin
from ytdl_sub.config.plugin.plugin_operation import PluginOperation from ytdl_sub.config.plugin.plugin_operation import PluginOperation
from ytdl_sub.config.validators.options import OptionsDictValidator from ytdl_sub.config.validators.options import ToggleableOptionsDictValidator
from ytdl_sub.entries.entry import Entry from ytdl_sub.entries.entry import Entry
from ytdl_sub.entries.script.variable_definitions import VARIABLES from ytdl_sub.entries.script.variable_definitions import VARIABLES
from ytdl_sub.entries.script.variable_definitions import VariableDefinitions from ytdl_sub.entries.script.variable_definitions import VariableDefinitions
@ -28,7 +28,7 @@ class FileConvertWithValidator(StringSelectValidator):
_select_values = {"yt-dlp", "ffmpeg"} _select_values = {"yt-dlp", "ffmpeg"}
class FileConvertOptions(OptionsDictValidator): class FileConvertOptions(ToggleableOptionsDictValidator):
""" """
Converts video files from one extension to another. Converts video files from one extension to another.
@ -56,7 +56,7 @@ class FileConvertOptions(OptionsDictValidator):
""" """
_required_keys = {"convert_to"} _required_keys = {"convert_to"}
_optional_keys = {"convert_with", "ffmpeg_post_process_args"} _optional_keys = {"enable", "convert_with", "ffmpeg_post_process_args"}
@classmethod @classmethod
def partial_validate(cls, name: str, value: Any) -> None: def partial_validate(cls, name: str, value: Any) -> None:

View file

@ -1,4 +1,3 @@
import json
from typing import Dict from typing import Dict
from typing import Optional from typing import Optional
@ -8,6 +7,7 @@ from ytdl_sub.config.validators.options import OptionsValidator
from ytdl_sub.entries.entry import Entry from ytdl_sub.entries.entry import Entry
from ytdl_sub.utils.exceptions import StringFormattingException from ytdl_sub.utils.exceptions import StringFormattingException
from ytdl_sub.utils.logger import Logger from ytdl_sub.utils.logger import Logger
from ytdl_sub.utils.script import ScriptUtils
from ytdl_sub.validators.string_formatter_validators import ListFormatterValidator from ytdl_sub.validators.string_formatter_validators import ListFormatterValidator
from ytdl_sub.ytdl_additions.enhanced_download_archive import EnhancedDownloadArchive from ytdl_sub.ytdl_additions.enhanced_download_archive import EnhancedDownloadArchive
@ -53,7 +53,9 @@ class FilterExcludePlugin(Plugin[FilterExcludeOptions]):
return entry return entry
for formatter in self.plugin_options.list: for formatter in self.plugin_options.list:
out = json.loads(self.overrides.apply_formatter(formatter=formatter, entry=entry)) out = ScriptUtils.bool_formatter_output(
self.overrides.apply_formatter(formatter=formatter, entry=entry)
)
if bool(out): if bool(out):
logger.info( logger.info(
"Filtering '%s' from the filter %s evaluating to True", "Filtering '%s' from the filter %s evaluating to True",

View file

@ -1,4 +1,3 @@
import json
from typing import Dict from typing import Dict
from typing import Optional from typing import Optional
@ -8,6 +7,7 @@ from ytdl_sub.config.validators.options import OptionsValidator
from ytdl_sub.entries.entry import Entry from ytdl_sub.entries.entry import Entry
from ytdl_sub.utils.exceptions import StringFormattingException from ytdl_sub.utils.exceptions import StringFormattingException
from ytdl_sub.utils.logger import Logger from ytdl_sub.utils.logger import Logger
from ytdl_sub.utils.script import ScriptUtils
from ytdl_sub.validators.string_formatter_validators import ListFormatterValidator from ytdl_sub.validators.string_formatter_validators import ListFormatterValidator
from ytdl_sub.ytdl_additions.enhanced_download_archive import EnhancedDownloadArchive from ytdl_sub.ytdl_additions.enhanced_download_archive import EnhancedDownloadArchive
@ -61,7 +61,9 @@ class FilterIncludePlugin(Plugin[FilterIncludeOptions]):
return entry return entry
for formatter in self.plugin_options.list: for formatter in self.plugin_options.list:
out = json.loads(self.overrides.apply_formatter(formatter=formatter, entry=entry)) out = ScriptUtils.bool_formatter_output(
self.overrides.apply_formatter(formatter=formatter, entry=entry)
)
if not bool(out): if not bool(out):
logger.info( logger.info(
"Filtering '%s' from the filter %s evaluating to False", "Filtering '%s' from the filter %s evaluating to False",

View file

@ -4,7 +4,7 @@ from typing import List
from typing import Tuple from typing import Tuple
from ytdl_sub.config.plugin.plugin import Plugin from ytdl_sub.config.plugin.plugin import Plugin
from ytdl_sub.config.validators.options import OptionsDictValidator from ytdl_sub.config.validators.options import ToggleableOptionsDictValidator
from ytdl_sub.utils.logger import Logger from ytdl_sub.utils.logger import Logger
from ytdl_sub.validators.validators import StringListValidator from ytdl_sub.validators.validators import StringListValidator
@ -44,19 +44,17 @@ def combine_filters(filters: List[str], to_combine: List[str]) -> List[str]:
if not filters: if not filters:
return copy.deepcopy(to_combine) return copy.deepcopy(to_combine)
if len(to_combine) > 1:
raise ValueError("Match-filters to combine only supports 1 at this time")
output_filters: List[str] = [] output_filters: List[str] = []
filter_to_combine: str = to_combine[0]
for match_filter in filters: for match_filter in filters:
output_filters.append(f"{match_filter} & {filter_to_combine}") output_filters.append(match_filter)
for filter_combine in to_combine:
output_filters[-1] += f" & {filter_combine}"
return output_filters return output_filters
class MatchFiltersOptions(OptionsDictValidator): class MatchFiltersOptions(ToggleableOptionsDictValidator):
""" """
Set ``--match-filters`` to pass into yt-dlp to filter entries from being downloaded. Set ``--match-filters`` to pass into yt-dlp to filter entries from being downloaded.
Uses the same syntax as yt-dlp. An entry will be downloaded if any one of the filters are met. Uses the same syntax as yt-dlp. An entry will be downloaded if any one of the filters are met.
@ -74,7 +72,7 @@ class MatchFiltersOptions(OptionsDictValidator):
# - "availability=?public" # - "availability=?public"
""" """
_optional_keys = {"filters"} _optional_keys = {"enable", "filters"}
@classmethod @classmethod
def partial_validate(cls, name: str, value: Any) -> None: def partial_validate(cls, name: str, value: Any) -> None:

View file

@ -1,6 +1,4 @@
import copy
from collections import defaultdict from collections import defaultdict
from typing import Any
from typing import Dict from typing import Dict
from typing import List from typing import List
@ -14,10 +12,8 @@ from ytdl_sub.entries.script.variable_definitions import VariableDefinitions
from ytdl_sub.utils.file_handler import FileMetadata from ytdl_sub.utils.file_handler import FileMetadata
from ytdl_sub.utils.logger import Logger from ytdl_sub.utils.logger import Logger
from ytdl_sub.validators.audo_codec_validator import AUDIO_CODEC_EXTS from ytdl_sub.validators.audo_codec_validator import AUDIO_CODEC_EXTS
from ytdl_sub.validators.strict_dict_validator import StrictDictValidator
from ytdl_sub.validators.string_formatter_validators import ListFormatterValidator from ytdl_sub.validators.string_formatter_validators import ListFormatterValidator
from ytdl_sub.validators.string_formatter_validators import StringFormatterValidator from ytdl_sub.validators.string_formatter_validators import StringFormatterValidator
from ytdl_sub.validators.validators import BoolValidator
v: VariableDefinitions = VARIABLES v: VariableDefinitions = VARIABLES
@ -40,31 +36,6 @@ def _is_multi_field(tag_name: str) -> bool:
} }
class MusicTagsValidator(StrictDictValidator):
"""
Validator for the music_tag's `tags` field. Treat each value as a list.
Can still specify it like a single value but under-the-hood it's a list of a single element.
"""
_optional_keys = set(list(mediafile.MediaFile.sorted_fields()))
def __init__(self, name, value):
super().__init__(name, value)
self._tags: Dict[str, List[StringFormatterValidator]] = {}
for key in self._keys:
self._tags[key] = self._validate_key(key=key, validator=ListFormatterValidator).list
@property
def as_lists(self) -> Dict[str, List[StringFormatterValidator]]:
"""
Returns
-------
Tag formatter(s) as a list
"""
return self._tags
class MusicTagsOptions(OptionsDictValidator): class MusicTagsOptions(OptionsDictValidator):
""" """
Adds tags to every download audio file using Adds tags to every download audio file using
@ -93,40 +64,24 @@ class MusicTagsOptions(OptionsDictValidator):
- "ytdl-sub" - "ytdl-sub"
""" """
_optional_keys = {"tags", "embed_thumbnail"} _optional_keys = set(list(mediafile.MediaFile.sorted_fields()))
_allow_extra_keys = True
def __init__(self, name, value): def __init__(self, name, value):
super().__init__(name, value) super().__init__(name, value)
self._embed_thumbnail = self._validate_key_if_present( self._tags: Dict[str, List[StringFormatterValidator]] = {}
key="embed_thumbnail", validator=BoolValidator for key in self._keys:
) self._tags[key] = self._validate_key(key=key, validator=ListFormatterValidator).list
new_tags_dict: Dict[str, Any] = copy.deepcopy(value)
old_tags_dict = new_tags_dict.pop("tags", {})
new_tags_dict.pop("embed_thumbnail", None)
self._is_old_format = len(old_tags_dict) > 0 or self._embed_thumbnail is not None
self._tags = MusicTagsValidator(name=name, value=dict(old_tags_dict, **new_tags_dict))
@property @property
def tags(self) -> MusicTagsValidator: def as_lists(self) -> Dict[str, List[StringFormatterValidator]]:
""" """
Key, values of tag names, tag values. Supports source and override variables. Returns
Supports lists which will get written to MP3s as id3v2.4 multi-tags. -------
Tag formatter(s) as a list
""" """
return self._tags return self._tags
@property
def embed_thumbnail(self) -> bool:
"""
Optional. Whether to embed the thumbnail into the audio file.
"""
if self._embed_thumbnail is None:
return False
return self._embed_thumbnail.value
class MusicTagsPlugin(Plugin[MusicTagsOptions]): class MusicTagsPlugin(Plugin[MusicTagsOptions]):
plugin_options_type = MusicTagsOptions plugin_options_type = MusicTagsOptions
@ -142,23 +97,9 @@ class MusicTagsPlugin(Plugin[MusicTagsOptions]):
f"to audio using the audio_extract plugin." f"to audio using the audio_extract plugin."
) )
# pylint: disable=protected-access
if self.plugin_options._is_old_format:
logger.warning(
"music_tags.tags is now deprecated. Place your tags directly under music_tags "
"instead. The old format will be removed in October of 2023. See "
"https://ytdl-sub.readthedocs.io/en/latest/deprecation_notices.html#music-tags "
"for more details."
)
if self.plugin_options.embed_thumbnail:
logger.warning(
"music_tags.embed_thumbnail is also deprecated. Use the dedicated "
"embed_thumbnail plugin instead. This will be removed in October of 2023."
)
# Resolve the tags into this dict # Resolve the tags into this dict
tags_to_write: Dict[str, List[str]] = defaultdict(list) tags_to_write: Dict[str, List[str]] = defaultdict(list)
for tag_name, tag_formatters in self.plugin_options.tags.as_lists.items(): for tag_name, tag_formatters in self.plugin_options.as_lists.items():
for tag_formatter in tag_formatters: for tag_formatter in tag_formatters:
tag_value = self.overrides.apply_formatter(formatter=tag_formatter, entry=entry) tag_value = self.overrides.apply_formatter(formatter=tag_formatter, entry=entry)
tags_to_write[tag_name].append(tag_value) tags_to_write[tag_name].append(tag_value)
@ -180,19 +121,10 @@ class MusicTagsPlugin(Plugin[MusicTagsOptions]):
) )
setattr(audio_file, tag_name, tag_value[0]) setattr(audio_file, tag_name, tag_value[0])
if self.plugin_options.embed_thumbnail and entry.is_thumbnail_downloaded():
with open(entry.get_download_thumbnail_path(), "rb") as thumb:
mediafile_img = mediafile.Image(
data=thumb.read(), desc="cover", type=mediafile.ImageType.front
)
audio_file.images = [mediafile_img]
audio_file.save() audio_file.save()
# report the tags written # report the tags written
title = f"{'Embedded Thumbnail, ' if self.plugin_options.embed_thumbnail else ''}Music Tags"
return FileMetadata.from_dict( return FileMetadata.from_dict(
title="Music Tags",
value_dict=tags_to_write, value_dict=tags_to_write,
title=title,
) )

View file

@ -8,7 +8,7 @@ from typing import List
from typing import Optional from typing import Optional
from ytdl_sub.config.plugin.plugin import Plugin from ytdl_sub.config.plugin.plugin import Plugin
from ytdl_sub.config.validators.options import OptionsDictValidator from ytdl_sub.config.validators.options import ToggleableOptionsDictValidator
from ytdl_sub.entries.entry import Entry from ytdl_sub.entries.entry import Entry
from ytdl_sub.utils.file_handler import FileHandler from ytdl_sub.utils.file_handler import FileHandler
from ytdl_sub.utils.file_handler import FileMetadata from ytdl_sub.utils.file_handler import FileMetadata
@ -23,13 +23,13 @@ from ytdl_sub.validators.string_formatter_validators import StringFormatterValid
from ytdl_sub.validators.validators import BoolValidator from ytdl_sub.validators.validators import BoolValidator
class SharedNfoTagsOptions(OptionsDictValidator): class SharedNfoTagsOptions(ToggleableOptionsDictValidator):
""" """
Shared code between NFO tags and Ouptut Directory NFO Tags Shared code between NFO tags and Ouptut Directory NFO Tags
""" """
_required_keys = {"nfo_name", "nfo_root", "tags"} _required_keys = {"nfo_name", "nfo_root", "tags"}
_optional_keys = {"kodi_safe"} _optional_keys = {"enable", "kodi_safe"}
@classmethod @classmethod
def partial_validate(cls, name: str, value: Any) -> None: def partial_validate(cls, name: str, value: Any) -> None:

View file

@ -32,7 +32,7 @@ class OutputDirectoryNfoTagsOptions(SharedNfoTagsOptions):
# Hack to make it so collection named seasons do not error # Hack to make it so collection named seasons do not error
# when adding output_directory_nfo info for plex # when adding output_directory_nfo info for plex
_required_keys = set() _required_keys = set()
_optional_keys = {"kodi_safe", "nfo_name", "nfo_root", "tags"} _optional_keys = {"enable", "kodi_safe", "nfo_name", "nfo_root", "tags"}
@property @property
def nfo_root(self) -> StringFormatterValidator: def nfo_root(self) -> StringFormatterValidator:

View file

@ -8,7 +8,7 @@ from typing import Set
from ytdl_sub.config.overrides import Overrides from ytdl_sub.config.overrides import Overrides
from ytdl_sub.config.plugin.plugin import Plugin from ytdl_sub.config.plugin.plugin import Plugin
from ytdl_sub.config.plugin.plugin_operation import PluginOperation from ytdl_sub.config.plugin.plugin_operation import PluginOperation
from ytdl_sub.config.validators.options import OptionsDictValidator from ytdl_sub.config.validators.options import ToggleableOptionsDictValidator
from ytdl_sub.entries.entry import Entry from ytdl_sub.entries.entry import Entry
from ytdl_sub.script.parser import parse from ytdl_sub.script.parser import parse
from ytdl_sub.script.utils.exceptions import RuntimeException from ytdl_sub.script.utils.exceptions import RuntimeException
@ -120,7 +120,7 @@ class FromSourceVariablesRegex(DictValidator):
} }
class RegexOptions(OptionsDictValidator): class RegexOptions(ToggleableOptionsDictValidator):
r""" r"""
.. attention:: .. attention::
@ -218,7 +218,7 @@ class RegexOptions(OptionsDictValidator):
""" """
_required_keys = {"from"} _required_keys = {"from"}
_optional_keys = {"skip_if_match_fails"} _optional_keys = {"enable", "skip_if_match_fails"}
@classmethod @classmethod
def partial_validate(cls, name: str, value: Any) -> None: def partial_validate(cls, name: str, value: Any) -> None:

View file

@ -6,7 +6,7 @@ from typing import Set
from ytdl_sub.config.plugin.plugin import Plugin from ytdl_sub.config.plugin.plugin import Plugin
from ytdl_sub.config.plugin.plugin_operation import PluginOperation from ytdl_sub.config.plugin.plugin_operation import PluginOperation
from ytdl_sub.config.validators.options import OptionsDictValidator from ytdl_sub.config.validators.options import ToggleableOptionsDictValidator
from ytdl_sub.downloaders.ytdl_options_builder import YTDLOptionsBuilder from ytdl_sub.downloaders.ytdl_options_builder import YTDLOptionsBuilder
from ytdl_sub.entries.entry import Entry from ytdl_sub.entries.entry import Entry
from ytdl_sub.entries.script.variable_definitions import VARIABLES from ytdl_sub.entries.script.variable_definitions import VARIABLES
@ -31,7 +31,7 @@ class SubtitlesTypeValidator(StringSelectValidator):
_select_values = SUBTITLE_EXTENSIONS _select_values = SUBTITLE_EXTENSIONS
class SubtitleOptions(OptionsDictValidator): class SubtitleOptions(ToggleableOptionsDictValidator):
""" """
Defines how to download and store subtitles. Using this plugin creates two new variables: Defines how to download and store subtitles. Using this plugin creates two new variables:
``lang`` and ``subtitles_ext``. ``lang`` is dynamic since you can download multiple subtitles. ``lang`` and ``subtitles_ext``. ``lang`` is dynamic since you can download multiple subtitles.
@ -52,6 +52,7 @@ class SubtitleOptions(OptionsDictValidator):
""" """
_optional_keys = { _optional_keys = {
"enable",
"subtitles_name", "subtitles_name",
"subtitles_type", "subtitles_type",
"embed_subtitles", "embed_subtitles",

View file

@ -6,7 +6,7 @@ from typing import Tuple
from ytdl_sub.config.overrides import Overrides from ytdl_sub.config.overrides import Overrides
from ytdl_sub.config.plugin.plugin import Plugin from ytdl_sub.config.plugin.plugin import Plugin
from ytdl_sub.config.validators.options import OptionsDictValidator from ytdl_sub.config.validators.options import ToggleableOptionsDictValidator
from ytdl_sub.entries.entry import Entry from ytdl_sub.entries.entry import Entry
from ytdl_sub.utils.file_handler import FileMetadata from ytdl_sub.utils.file_handler import FileMetadata
from ytdl_sub.utils.logger import Logger from ytdl_sub.utils.logger import Logger
@ -59,7 +59,7 @@ class RandomizedRangeValidator(StrictDictValidator):
return int(self.randomized_float()) return int(self.randomized_float())
class ThrottleProtectionOptions(OptionsDictValidator): class ThrottleProtectionOptions(ToggleableOptionsDictValidator):
""" """
Provides options to make ytdl-sub look more 'human-like' to protect from throttling. For Provides options to make ytdl-sub look more 'human-like' to protect from throttling. For
range-based values, a random number will be chosen within the range to avoid sleeps looking range-based values, a random number will be chosen within the range to avoid sleeps looking
@ -85,6 +85,7 @@ class ThrottleProtectionOptions(OptionsDictValidator):
""" """
_optional_keys = { _optional_keys = {
"enable",
"sleep_per_download_s", "sleep_per_download_s",
"sleep_per_subscription_s", "sleep_per_subscription_s",
"max_downloads_per_subscription", "max_downloads_per_subscription",

View file

@ -1,9 +1,7 @@
import copy
from typing import Any
from typing import Dict from typing import Dict
from ytdl_sub.config.plugin.plugin import Plugin from ytdl_sub.config.plugin.plugin import Plugin
from ytdl_sub.config.validators.options import OptionsDictValidator from ytdl_sub.config.validators.options import OptionsValidator
from ytdl_sub.entries.entry import Entry from ytdl_sub.entries.entry import Entry
from ytdl_sub.utils.ffmpeg import add_ffmpeg_metadata_key_values from ytdl_sub.utils.ffmpeg import add_ffmpeg_metadata_key_values
from ytdl_sub.utils.file_handler import FileMetadata from ytdl_sub.utils.file_handler import FileMetadata
@ -13,7 +11,7 @@ from ytdl_sub.validators.string_formatter_validators import DictFormatterValidat
logger = Logger.get("video-tags") logger = Logger.get("video-tags")
class VideoTagsOptions(OptionsDictValidator): class VideoTagsOptions(DictFormatterValidator, OptionsValidator):
""" """
Adds tags to every downloaded video file using ffmpeg ``-metadata key=value`` args. Adds tags to every downloaded video file using ffmpeg ``-metadata key=value`` args.
@ -27,34 +25,6 @@ class VideoTagsOptions(OptionsDictValidator):
description: "{description}" description: "{description}"
""" """
_optional_keys = {"tags"}
_allow_extra_keys = True
@classmethod
def partial_validate(cls, name: str, value: Any) -> None:
"""
Partially validate video tags
"""
if isinstance(value, dict):
value["tags"] = value.get("tags", {})
_ = cls(name, value)
def __init__(self, name, value):
super().__init__(name, value)
new_tags_dict: Dict[str, Any] = copy.deepcopy(value)
old_tags_dict = new_tags_dict.pop("tags", {})
self._is_old_format = len(old_tags_dict) > 0
self._tags = DictFormatterValidator(name=name, value=dict(old_tags_dict, **new_tags_dict))
@property
def tags(self) -> DictFormatterValidator:
"""
Key/values of tag names/values. Supports source and override variables.
"""
return self._tags
class VideoTagsPlugin(Plugin[VideoTagsOptions]): class VideoTagsPlugin(Plugin[VideoTagsOptions]):
plugin_options_type = VideoTagsOptions plugin_options_type = VideoTagsOptions
@ -63,17 +33,8 @@ class VideoTagsPlugin(Plugin[VideoTagsOptions]):
""" """
Tags the entry's audio file using values defined in the metadata options Tags the entry's audio file using values defined in the metadata options
""" """
# pylint: disable=protected-access
if self.plugin_options._is_old_format:
logger.warning(
"video_tags.tags is now deprecated. Place your tags directly under video_tags "
"instead. The old format will be removed in October of 2023. See "
"https://ytdl-sub.readthedocs.io/en/latest/deprecation_notices.html#video-tags "
"for more details."
)
tags_to_write: Dict[str, str] = {} tags_to_write: Dict[str, str] = {}
for tag_name, tag_formatter in self.plugin_options.tags.dict.items(): for tag_name, tag_formatter in self.plugin_options.dict.items():
tag_value = self.overrides.apply_formatter(formatter=tag_formatter, entry=entry) tag_value = self.overrides.apply_formatter(formatter=tag_formatter, entry=entry)
tags_to_write[tag_name] = tag_value tags_to_write[tag_name] = tag_value

View file

@ -1,36 +1,36 @@
presets: presets:
best_video_quality:
format: "bestvideo+bestaudio/best"
ytdl_options:
merge_output_format: "mp4"
max_1080p:
format: "(bv*[height<=1080]+bestaudio/best[height<=1080])"
ytdl_options:
merge_output_format: "mp4"
chunk_initial_download:
ytdl_options:
max_downloads: 20
playlistreverse: True
break_on_existing: False
break_on_reject: True
"Only Recent": "Only Recent":
preset:
- "Only Recent Archive"
# Only fetch videos after today minus date_range # Only fetch videos after today minus date_range
date_range:
after: "today-{only_recent_date_range}"
# Only keep files uploaded after date_range # Only keep files uploaded after date_range
output_options: output_options:
keep_files_after: "today-{only_recent_date_range}" keep_files_after: "today-{only_recent_date_range}"
keep_max_files: "{only_recent_max_files}" keep_max_files: "{only_recent_max_files}"
overrides:
only_recent_max_files: 0
"Only Recent Archive":
# Only fetch videos after today minus date_range
date_range:
after: "today-{only_recent_date_range}"
# Set the default date_range to 2 months # Set the default date_range to 2 months
overrides: overrides:
date_range: "2months" # keep for legacy-reasons date_range: "2months" # keep for legacy-reasons
only_recent_date_range: "{date_range}" only_recent_date_range: "{date_range}"
only_recent_max_files: 0
chunk_initial_download:
ytdl_options:
max_downloads: 20
playlistreverse: True
break_on_existing: False
"Download in Chunks":
preset:
- chunk_initial_download

View file

@ -0,0 +1,40 @@
presets:
best_video_quality:
format: "bestvideo+bestaudio/best"
ytdl_options:
merge_output_format: "mp4"
"Best Video Quality":
preset:
- best_video_quality
"Max 2160p":
format: "(bv*[height<=2160]+bestaudio/best[height<=2160])"
ytdl_options:
merge_output_format: "mp4"
"Max 1440p":
format: "(bv*[height<=1440]+bestaudio/best[height<=1440])"
ytdl_options:
merge_output_format: "mp4"
max_1080p:
format: "(bv*[height<=1080]+bestaudio/best[height<=1080])"
ytdl_options:
merge_output_format: "mp4"
"Max 1080p":
preset:
- max_1080p
"Max 720p":
format: "(bv*[height<=720]+bestaudio/best[height<=720])"
ytdl_options:
merge_output_format: "mp4"
"Max 480p":
format: "(bv*[height<=480]+bestaudio/best[height<=480])"
ytdl_options:
merge_output_format: "mp4"

View file

@ -20,6 +20,11 @@ presets:
uid: "avatar_uncropped" uid: "avatar_uncropped"
- name: "{banner_uncropped_thumbnail_file_name}" - name: "{banner_uncropped_thumbnail_file_name}"
uid: "banner_uncropped" uid: "banner_uncropped"
source_thumbnails:
- name: "{avatar_uncropped_thumbnail_file_name}"
uid: "avatar_uncropped"
- name: "{banner_uncropped_thumbnail_file_name}"
uid: "banner_uncropped"
- url: "{url2}" - url: "{url2}"
- url: "{url3}" - url: "{url3}"
- url: "{url4}" - url: "{url4}"

View file

@ -0,0 +1,21 @@
presets:
# assumes that each chapter in an entry is a song
_albums_from_chapters:
preset: "Single"
chapters:
embed_chapters: True
split_by_chapters:
when_no_chapters: "pass"
overrides:
track_title: "{chapter_title}" # Chapter title is the track title
track_album: "{title}" # Video's title is the album title
track_number: "{chapter_index}"
track_number_padded: "{chapter_index_padded}"
track_total: "{chapter_count}"
"YouTube Full Albums":
preset:
- "_albums_from_chapters"

View file

@ -0,0 +1,36 @@
presets:
# assumes that each entry is a song in one album
_albums_from_playlists:
preset:
- "_music_base"
download:
- url: "{url}"
include_sibling_metadata: True
overrides:
track_album: "{playlist_title}"
track_number: "{playlist_index}"
track_number_padded: "{playlist_index_padded}"
track_total: "{playlist_count}"
track_year: "{playlist_max_upload_year}"
"YouTube Releases":
preset:
- "_albums_from_playlists"
"Bandcamp":
preset:
- "_albums_from_playlists"
regex:
from:
title:
match:
- ".*? - (.*)" # Captures 'Some - Song' from 'Emily Hopkins - Some - Song'
capture_group_names:
- "captured_track_title"
capture_group_defaults:
- "{title}"
overrides:
track_title: "{captured_track_title}"

View file

@ -1,163 +0,0 @@
presets:
_music_base:
output_options:
output_directory: "{music_directory}"
file_name: "{track_full_path}"
thumbnail_name: "{album_cover_path}"
maintain_download_archive: True
ytdl_options:
break_on_existing: True
format: "ba[ext=webm]/ba"
audio_extract:
codec: "best"
music_tags:
artist: "{track_artist}"
albumartist: "{track_album_artist}"
title: "{track_title}"
album: "{track_album}"
track: "{track_number}"
tracktotal: "{track_total}"
year: "{track_year}"
# multi-tags
artists:
- "{track_artist}"
albumartists:
- "{track_album_artist}"
genres:
- "{track_genre}"
overrides:
# MUST DEFINE:
# music_directory
# Defaults
track_genre_default: "Unset"
# Subscription overrides
subscription_indent_1: "{track_genre_default}"
subscription_value: ""
url: "{subscription_value}"
# Track Overrides
track_title: "{title}"
track_album: "{title}"
track_artist: "{subscription_name}"
track_album_artist: "{track_artist}"
track_number: "1"
track_number_padded: "01"
track_total: "1"
track_year: "{upload_year}"
track_genre: "{subscription_indent_1}"
# Directory Overrides
artist_dir: "{track_artist_sanitized}"
album_dir: "[{track_year}] {track_album_sanitized}"
track_file_name: "{track_number_padded} - {track_title_sanitized}.{ext}"
track_full_path: "{artist_dir}/{album_dir}/{track_file_name}"
album_cover_path: "{artist_dir}/{album_dir}/folder.{thumbnail_ext}"
"Single":
preset:
- "_music_base"
download:
- url: "{url}"
include_sibling_metadata: False
_albums_from_playlists:
preset:
- "_music_base"
download:
- url: "{url}"
include_sibling_metadata: True
overrides:
track_album: "{playlist_title}"
track_number: "{playlist_index}"
track_number_padded: "{playlist_index_padded}"
track_total: "{playlist_count}"
track_year: "{playlist_max_upload_year}"
_albums_from_chapters:
preset: "Single"
chapters:
embed_chapters: True
split_by_chapters:
when_no_chapters: "pass"
overrides:
track_title: "{chapter_title}" # Chapter title is the track title
track_album: "{title}" # Video's title is the album title
track_number: "{chapter_index}"
track_number_padded: "{chapter_index_padded}"
track_total: "{chapter_count}"
"SoundCloud Discography":
preset: "_music_base"
# Download using the multi_url strategy
download:
download_strategy: "multi_url"
urls:
# The first URL will be all the artist's tracks.
# Treat these as singles - an album with a single track
- url: "{url}/tracks"
include_sibling_metadata: False
variables:
sc_track_album: "{title}"
sc_track_number: "1"
sc_track_number_padded: "01"
sc_track_total: "1"
sc_track_year: "{upload_year}"
# Set the second URL to the artist's albums. If a track belongs to both
# to an album and tracks (in the URL above), it will resolve to this
# URL and include the album metadata we set below.
- url: "{url}/albums"
include_sibling_metadata: True
variables:
sc_track_album: "{playlist_title}"
sc_track_number: "{playlist_index}"
sc_track_number_padded: "{playlist_index_padded}"
sc_track_total: "{playlist_count}"
sc_track_year: "{playlist_max_upload_year}"
# Override various track properties using playlist variables.
overrides:
track_album: "{sc_track_album}"
track_number: "{sc_track_number}"
track_number_padded: "{sc_track_number_padded}"
track_total: "{sc_track_total}"
track_year: "{sc_track_year}"
"YouTube Releases":
preset:
- "_albums_from_playlists"
"YouTube Full Albums":
preset:
- "_albums_from_chapters"
"Bandcamp":
preset:
- "_albums_from_playlists"
regex:
from:
title:
match:
- ".*? - (.*)" # Captures 'Some - Song' from 'Emily Hopkins - Some - Song'
capture_group_names:
- "captured_track_title"
capture_group_defaults:
- "{title}"
overrides:
track_title: "{captured_track_title}"

View file

@ -0,0 +1,38 @@
presets:
"SoundCloud Discography":
preset: "_music_base"
# Download using the multi_url strategy
download:
download_strategy: "multi_url"
urls:
# The first URL will be all the artist's tracks.
# Treat these as singles - an album with a single track
- url: "{url}/tracks"
include_sibling_metadata: False
variables:
sc_track_album: "{title}"
sc_track_number: "1"
sc_track_number_padded: "01"
sc_track_total: "1"
sc_track_year: "{upload_year}"
# Set the second URL to the artist's albums. If a track belongs to both
# to an album and tracks (in the URL above), it will resolve to this
# URL and include the album metadata we set below.
- url: "{url}/albums"
include_sibling_metadata: True
variables:
sc_track_album: "{playlist_title}"
sc_track_number: "{playlist_index}"
sc_track_number_padded: "{playlist_index_padded}"
sc_track_total: "{playlist_count}"
sc_track_year: "{playlist_max_upload_year}"
# Override various track properties using playlist variables.
overrides:
track_album: "{sc_track_album}"
track_number: "{sc_track_number}"
track_number_padded: "{sc_track_number_padded}"
track_total: "{sc_track_total}"
track_year: "{sc_track_year}"

View file

@ -0,0 +1,72 @@
presets:
_music_base:
output_options:
output_directory: "{music_directory}"
file_name: "{track_full_path}"
thumbnail_name: "{album_cover_path}"
maintain_download_archive: True
ytdl_options:
break_on_existing: True
format: "ba[ext=webm]/ba"
audio_extract:
codec: "best"
music_tags:
artist: "{track_artist}"
albumartist: "{track_album_artist}"
title: "{track_title}"
album: "{track_album}"
track: "{track_number}"
tracktotal: "{track_total}"
year: "{track_year}"
# multi-tags
artists:
- "{track_artist}"
albumartists:
- "{track_album_artist}"
genres:
- "{track_genre}"
overrides:
# MUST DEFINE:
# music_directory
# Defaults
track_genre_default: "Unset"
# Subscription overrides
subscription_indent_1: "{track_genre_default}"
subscription_value: ""
url: "{subscription_value}"
# Track Overrides
track_title: "{title}"
track_album: "{title}"
track_artist: "{subscription_name}"
track_album_artist: "{track_artist}"
track_number: "1"
track_number_padded: "01"
track_total: "1"
track_year: "{upload_year}"
track_genre: "{subscription_indent_1}"
# Directory Overrides
artist_dir: "{track_artist_sanitized}"
album_dir: "[{track_year}] {track_album_sanitized}"
track_file_name: "{track_number_padded} - {track_title_sanitized}.{ext}"
track_full_path: "{artist_dir}/{album_dir}/{track_file_name}"
album_cover_path: "{artist_dir}/{album_dir}/folder.{thumbnail_ext}"
"Single":
preset:
- "_music_base"
download:
- url: "{url}"
include_sibling_metadata: False

View file

@ -0,0 +1,33 @@
presets:
_music_video_base:
output_options:
output_directory: "{music_video_directory}"
file_name: "{music_video_file_name}.{ext}"
thumbnail_name: "{music_video_file_name}.jpg"
maintain_download_archive: True
ytdl_options:
break_on_existing: True
overrides:
# MUST DEFINE:
# music_video_directory
# Defaults
music_video_genre_default: "ytdl-sub"
music_video_album_default: "Music Videos"
# Subscription overrides
subscription_indent_1: "{music_video_genre_default}"
# Music Video Overrides
music_video_title: "{title}"
music_video_album: "{music_video_album_default}"
music_video_artist: "{subscription_name}"
music_video_year: "{upload_year}"
music_video_date: "{upload_date_standardized}"
music_video_genre: "{subscription_indent_1}"
# Directory Overrides
music_video_file_name_suffix: ""
music_video_file_name: "{music_video_artist_sanitized}/{music_video_title_sanitized}{music_video_file_name_suffix}"

View file

@ -1,58 +1,4 @@
presets: presets:
_music_video_base:
output_options:
output_directory: "{music_video_directory}"
file_name: "{music_video_file_name}.{ext}"
thumbnail_name: "{music_video_file_name}.jpg"
maintain_download_archive: True
ytdl_options:
break_on_existing: True
overrides:
# MUST DEFINE:
# music_video_directory
# Defaults
music_video_genre_default: "ytdl-sub"
music_video_album_default: "Music Videos"
# Subscription overrides
subscription_indent_1: "{music_video_genre_default}"
# Music Video Overrides
music_video_title: "{title}"
music_video_album: "{music_video_album_default}"
music_video_artist: "{subscription_name}"
music_video_year: "{upload_year}"
music_video_date: "{upload_date_standardized}"
music_video_genre: "{subscription_indent_1}"
# Directory Overrides
music_video_file_name_suffix: ""
music_video_file_name: "{music_video_artist_sanitized}/{music_video_title_sanitized}{music_video_file_name_suffix}"
_music_video_nfo:
nfo_tags:
nfo_name: "{music_video_file_name}.nfo"
nfo_root: "musicvideo"
tags:
artist: "{music_video_artist}"
title: "{music_video_title}"
album: "{music_video_album}"
genre:
- "{music_video_genre}"
# Kodi v20 to not use year, so removing
premiered: "{music_video_date}"
_music_video_tags:
video_tags:
artist: "{music_video_artist}"
title: "{music_video_title}"
album: "{music_video_album}"
genre: "{music_video_genre}"
year: "{music_video_year}"
premiered: "{music_video_date}"
"Jellyfin Music Videos": "Jellyfin Music Videos":
preset: preset:
@ -76,3 +22,25 @@ presets:
- "_plex_video_base" - "_plex_video_base"
- "_music_video_base" - "_music_video_base"
- "_music_video_tags" - "_music_video_tags"
_music_video_nfo:
nfo_tags:
nfo_name: "{music_video_file_name}.nfo"
nfo_root: "musicvideo"
tags:
artist: "{music_video_artist}"
title: "{music_video_title}"
album: "{music_video_album}"
genre:
- "{music_video_genre}"
# Kodi v20 to not use year, so removing
premiered: "{music_video_date}"
_music_video_tags:
video_tags:
artist: "{music_video_artist}"
title: "{music_video_title}"
album: "{music_video_album}"
genre: "{music_video_genre}"
year: "{music_video_year}"
premiered: "{music_video_date}"

View file

@ -6,7 +6,7 @@ presets:
output_options: output_options:
output_directory: "{tv_show_directory}/{tv_show_name_sanitized}" output_directory: "{tv_show_directory}/{tv_show_name_sanitized}"
file_name: "{episode_file_path}.{ext}" file_name: "{episode_file_path}.{ext}"
thumbnail_name: "{episode_file_path}-thumb.jpg" thumbnail_name: "{thumbnail_file_name}"
info_json_name: "{episode_file_path}.{info_json_ext}" info_json_name: "{episode_file_path}.{info_json_ext}"
maintain_download_archive: True maintain_download_archive: True
@ -43,102 +43,7 @@ presets:
episode_date_standardized: "{upload_date_standardized}" episode_date_standardized: "{upload_date_standardized}"
episode_file_name: "s{season_number_padded}.e{episode_number_padded} - {file_title}" episode_file_name: "s{season_number_padded}.e{episode_number_padded} - {file_title}"
episode_file_path: "{season_directory_name_sanitized}/{episode_file_name_sanitized}" episode_file_path: "{season_directory_name_sanitized}/{episode_file_name_sanitized}"
thumbnail_file_name: "{episode_file_path}-thumb.jpg"
_episode_video_tags:
video_tags:
show: "{tv_show_name}"
genre: "{tv_show_genre}"
episode_id: "{episode_number}"
title: "{episode_title}"
synopsis: "{episode_plot}"
year: "{episode_year}"
date: "{episode_date_standardized}"
contentRating: "{episode_content_rating}"
_episode_nfo_tags:
nfo_tags:
nfo_name: "{episode_file_path}.nfo"
nfo_root: "episodedetails"
tags:
genre:
- "{tv_show_genre}"
season: "{season_number}"
episode: "{episode_number}"
title: "{episode_title}"
plot: "{episode_plot}"
year: "{episode_year}"
aired: "{episode_date_standardized}"
mpaa: "{episode_content_rating}"
output_directory_nfo_tags:
nfo_name: "tvshow.nfo"
nfo_root: "tvshow"
tags:
title: "{tv_show_name}"
genre:
- "{tv_show_genre}"
mpaa: "{tv_show_content_rating}"
####################################################################################################
_season_by_year:
overrides:
season_number: "{upload_year}"
season_number_padded: "{season_number}"
_season_by_year_month:
overrides:
season_number: "{upload_year}{upload_month_padded}"
season_number_padded: "{season_number}"
####################################################################################################
season_by_year_month__episode_by_day:
preset:
- "_season_by_year_month"
overrides:
episode_number: "{upload_day}{upload_date_index_padded}"
episode_number_padded: "{upload_day_padded}{upload_date_index_padded}"
season_by_year__episode_by_month_day:
preset:
- "_season_by_year"
overrides:
episode_number: "{upload_month}{upload_day_padded}{upload_date_index_padded}"
episode_number_padded: "{upload_month_padded}{upload_day_padded}{upload_date_index_padded}"
season_by_year__episode_by_month_day_reversed:
preset:
- "_season_by_year"
overrides:
episode_number: "{upload_day_of_year_reversed}{upload_date_index_reversed_padded}"
episode_number_padded: "{upload_day_of_year_reversed_padded}{upload_date_index_reversed_padded}"
season_by_year__episode_by_download_index:
preset:
- "_season_by_year"
overrides:
episode_number: "{download_index}"
episode_number_padded: "{download_index_padded6}"
###############
season_by_collection__episode_by_year_month_day:
overrides:
episode_number: "{upload_year_truncated}{upload_month_padded}{upload_day_padded}{upload_date_index_padded}"
episode_number_padded: "{episode_number}"
season_by_collection__episode_by_year_month_day_reversed:
overrides:
episode_number: "{upload_year_truncated_reversed}{upload_month_reversed_padded}{upload_day_reversed_padded}{upload_date_index_reversed_padded}"
episode_number_padded: "{episode_number}"
season_by_collection__episode_by_playlist_index:
overrides:
episode_number: "{playlist_index}"
episode_number_padded: "{playlist_index_padded6}"
season_by_collection__episode_by_playlist_index_reversed:
overrides:
episode_number: "{playlist_index_reversed}"
episode_number_padded: "{playlist_index_reversed_padded6}"

View file

@ -47,3 +47,38 @@ presets:
collection_season_number_padded: "{ %pad_zero(%int(collection_season_number), 2) }" collection_season_number_padded: "{ %pad_zero(%int(collection_season_number), 2) }"
season_number: "{collection_season_number}" season_number: "{collection_season_number}"
season_number_padded: "{collection_season_number_padded}" season_number_padded: "{collection_season_number_padded}"
_episode_video_tags:
video_tags:
show: "{tv_show_name}"
genre: "{tv_show_genre}"
episode_id: "{episode_number}"
title: "{episode_title}"
synopsis: "{episode_plot}"
year: "{episode_year}"
date: "{episode_date_standardized}"
contentRating: "{episode_content_rating}"
_episode_nfo_tags:
nfo_tags:
nfo_name: "{episode_file_path}.nfo"
nfo_root: "episodedetails"
tags:
genre:
- "{tv_show_genre}"
season: "{season_number}"
episode: "{episode_number}"
title: "{episode_title}"
plot: "{episode_plot}"
year: "{episode_year}"
aired: "{episode_date_standardized}"
mpaa: "{episode_content_rating}"
output_directory_nfo_tags:
nfo_name: "tvshow.nfo"
nfo_root: "tvshow"
tags:
title: "{tv_show_name}"
genre:
- "{tv_show_genre}"
mpaa: "{tv_show_content_rating}"

View file

@ -30,3 +30,45 @@ presets:
preset: preset:
- "plex_tv_show_by_date" - "plex_tv_show_by_date"
- "season_by_year__episode_by_month_day" - "season_by_year__episode_by_month_day"
####################################################################################################
_season_by_year:
overrides:
season_number: "{upload_year}"
season_number_padded: "{season_number}"
_season_by_year_month:
overrides:
season_number: "{upload_year}{upload_month_padded}"
season_number_padded: "{season_number}"
####################################################################################################
season_by_year_month__episode_by_day:
preset:
- "_season_by_year_month"
overrides:
episode_number: "{upload_day}{upload_date_index_padded}"
episode_number_padded: "{upload_day_padded}{upload_date_index_padded}"
season_by_year__episode_by_month_day:
preset:
- "_season_by_year"
overrides:
episode_number: "{upload_month}{upload_day_padded}{upload_date_index_padded}"
episode_number_padded: "{upload_month_padded}{upload_day_padded}{upload_date_index_padded}"
season_by_year__episode_by_month_day_reversed:
preset:
- "_season_by_year"
overrides:
episode_number: "{upload_day_of_year_reversed}{upload_date_index_reversed_padded}"
episode_number_padded: "{upload_day_of_year_reversed_padded}{upload_date_index_reversed_padded}"
season_by_year__episode_by_download_index:
preset:
- "_season_by_year"
overrides:
episode_number: "{download_index}"
episode_number_padded: "{download_index_padded6}"

View file

@ -1,6 +1,5 @@
presets: presets:
kodi_tv_show_collection: kodi_tv_show_collection:
preset: preset:
- "_kodi_tv_show" - "_kodi_tv_show"
@ -34,6 +33,11 @@ presets:
uid: "avatar_uncropped" uid: "avatar_uncropped"
- name: "{tv_show_fanart_file_name}" - name: "{tv_show_fanart_file_name}"
uid: "banner_uncropped" uid: "banner_uncropped"
source_thumbnails:
- name: "{tv_show_poster_file_name}"
uid: "avatar_uncropped"
- name: "{tv_show_fanart_file_name}"
uid: "banner_uncropped"
output_directory_nfo_tags: output_directory_nfo_tags:
tags: tags:
@ -704,3 +708,26 @@ presets:
- tag: "{collection_season_40_name}" - tag: "{collection_season_40_name}"
attributes: attributes:
number: "40" number: "40"
###############
season_by_collection__episode_by_year_month_day:
overrides:
episode_number: "{upload_year_truncated}{upload_month_padded}{upload_day_padded}{upload_date_index_padded}"
episode_number_padded: "{episode_number}"
season_by_collection__episode_by_year_month_day_reversed:
overrides:
episode_number: "{upload_year_truncated_reversed}{upload_month_reversed_padded}{upload_day_reversed_padded}{upload_date_index_reversed_padded}"
episode_number_padded: "{episode_number}"
season_by_collection__episode_by_playlist_index:
overrides:
episode_number: "{playlist_index}"
episode_number_padded: "{playlist_index_padded6}"
season_by_collection__episode_by_playlist_index_reversed:
overrides:
episode_number: "{playlist_index_reversed}"
episode_number_padded: "{playlist_index_reversed_padded6}"

View file

@ -73,12 +73,18 @@ class ArrayFunctions:
return Array(output) return Array(output)
@staticmethod @staticmethod
def array_at(array: Array, idx: Integer) -> AnyArgument: def array_at(array: Array, idx: Integer, default: Optional[AnyArgument] = None) -> AnyArgument:
""" """
:description: :description:
Return the element in the Array at index ``idx``. Return the element in the Array at index ``idx``. If ``idx`` exceeds the array length,
either return ``default`` if provided or throw an error.
""" """
return array.value[idx.value] try:
return array.value[idx.value]
except IndexError:
if default is not None:
return default
raise
@staticmethod @staticmethod
def array_first(array: Array, fallback: AnyArgument) -> AnyArgument: def array_first(array: Array, fallback: AnyArgument) -> AnyArgument:

View file

@ -1,8 +1,10 @@
from typing import Union from typing import Union
from ytdl_sub.script.types.resolvable import AnyArgument
from ytdl_sub.script.types.resolvable import Boolean from ytdl_sub.script.types.resolvable import Boolean
from ytdl_sub.script.types.resolvable import ReturnableArgumentA from ytdl_sub.script.types.resolvable import ReturnableArgumentA
from ytdl_sub.script.types.resolvable import ReturnableArgumentB from ytdl_sub.script.types.resolvable import ReturnableArgumentB
from ytdl_sub.script.utils.exceptions import FunctionRuntimeException
class ConditionalFunctions: class ConditionalFunctions:
@ -19,6 +21,39 @@ class ConditionalFunctions:
return true return true
return false return false
@staticmethod
def elif_(*if_elif_else: AnyArgument) -> AnyArgument:
"""
:description:
Conditional ``if`` statement that is capable of doing else-ifs (``elif``) via
adjacent arguments. It is expected for there to be an odd number of arguments >= 3 to
supply at least one conditional and an else.
:usage:
.. code-block:: python
%elif(
condition1,
return1,
condition2,
return2,
...
else_return
)
"""
arguments = list(if_elif_else)
if len(arguments) < 3:
raise FunctionRuntimeException("elif requires at least 3 arguments")
if len(arguments) % 2 == 0:
raise FunctionRuntimeException("elif must have an odd number of arguments")
for idx in range(0, len(arguments) - 1, 2):
if bool(arguments[idx].value):
return arguments[idx + 1]
return arguments[-1]
@staticmethod @staticmethod
def if_passthrough( def if_passthrough(
maybe_true_arg: ReturnableArgumentA, else_arg: ReturnableArgumentB maybe_true_arg: ReturnableArgumentA, else_arg: ReturnableArgumentB

View file

@ -1,5 +1,6 @@
from typing import Optional from typing import Optional
from ytdl_sub.script.types.array import Array
from ytdl_sub.script.types.resolvable import AnyArgument from ytdl_sub.script.types.resolvable import AnyArgument
from ytdl_sub.script.types.resolvable import Boolean from ytdl_sub.script.types.resolvable import Boolean
from ytdl_sub.script.types.resolvable import Integer from ytdl_sub.script.types.resolvable import Integer
@ -80,6 +81,22 @@ class StringFunctions:
return String(string.value.replace(old.value, new.value)) return String(string.value.replace(old.value, new.value))
@staticmethod
def split(string: String, sep: String, max_split: Optional[Integer] = None) -> Array:
"""
:description:
Splits the input string into multiple strings.
"""
if max_split is not None:
return Array(
[
String(split_val)
for split_val in string.value.split(sep=sep.value, maxsplit=max_split.value)
]
)
return Array([String(split_val) for split_val in string.value.split(sep=sep.value)])
@staticmethod @staticmethod
def concat(*values: String) -> String: def concat(*values: String) -> String:
""" """

View file

@ -248,6 +248,52 @@ class Script:
for variable_name, resolved in resolved_variables.items(): for variable_name, resolved in resolved_variables.items():
self._variables[variable_name] = SyntaxTree(ast=[resolved]) self._variables[variable_name] = SyntaxTree(ast=[resolved])
def _recursive_get_unresolved_output_filter_variables(
self, current_var: SyntaxTree, subset_to_resolve: Set[str], unresolvable: Set[Variable]
) -> Set[str]:
for var_dep in current_var.variables:
if var_dep in unresolvable:
raise ScriptVariableNotResolved(
f"Output filter variable contains the variable {var_dep} "
f"which is set as unresolvable"
)
subset_to_resolve.add(var_dep.name)
subset_to_resolve |= self._recursive_get_unresolved_output_filter_variables(
current_var=self._variables[var_dep.name],
subset_to_resolve=subset_to_resolve,
unresolvable=unresolvable,
)
return subset_to_resolve
def _get_unresolved_output_filter(
self,
unresolved: Dict[Variable, SyntaxTree],
output_filter: Set[str],
unresolvable: Set[Variable],
) -> Dict[Variable, SyntaxTree]:
"""
When an output filter is applied, only a subset of variables that the filter
depends on need to be resolved.
"""
subset_to_resolve: Set[str] = set()
for output_filter_variable in output_filter:
subset_to_resolve.add(output_filter_variable)
if output_filter_variable not in self._variables:
raise ScriptVariableNotResolved(
"Tried to specify an output filter variable that does not exist"
)
subset_to_resolve |= self._recursive_get_unresolved_output_filter_variables(
current_var=self._variables[output_filter_variable],
subset_to_resolve=subset_to_resolve,
unresolvable=unresolvable,
)
return {var: syntax for var, syntax in unresolved.items() if var.name in subset_to_resolve}
def _resolve( def _resolve(
self, self,
pre_resolved: Optional[Dict[str, Resolvable]] = None, pre_resolved: Optional[Dict[str, Resolvable]] = None,
@ -288,6 +334,13 @@ class Script:
if Variable(name) not in unresolved_filter if Variable(name) not in unresolved_filter
} }
if output_filter:
unresolved = self._get_unresolved_output_filter(
unresolved=unresolved,
output_filter=output_filter,
unresolvable=unresolvable,
)
while unresolved: while unresolved:
unresolved_count: int = len(unresolved) unresolved_count: int = len(unresolved)

View file

@ -3,6 +3,9 @@ from pathlib import Path
from typing import Any from typing import Any
from typing import Dict from typing import Dict
from typing import List from typing import List
from typing import Optional
from mergedeep import mergedeep
from ytdl_sub.config.config_file import ConfigFile from ytdl_sub.config.config_file import ConfigFile
from ytdl_sub.config.preset import Preset from ytdl_sub.config.preset import Preset
@ -69,7 +72,11 @@ class Subscription(SubscriptionDownload):
@classmethod @classmethod
def from_file_path( def from_file_path(
cls, config: ConfigFile, subscription_path: str | Path cls,
config: ConfigFile,
subscription_path: str | Path,
subscription_matches: Optional[List[str]] = None,
subscription_override_dict: Optional[Dict] = None,
) -> List["Subscription"]: ) -> List["Subscription"]:
""" """
Loads subscriptions from a file. Loads subscriptions from a file.
@ -80,6 +87,10 @@ class Subscription(SubscriptionDownload):
Validated instance of the config Validated instance of the config
subscription_path: subscription_path:
File path to the subscription yaml file File path to the subscription yaml file
subscription_matches:
Optional list, only output subscriptions that match one or more of these values
subscription_override_dict:
Optional dict containing overrides to every subscription
Returns Returns
------- -------
@ -91,16 +102,16 @@ class Subscription(SubscriptionDownload):
If subscription file is misconfigured If subscription file is misconfigured
""" """
subscriptions: List["Subscription"] = [] subscriptions: List["Subscription"] = []
subscription_dict = load_yaml(file_path=subscription_path) subscription_object = load_yaml(file_path=subscription_path)
has_file_preset = FILE_PRESET_APPLY_KEY in subscription_dict has_file_preset = FILE_PRESET_APPLY_KEY in subscription_object
# If a file preset is present... # If a file preset is present...
if has_file_preset: if has_file_preset:
# Validate it (make sure it is a dict) # Validate it (make sure it is a dict)
file_preset = LiteralDictValidator( file_preset = LiteralDictValidator(
name=f"{subscription_path}.{FILE_PRESET_APPLY_KEY}", name=f"{subscription_path}.{FILE_PRESET_APPLY_KEY}",
value=subscription_dict[FILE_PRESET_APPLY_KEY], value=subscription_object[FILE_PRESET_APPLY_KEY],
) )
# Deep copy the config and add this file preset to its preset list # Deep copy the config and add this file preset to its preset list
@ -108,7 +119,9 @@ class Subscription(SubscriptionDownload):
config.presets.dict[FILE_PRESET_APPLY_KEY] = file_preset.dict config.presets.dict[FILE_PRESET_APPLY_KEY] = file_preset.dict
subscriptions_dict: Dict[str, Any] = { subscriptions_dict: Dict[str, Any] = {
key: obj for key, obj in subscription_dict.items() if key not in [FILE_PRESET_APPLY_KEY] key: obj
for key, obj in subscription_object.items()
if key not in [FILE_PRESET_APPLY_KEY]
} }
subscriptions_dicts = SubscriptionValidator( subscriptions_dicts = SubscriptionValidator(
@ -121,11 +134,27 @@ class Subscription(SubscriptionDownload):
global_presets_to_apply=[FILE_PRESET_APPLY_KEY] if has_file_preset else [] global_presets_to_apply=[FILE_PRESET_APPLY_KEY] if has_file_preset else []
) )
for subscription_key, subscription_object in subscriptions_dicts.items(): if subscriptions_dicts and subscription_matches:
logger.info("Filtering subscriptions by name based on --match arguments")
subscriptions_dicts = {
subscription_name: subscription_object
for subscription_name, subscription_object in subscriptions_dicts.items()
if any(match in subscription_name for match in subscription_matches)
}
for subscription_name, subscription_object in subscriptions_dicts.items():
# Hard-override subscriptions here
mergedeep.merge(
subscription_object,
subscription_override_dict or {},
strategy=mergedeep.Strategy.ADDITIVE,
)
subscriptions.append( subscriptions.append(
cls.from_dict( cls.from_dict(
config=config, config=config,
preset_name=subscription_key, preset_name=subscription_name,
preset_dict=subscription_object, preset_dict=subscription_object,
) )
) )

View file

@ -190,7 +190,7 @@ class SubscriptionDownload(BaseSubscription, ABC):
------- -------
List of plugins defined in the subscription, initialized and ready to use. List of plugins defined in the subscription, initialized and ready to use.
""" """
return [ plugins = [
plugin_type( plugin_type(
options=plugin_options, options=plugin_options,
overrides=self.overrides, overrides=self.overrides,
@ -198,6 +198,7 @@ class SubscriptionDownload(BaseSubscription, ABC):
) )
for plugin_type, plugin_options in self.plugins.zipped() for plugin_type, plugin_options in self.plugins.zipped()
] ]
return [plugin for plugin in plugins if plugin.is_enabled]
@classmethod @classmethod
def _cleanup_entry_files(cls, entry: Entry): def _cleanup_entry_files(cls, entry: Entry):

View file

@ -84,10 +84,6 @@ class SubscriptionYTDLOptions:
"extract_flat": "discard", # do not store info.json in mem since its in file "extract_flat": "discard", # do not store info.json in mem since its in file
} }
@property
def _download_only_options(self) -> Dict:
return {"break_on_reject": True}
@property @property
def _output_options(self) -> Dict: def _output_options(self) -> Dict:
ytdl_options = {} ytdl_options = {}
@ -95,10 +91,12 @@ class SubscriptionYTDLOptions:
if self._preset.output_options.maintain_download_archive: if self._preset.output_options.maintain_download_archive:
ytdl_options["download_archive"] = self._enhanced_download_archive.working_file_path ytdl_options["download_archive"] = self._enhanced_download_archive.working_file_path
if self._preset.output_options.keep_max_files: if self._preset.output_options.keep_max_files:
# yt-dlp has a weird bug with max_downloads=1, set to 2 for safe measure keep_max_files = int(
ytdl_options["max_downloads"] = max( self._overrides.apply_formatter(self._preset.output_options.keep_max_files)
int(self._overrides.apply_formatter(self._preset.output_options.keep_max_files)), 2
) )
if keep_max_files > 0:
# yt-dlp has a weird bug with max_downloads=1, set to 2 for safe measure
ytdl_options["max_downloads"] = max(keep_max_files, 2)
return ytdl_options return ytdl_options
@ -176,6 +174,7 @@ class SubscriptionYTDLOptions:
self._output_options, self._output_options,
self._plugin_match_filters, self._plugin_match_filters,
self._plugin_ytdl_options(FormatPlugin), self._plugin_ytdl_options(FormatPlugin),
self._plugin_ytdl_options(AudioExtractPlugin), # will override format
self._user_ytdl_options, # user ytdl options... self._user_ytdl_options, # user ytdl options...
self._info_json_only_options, # then info_json_only options self._info_json_only_options, # then info_json_only options
) )
@ -193,10 +192,9 @@ class SubscriptionYTDLOptions:
self._plugin_ytdl_options(FileConvertPlugin), self._plugin_ytdl_options(FileConvertPlugin),
self._plugin_ytdl_options(SubtitlesPlugin), self._plugin_ytdl_options(SubtitlesPlugin),
self._plugin_ytdl_options(ChaptersPlugin), self._plugin_ytdl_options(ChaptersPlugin),
self._plugin_ytdl_options(AudioExtractPlugin),
self._plugin_ytdl_options(FormatPlugin), self._plugin_ytdl_options(FormatPlugin),
self._plugin_ytdl_options(AudioExtractPlugin), # will override format
self._user_ytdl_options, # user ytdl options... self._user_ytdl_options, # user ytdl options...
self._download_only_options, # then download_only options
) )
# Add dry run options last if enabled # Add dry run options last if enabled
if self._dry_run: if self._dry_run:

View file

@ -210,6 +210,14 @@ class Logger:
finally: finally:
redirect_stream.flush() redirect_stream.flush()
@classmethod
def _append_to_error_log(cls):
# Any time an exception occurs, dump all debug logs into the error log
with open(cls.debug_log_filename(), mode="r", encoding="utf-8") as debug_logs, open(
cls.error_log_filename(), mode="a", encoding="utf-8"
) as error_logs:
error_logs.writelines(debug_logs.readlines())
@classmethod @classmethod
def log_exception(cls, exception: Exception, log_filepath: Optional[Path] = None): def log_exception(cls, exception: Exception, log_filepath: Optional[Path] = None):
""" """
@ -248,14 +256,10 @@ class Logger:
log_filepath if log_filepath else Logger.error_log_filename(), log_filepath if log_filepath else Logger.error_log_filename(),
) )
# Any time an exception occurs, dump all debug logs into the error log cls._append_to_error_log()
with open(cls.debug_log_filename(), mode="r", encoding="utf-8") as debug_logs, open(
cls.error_log_filename(), mode="a", encoding="utf-8"
) as error_logs:
error_logs.writelines(debug_logs.readlines())
@classmethod @classmethod
def cleanup(cls, cleanup_error_log: bool = False): def cleanup(cls, has_error: bool = False):
""" """
Cleans up debug log file left behind Cleans up debug log file left behind
""" """
@ -263,9 +267,11 @@ class Logger:
for handler in logger.handlers: for handler in logger.handlers:
handler.close() handler.close()
cls._DEBUG_LOGGER_FILE.close() if has_error:
FileHandler.delete(cls.debug_log_filename()) cls._append_to_error_log()
else:
if cleanup_error_log:
cls._ERROR_LOG_FILE.close() cls._ERROR_LOG_FILE.close()
FileHandler.delete(cls.error_log_filename()) FileHandler.delete(cls.error_log_filename())
cls._DEBUG_LOGGER_FILE.close()
FileHandler.delete(cls.debug_log_filename())

View file

@ -38,3 +38,15 @@ class ScriptUtils:
out = f"{{%from_json('''{dumped_json}''')}}" out = f"{{%from_json('''{dumped_json}''')}}"
return out return out
@classmethod
def bool_formatter_output(cls, output: str) -> bool:
"""
Translate formatter output to a boolean
"""
if not output or output.lower() == "false":
return False
try:
return bool(json.loads(output))
except Exception: # pylint: disable=broad-except
return True

View file

@ -14,10 +14,8 @@ from ytdl_sub.script.utils.exceptions import RuntimeException
from ytdl_sub.utils.exceptions import StringFormattingException from ytdl_sub.utils.exceptions import StringFormattingException
from ytdl_sub.utils.script import ScriptUtils from ytdl_sub.utils.script import ScriptUtils
_BASE_SCRIPT: Script = Script( BASE_SCRIPT: Script = Script(
ScriptUtils.add_sanitized_variables( dict(ScriptUtils.add_sanitized_variables(VARIABLE_SCRIPTS), **CUSTOM_FUNCTION_SCRIPTS)
dict(copy.deepcopy(VARIABLE_SCRIPTS), **copy.deepcopy(CUSTOM_FUNCTION_SCRIPTS))
)
) )
@ -37,7 +35,7 @@ class Scriptable(ABC):
""" """
Initializes with base values Initializes with base values
""" """
self._script = copy.deepcopy(_BASE_SCRIPT) self._script = copy.deepcopy(BASE_SCRIPT)
self._unresolvable = copy.deepcopy(UNRESOLVED_VARIABLES) self._unresolvable = copy.deepcopy(UNRESOLVED_VARIABLES)
@property @property

View file

@ -6,8 +6,10 @@ from typing import final
from ytdl_sub.entries.script.variable_definitions import VARIABLES from ytdl_sub.entries.script.variable_definitions import VARIABLES
from ytdl_sub.script.parser import parse from ytdl_sub.script.parser import parse
from ytdl_sub.script.script import Script from ytdl_sub.script.script import Script
from ytdl_sub.script.types.syntax_tree import SyntaxTree
from ytdl_sub.script.utils.exceptions import RuntimeException
from ytdl_sub.script.utils.exceptions import ScriptVariableNotResolved
from ytdl_sub.script.utils.exceptions import UserException from ytdl_sub.script.utils.exceptions import UserException
from ytdl_sub.script.utils.exceptions import VariableDoesNotExist
from ytdl_sub.utils.exceptions import StringFormattingVariableNotFoundException from ytdl_sub.utils.exceptions import StringFormattingVariableNotFoundException
from ytdl_sub.validators.validators import DictValidator from ytdl_sub.validators.validators import DictValidator
from ytdl_sub.validators.validators import ListValidator from ytdl_sub.validators.validators import ListValidator
@ -90,7 +92,7 @@ class OverridesStringFormatterValidator(StringFormatterValidator):
# pylint: enable=line-too-long # pylint: enable=line-too-long
class OverridesIntegerFormatterValidator(StringFormatterValidator): class OverridesIntegerFormatterValidator(OverridesStringFormatterValidator):
_expected_value_type_name = "integer" _expected_value_type_name = "integer"
def post_process(self, resolved: str) -> str: def post_process(self, resolved: str) -> str:
@ -104,6 +106,10 @@ class OverridesIntegerFormatterValidator(StringFormatterValidator):
return resolved return resolved
class OverridesBooleanFormatterValidator(OverridesStringFormatterValidator):
_expected_value_type_name = "boolean"
class ListFormatterValidator(ListValidator[StringFormatterValidator]): class ListFormatterValidator(ListValidator[StringFormatterValidator]):
_inner_list_type = StringFormatterValidator _inner_list_type = StringFormatterValidator
@ -142,21 +148,67 @@ class OverridesDictFormatterValidator(DictFormatterValidator):
_key_validator = OverridesStringFormatterValidator _key_validator = OverridesStringFormatterValidator
def to_variable_dependency_format_string(script: Script, parsed_format_string: SyntaxTree) -> str:
"""
Create a dummy format string that contains all variable deps as a string.
"""
dummy_format_string = ""
for var in parsed_format_string.variables:
dummy_format_string += f"{{ {var.name} }}"
# pylint: disable=protected-access
for variable_dependency in script._variables[var.name].variables:
dummy_format_string += f"{{ {variable_dependency.name} }}"
# pylint: enable=protected-access
return dummy_format_string
def _validate_formatter( def _validate_formatter(
mock_script: Script, mock_script: Script,
unresolved_variables: Set[str], unresolved_variables: Set[str],
formatter_validator: Union[StringFormatterValidator, OverridesStringFormatterValidator], formatter_validator: Union[StringFormatterValidator, OverridesStringFormatterValidator],
) -> None: ) -> None:
try: is_static_formatter = False
unresolvable = unresolved_variables unresolvable = unresolved_variables
if isinstance(formatter_validator, OverridesStringFormatterValidator): if isinstance(formatter_validator, OverridesStringFormatterValidator):
unresolvable = unresolved_variables.union({VARIABLES.entry_metadata.variable_name}) is_static_formatter = True
unresolvable = unresolved_variables.union({VARIABLES.entry_metadata.variable_name})
parsed = parse(
text=formatter_validator.format_string,
)
variable_names = {var.name for var in parsed.variables}
custom_function_names = {f"%{func.name}" for func in parsed.custom_functions}
if not variable_names.issubset(mock_script.variable_names):
raise StringFormattingVariableNotFoundException(
"contains the following variables that do not exist: "
f"{', '.join(sorted(variable_names - mock_script.variable_names))}"
)
if not custom_function_names.issubset(mock_script.function_names):
raise StringFormattingVariableNotFoundException(
"contains the following custom functions that do not exist: "
f"{', '.join(sorted(custom_function_names - mock_script.function_names))}"
)
if unresolved := variable_names.intersection(unresolvable):
raise StringFormattingVariableNotFoundException(
"contains the following variables that are unresolved when executing this "
f"formatter: {', '.join(sorted(unresolved))}"
)
try:
mock_script.resolve_once( mock_script.resolve_once(
{"tmp_var": formatter_validator.format_string}, {
"tmp_var": to_variable_dependency_format_string(
script=mock_script, parsed_format_string=parsed
)
},
unresolvable=unresolvable, unresolvable=unresolvable,
) )
except VariableDoesNotExist as exc: except RuntimeException as exc:
if isinstance(exc, ScriptVariableNotResolved) and is_static_formatter:
raise StringFormattingVariableNotFoundException(
"static formatters must contain variables that have no dependency to "
"entry variables"
) from exc
raise StringFormattingVariableNotFoundException(exc) from exc raise StringFormattingVariableNotFoundException(exc) from exc

View file

@ -113,7 +113,7 @@ def assert_logs(
for call_args in patched_debug.call_args_list: for call_args in patched_debug.call_args_list:
occurrences += int(expected_message in call_args.args[0]) occurrences += int(expected_message in call_args.args[0])
if expected_occurrences: if expected_occurrences is not None:
assert ( assert (
occurrences == expected_occurrences occurrences == expected_occurrences
), f"{expected_message} was expected {expected_occurrences} times, got {occurrences}" ), f"{expected_message} was expected {expected_occurrences} times, got {occurrences}"

View file

@ -5,25 +5,6 @@ from expected_transaction_log import assert_transaction_log_matches
from ytdl_sub.subscriptions.subscription import Subscription from ytdl_sub.subscriptions.subscription import Subscription
@pytest.fixture
def single_preset_dict_old_format(output_directory):
return {
"preset": "Single",
# test multi-tags
"music_tags": {"embed_thumbnail": True, "tags": {"genres": ["multi_tag_1", "multi_tag_2"]}},
"format": "worst[ext=mp4]",
"audio_extract": {"codec": "mp3", "quality": 320},
"ytdl_options": {
"postprocessor_args": {"ffmpeg": ["-bitexact"]}, # Must add this for reproducibility
},
"overrides": {
"track_artist": "YouTube",
"url": "https://www.youtube.com/watch?v=2lAe1cqCOXo",
"music_directory": output_directory,
},
}
@pytest.fixture @pytest.fixture
def single_preset_dict(output_directory): def single_preset_dict(output_directory):
return { return {
@ -32,7 +13,6 @@ def single_preset_dict(output_directory):
"music_tags": {"genres": ["multi_tag_1", "multi_tag_2"]}, "music_tags": {"genres": ["multi_tag_1", "multi_tag_2"]},
# test the new embed_thumbnail plugin # test the new embed_thumbnail plugin
"embed_thumbnail": True, "embed_thumbnail": True,
"format": "worst[ext=mp4]",
"audio_extract": {"codec": "mp3", "quality": 320}, "audio_extract": {"codec": "mp3", "quality": 320},
"ytdl_options": { "ytdl_options": {
"postprocessor_args": {"ffmpeg": ["-bitexact"]}, # Must add this for reproducibility "postprocessor_args": {"ffmpeg": ["-bitexact"]}, # Must add this for reproducibility
@ -55,7 +35,6 @@ def youtube_release_preset_dict(output_directory):
return { return {
"preset": "YouTube Releases", "preset": "YouTube Releases",
"audio_extract": {"codec": "vorbis", "quality": 140}, "audio_extract": {"codec": "vorbis", "quality": 140},
"format": "worst[ext=mp4]", # download the worst format so it is fast
"ytdl_options": { "ytdl_options": {
"postprocessor_args": {"ffmpeg": ["-bitexact"]}, # Must add this for reproducibility "postprocessor_args": {"ffmpeg": ["-bitexact"]}, # Must add this for reproducibility
}, },
@ -68,32 +47,6 @@ def youtube_release_preset_dict(output_directory):
class TestAudioExtract: class TestAudioExtract:
@pytest.mark.parametrize("dry_run", [True, False])
def test_audio_extract_single_song_old_format(
self,
default_config,
single_preset_dict_old_format,
output_directory,
dry_run,
):
subscription = Subscription.from_dict(
config=default_config,
preset_name="single_song_test",
preset_dict=single_preset_dict_old_format,
)
transaction_log = subscription.download(dry_run=dry_run)
assert_transaction_log_matches(
output_directory=output_directory,
transaction_log=transaction_log,
transaction_log_summary_file_name="plugins/test_audio_extract_single_old_format.txt",
)
assert_expected_downloads(
output_directory=output_directory,
dry_run=dry_run,
expected_download_summary_file_name="plugins/test_audio_extract_single_old_format.json",
)
@pytest.mark.parametrize("dry_run", [False]) @pytest.mark.parametrize("dry_run", [False])
def test_audio_extract_single_song( def test_audio_extract_single_song(
self, self,

View file

@ -40,20 +40,30 @@ def rolling_recent_channel_preset_dict(recent_preset_dict):
class TestDateRange: class TestDateRange:
@pytest.mark.parametrize("dry_run", [True, False]) @pytest.mark.parametrize("dry_run", [True, False])
@pytest.mark.parametrize("date_range_breaks", [True, False])
def test_recent_channel_download( def test_recent_channel_download(
self, self,
recent_preset_dict, recent_preset_dict,
tv_show_config, tv_show_config,
output_directory, output_directory,
dry_run, dry_run: bool,
date_range_breaks: bool,
): ):
recent_preset_dict["date_range"]["breaks"] = date_range_breaks
recent_channel_subscription = Subscription.from_dict( recent_channel_subscription = Subscription.from_dict(
config=tv_show_config, config=tv_show_config,
preset_name="recent", preset_name="recent",
preset_dict=recent_preset_dict, preset_dict=recent_preset_dict,
) )
transaction_log = recent_channel_subscription.download(dry_run=dry_run) with assert_logs(
logger=YTDLP.logger,
expected_message="RejectedVideoReached, stopping additional downloads",
log_level="debug",
expected_occurrences=1 if date_range_breaks else 0,
):
transaction_log = recent_channel_subscription.download(dry_run=dry_run)
assert_transaction_log_matches( assert_transaction_log_matches(
output_directory=output_directory, output_directory=output_directory,
transaction_log=transaction_log, transaction_log=transaction_log,

View file

@ -18,7 +18,6 @@ def channel_preset_dict(output_directory):
"format": "worst[ext=mp4]", # download the worst format so it is fast "format": "worst[ext=mp4]", # download the worst format so it is fast
"ytdl_options": { "ytdl_options": {
"max_views": 100000, # do not download the popular PJ concert "max_views": 100000, # do not download the popular PJ concert
"break_on_reject": False, # do not break from max views
}, },
"subtitles": { "subtitles": {
"subtitles_name": "{episode_file_path}.{lang}.{subtitles_ext}", "subtitles_name": "{episode_file_path}.{lang}.{subtitles_ext}",

View file

@ -199,3 +199,26 @@ class TestPlaylist:
dry_run=dry_run, dry_run=dry_run,
expected_download_summary_file_name="youtube/test_playlist.json", expected_download_summary_file_name="youtube/test_playlist.json",
) )
def test_playlist_download_from_cli_sub_with_override_arg(
self,
preset_dict_to_subscription_yaml_generator,
playlist_preset_dict,
output_directory,
):
# TODO: Fix CLI parsing on windows when dealing with spaces
if IS_WINDOWS:
return
# No config needed when using only prebuilt presets
with preset_dict_to_subscription_yaml_generator(
subscription_name="music_video_playlist_test", preset_dict=playlist_preset_dict
) as subscription_path:
args = (
f"--dry-run sub '{subscription_path}' --dl-override '--date_range.after 20240101'"
)
subscriptions = mock_run_from_cli(args=args)
assert len(subscriptions) == 1
assert subscriptions[0].transaction_log.is_empty

View file

@ -14,30 +14,6 @@ from ytdl_sub.utils.system import IS_WINDOWS
from ytdl_sub.utils.thumbnail import try_convert_download_thumbnail from ytdl_sub.utils.thumbnail import try_convert_download_thumbnail
@pytest.fixture
def single_video_preset_dict_old_video_tags_format(output_directory):
return {
"preset": "Jellyfin Music Videos",
"download": "https://youtube.com/watch?v=HKTNxEqsN3Q",
# override the output directory with our fixture-generated dir
"output_options": {
"maintain_download_archive": False,
},
"embed_thumbnail": True, # embed thumb into the video
"format": "worst[ext=mp4]", # download the worst format so it is fast
# also test video tags
"video_tags": {
"tags": {
"title": "{title}",
}
},
"overrides": {
"music_video_artist": "JMC",
"music_video_directory": output_directory,
},
}
@pytest.fixture @pytest.fixture
def single_video_preset_dict(output_directory): def single_video_preset_dict(output_directory):
return { return {
@ -100,25 +76,6 @@ def single_video_preset_dict_dl_args(single_video_preset_dict):
class TestYoutubeVideo: class TestYoutubeVideo:
def test_single_video_old_video_tags_format_download(
self,
default_config,
single_video_preset_dict_old_video_tags_format,
output_directory,
):
single_video_subscription = Subscription.from_dict(
config=default_config,
preset_name="music_video_single_video_test",
preset_dict=single_video_preset_dict_old_video_tags_format,
)
transaction_log = single_video_subscription.download(dry_run=True)
assert_transaction_log_matches(
output_directory=output_directory,
transaction_log=transaction_log,
transaction_log_summary_file_name="youtube/test_video.txt",
)
@pytest.mark.parametrize("dry_run", [True, False]) @pytest.mark.parametrize("dry_run", [True, False])
def test_single_video_download( def test_single_video_download(
self, self,

View file

@ -1,6 +1,6 @@
{ {
".ytdl-sub-chapters_from_comments-download-archive.json": "2510b2ff3c54aa4813a4f23ea079e1ec", ".ytdl-sub-chapters_from_comments-download-archive.json": "2510b2ff3c54aa4813a4f23ea079e1ec",
"JMC/Move 78 - Automated Improvisation [Full Album].jpg": "c12e6a6f242680d1096a1a99d74a62c6", "JMC/Move 78 - Automated Improvisation [Full Album].jpg": "c12e6a6f242680d1096a1a99d74a62c6",
"JMC/Move 78 - Automated Improvisation [Full Album].mp4": "f401b98c332b76ee1c87065e195d73ce", "JMC/Move 78 - Automated Improvisation [Full Album].mp4": "068526b2d8f85fdcf914df3e23d0b1fa",
"JMC/Move 78 - Automated Improvisation [Full Album].nfo": "039268e97673a6f2b391772ec3b52fac" "JMC/Move 78 - Automated Improvisation [Full Album].nfo": "039268e97673a6f2b391772ec3b52fac"
} }

View file

@ -1,6 +1,6 @@
{ {
".ytdl-sub-match_filter_test-download-archive.json": "30f10646149d9eea4eb970749f352f7d", ".ytdl-sub-match_filter_test-download-archive.json": "30f10646149d9eea4eb970749f352f7d",
"match_filter_test/Jesse's Minecraft Server [Trailer - Mar.21].jpg": "e7830aa8a64b0cde65ba3f7e5fc56530", "match_filter_test/Jesse's Minecraft Server [Trailer - Mar.21].jpg": "e7830aa8a64b0cde65ba3f7e5fc56530",
"match_filter_test/Jesse's Minecraft Server [Trailer - Mar.21].mp4": "70204418e9af11a696611aa19571cf3a", "match_filter_test/Jesse's Minecraft Server [Trailer - Mar.21].mp4": "f17a540070964a199b35f981561d94e4",
"match_filter_test/Jesse's Minecraft Server [Trailer - Mar.21].nfo": "d85f4500bb5d8a2425d734a23b5a944c" "match_filter_test/Jesse's Minecraft Server [Trailer - Mar.21].nfo": "d85f4500bb5d8a2425d734a23b5a944c"
} }

View file

@ -1,15 +1,15 @@
{ {
".ytdl-sub-Proved Records-download-archive.json": "c3fb0b4f31caaa10ac7954ea93da33c4", ".ytdl-sub-Proved Records-download-archive.json": "c3fb0b4f31caaa10ac7954ea93da33c4",
"Proved Records/[2017] Alfa Mist - Nocturne [Full Album]/01 - 01. Intro (Feat. Racheal Ofori & Barney Artist).mp3": "29d09da874133ce5c5f7459c2fa6cd85", "Proved Records/[2017] Alfa Mist - Nocturne [Full Album]/01 - 01. Intro (Feat. Racheal Ofori & Barney Artist).mp3": "1e3583c9c1dc166b7baf98b81b4ca106",
"Proved Records/[2017] Alfa Mist - Nocturne [Full Album]/02 - 02. Answers (Feat. Rick David & Kaya Thomas - Dyke).mp3": "d5432e6a4f7af9810b0e7c8eebe4898a", "Proved Records/[2017] Alfa Mist - Nocturne [Full Album]/02 - 02. Answers (Feat. Rick David & Kaya Thomas - Dyke).mp3": "bb4c9085a4345515e6dffcf2c09f0f0f",
"Proved Records/[2017] Alfa Mist - Nocturne [Full Album]/03 - 03. Blaze (Feat. Kaya Thomas - Dyke).mp3": "cf03da6688a73373f9295dc595156e11", "Proved Records/[2017] Alfa Mist - Nocturne [Full Album]/03 - 03. Blaze (Feat. Kaya Thomas - Dyke).mp3": "c6766f80d0f9993ec6051555cd37dc32",
"Proved Records/[2017] Alfa Mist - Nocturne [Full Album]/04 - 04. What If (Interlude).mp3": "b7b943b6f3395c05433b8532364d63d6", "Proved Records/[2017] Alfa Mist - Nocturne [Full Album]/04 - 04. What If (Interlude).mp3": "2139e6eedc4d1bc2025f8ef78b0bd2af",
"Proved Records/[2017] Alfa Mist - Nocturne [Full Album]/05 - 05. No Peace (Feat. Tom Misch).mp3": "0d9719268900d4abcd8c0f51ad3af92c", "Proved Records/[2017] Alfa Mist - Nocturne [Full Album]/05 - 05. No Peace (Feat. Tom Misch).mp3": "8194d6a4018121e40b1733a0fa7a382e",
"Proved Records/[2017] Alfa Mist - Nocturne [Full Album]/06 - 06. Closer (Feat. Lester Duval).mp3": "bbd9ec616a0f7d3c5d13c04bb118681e", "Proved Records/[2017] Alfa Mist - Nocturne [Full Album]/06 - 06. Closer (Feat. Lester Duval).mp3": "e244183723e23c65156076b5f7ffcf56",
"Proved Records/[2017] Alfa Mist - Nocturne [Full Album]/07 - 07. Delusions Rumination (Interlude) (Feat. Racheal Ofori).mp3": "2e59ecdc0f8050bcf190a7898d7ae968", "Proved Records/[2017] Alfa Mist - Nocturne [Full Album]/07 - 07. Delusions Rumination (Interlude) (Feat. Racheal Ofori).mp3": "692224c169be4735c0892e05726cb335",
"Proved Records/[2017] Alfa Mist - Nocturne [Full Album]/08 - 08. Dreams (Feat. Carmody).mp3": "e58966f3326b876f0ab97459b681f76f", "Proved Records/[2017] Alfa Mist - Nocturne [Full Album]/08 - 08. Dreams (Feat. Carmody).mp3": "561c339d4d1232c15ad38a5afe8d61c3",
"Proved Records/[2017] Alfa Mist - Nocturne [Full Album]/09 - 09. Dreaming (Interlude) (Feat. Racheal Ofori).mp3": "d6939215b73457fce4e026f8c9b57ecc", "Proved Records/[2017] Alfa Mist - Nocturne [Full Album]/09 - 09. Dreaming (Interlude) (Feat. Racheal Ofori).mp3": "bdef7ad26848b45552891b9ea722659e",
"Proved Records/[2017] Alfa Mist - Nocturne [Full Album]/10 - 10. Hopeful (Feat. Jordan Rakei).mp3": "82d0e7d8070fd15d71c2e11c0285d426", "Proved Records/[2017] Alfa Mist - Nocturne [Full Album]/10 - 10. Hopeful (Feat. Jordan Rakei).mp3": "ffff4506702e708288eb54eff8fbaca4",
"Proved Records/[2017] Alfa Mist - Nocturne [Full Album]/11 - 11. Sunrise (Pillows) (Feat. Emmavie).mp3": "3edb11e8cd5f270dd42ee62d9a9aa4ff", "Proved Records/[2017] Alfa Mist - Nocturne [Full Album]/11 - 11. Sunrise (Pillows) (Feat. Emmavie).mp3": "41a88d6c07047bb4215fed0e71443f17",
"Proved Records/[2017] Alfa Mist - Nocturne [Full Album]/folder.jpg": "bd3685acc53072e591bae2505ecb0648" "Proved Records/[2017] Alfa Mist - Nocturne [Full Album]/folder.jpg": "bd3685acc53072e591bae2505ecb0648"
} }

View file

@ -1,5 +1,5 @@
{ {
".ytdl-sub-split_by_chapters_with_regex_video_no_chapters-download-archive.json": "4008e43668447f1a3a6a55520a6ff475", ".ytdl-sub-split_by_chapters_with_regex_video_no_chapters-download-archive.json": "4008e43668447f1a3a6a55520a6ff475",
"Project Zombie/[2010] Oblivion Mod Falcor p.1/01 - Oblivion Mod Falcor p.1.mp3": "d53121df33ac8c4a4699ec8919196552", "Project Zombie/[2010] Oblivion Mod Falcor p.1/01 - Oblivion Mod Falcor p.1.mp3": "b886f268a2a9b3b62f528fcf46699082",
"Project Zombie/[2010] Oblivion Mod Falcor p.1/folder.jpg": "fb95b510681676e81c321171fc23143e" "Project Zombie/[2010] Oblivion Mod Falcor p.1/folder.jpg": "fb95b510681676e81c321171fc23143e"
} }

View file

@ -1,15 +1,15 @@
{ {
".ytdl-sub-split_by_chapters_with_regex_video_preset-download-archive.json": "9798e8289742586d0efd295a97c6c906", ".ytdl-sub-split_by_chapters_with_regex_video_preset-download-archive.json": "9798e8289742586d0efd295a97c6c906",
"Alfa Mist/[2017] Nocturne/01 - Intro (Feat. Racheal Ofori & Barney Artist).mp3": "7be57c4dde9ba2c3fa72e9cc65b0f586", "Alfa Mist/[2017] Nocturne/01 - Intro (Feat. Racheal Ofori & Barney Artist).mp3": "d07f90d214416a11736e3f73c0955208",
"Alfa Mist/[2017] Nocturne/02 - Answers (Feat. Rick David & Kaya Thomas - Dyke).mp3": "5ee032da9965c51913fcfa0319d061bd", "Alfa Mist/[2017] Nocturne/02 - Answers (Feat. Rick David & Kaya Thomas - Dyke).mp3": "9057d3d7fcd98701e06d399a507c0e5b",
"Alfa Mist/[2017] Nocturne/03 - Blaze (Feat. Kaya Thomas - Dyke).mp3": "82c1f35bdccfb4ec146f33661ac94d6b", "Alfa Mist/[2017] Nocturne/03 - Blaze (Feat. Kaya Thomas - Dyke).mp3": "4cc4e7066112388e83bdd9e803e8051a",
"Alfa Mist/[2017] Nocturne/04 - What If (Interlude).mp3": "040b38c249e478d8832dcea8bfeca17f", "Alfa Mist/[2017] Nocturne/04 - What If (Interlude).mp3": "8d0f04c04f8c2b0565954dac66ee29a3",
"Alfa Mist/[2017] Nocturne/05 - No Peace (Feat. Tom Misch).mp3": "06f7e57fecdc7b4d2e37c7652791ce19", "Alfa Mist/[2017] Nocturne/05 - No Peace (Feat. Tom Misch).mp3": "e8166110c85e6b9af8d1067b8ffbb428",
"Alfa Mist/[2017] Nocturne/06 - Closer (Feat. Lester Duval).mp3": "91a03449c33bf9dcff5a9654a8525626", "Alfa Mist/[2017] Nocturne/06 - Closer (Feat. Lester Duval).mp3": "64e39199ddcd05b96c15c82a3c695d25",
"Alfa Mist/[2017] Nocturne/07 - Delusions Rumination (Interlude) (Feat. Racheal Ofori).mp3": "54c26621b8d4e74f37217c836479a077", "Alfa Mist/[2017] Nocturne/07 - Delusions Rumination (Interlude) (Feat. Racheal Ofori).mp3": "9a53bee41d1985e48e771a02b052986b",
"Alfa Mist/[2017] Nocturne/08 - Dreams (Feat. Carmody).mp3": "2704327ac5998086e77a77da164e2afd", "Alfa Mist/[2017] Nocturne/08 - Dreams (Feat. Carmody).mp3": "19c50e84c08eacdd1095e461306d9077",
"Alfa Mist/[2017] Nocturne/09 - Dreaming (Interlude) (Feat. Racheal Ofori).mp3": "75b8a26d3099aa2d9ba488b33c4eb1e7", "Alfa Mist/[2017] Nocturne/09 - Dreaming (Interlude) (Feat. Racheal Ofori).mp3": "f28df20fdbadf0ec4985b5f1f30c4e5d",
"Alfa Mist/[2017] Nocturne/10 - Hopeful (Feat. Jordan Rakei).mp3": "0bf0979c99c55ef986efaba55c0dc858", "Alfa Mist/[2017] Nocturne/10 - Hopeful (Feat. Jordan Rakei).mp3": "d7261e48713426ad4905919915a9b103",
"Alfa Mist/[2017] Nocturne/11 - Sunrise (Pillows) (Feat. Emmavie).mp3": "78950fbfe459de8e9aa951ab6fa153bf", "Alfa Mist/[2017] Nocturne/11 - Sunrise (Pillows) (Feat. Emmavie).mp3": "5d51491d0999e1f29918d4db4ed796d3",
"Alfa Mist/[2017] Nocturne/folder.jpg": "bd3685acc53072e591bae2505ecb0648" "Alfa Mist/[2017] Nocturne/folder.jpg": "bd3685acc53072e591bae2505ecb0648"
} }

View file

@ -1,7 +1,7 @@
{ {
".ytdl-sub-multiple_songs_test-download-archive.json": "54237df5e00d1598dfd39f341ee03d75", ".ytdl-sub-multiple_songs_test-download-archive.json": "54237df5e00d1598dfd39f341ee03d75",
"Project Zombie/[2011] Jesse's Minecraft Server/01 - Jesse's Minecraft Server [Trailer - Mar.21].ogg": "5657c5b92f8980b20d8bbee0fdc7e5d8", "Project Zombie/[2011] Jesse's Minecraft Server/01 - Jesse's Minecraft Server [Trailer - Mar.21].ogg": "e7a8a94ebe9f02f086f4bbf3df3946f6",
"Project Zombie/[2011] Jesse's Minecraft Server/02 - Jesse's Minecraft Server [Trailer - Feb.27].ogg": "a2a3a34e02e26a6c0265530d4499473b", "Project Zombie/[2011] Jesse's Minecraft Server/02 - Jesse's Minecraft Server [Trailer - Feb.27].ogg": "aeea4b086507fd7fde3c09c0bd868950",
"Project Zombie/[2011] Jesse's Minecraft Server/03 - Jesse's Minecraft Server [Trailer - Feb.1].ogg": "0a385da3aa06b994a69b8ab812b44975", "Project Zombie/[2011] Jesse's Minecraft Server/03 - Jesse's Minecraft Server [Trailer - Feb.1].ogg": "80200d21a46c7f521bfb23aef2f87e34",
"Project Zombie/[2011] Jesse's Minecraft Server/folder.jpg": "e7830aa8a64b0cde65ba3f7e5fc56530" "Project Zombie/[2011] Jesse's Minecraft Server/folder.jpg": "e7830aa8a64b0cde65ba3f7e5fc56530"
} }

View file

@ -1,5 +1,5 @@
{ {
".ytdl-sub-single_song_test-download-archive.json": "c8ff22ec3304c9f8dab18cedaed4e8b4", ".ytdl-sub-single_song_test-download-archive.json": "c8ff22ec3304c9f8dab18cedaed4e8b4",
"YouTube/[2019] YouTube Rewind 2019 For the Record #YouTubeRewind/01 - YouTube Rewind 2019 For the Record #YouTubeRewind.mp3": "991b0eb62683c2194c5bdfa9a61ab18e", "YouTube/[2019] YouTube Rewind 2019 For the Record #YouTubeRewind/01 - YouTube Rewind 2019 For the Record #YouTubeRewind.mp3": "829eb7dcc5dcae41240701dec4e1708d",
"YouTube/[2019] YouTube Rewind 2019 For the Record #YouTubeRewind/folder.jpg": "50ee47c80f679029f5d3503bb91b045a" "YouTube/[2019] YouTube Rewind 2019 For the Record #YouTubeRewind/folder.jpg": "50ee47c80f679029f5d3503bb91b045a"
} }

View file

@ -1,5 +1,5 @@
{ {
".ytdl-sub-single_song_best_test-download-archive.json": "f179ccfe0a9b3a76ae62122a3ccb58fd", ".ytdl-sub-single_song_best_test-download-archive.json": "0f9484ed868dcbeef82810a3cf7b0eea",
"YouTube/[2019] YouTube Rewind 2019 For the Record #YouTubeRewind/01 - YouTube Rewind 2019 For the Record #YouTubeRewind.m4a": "114b35c29df84146cd98207f807b837b", "YouTube/[2019] YouTube Rewind 2019 For the Record #YouTubeRewind/01 - YouTube Rewind 2019 For the Record #YouTubeRewind.opus": "73e4afdda9bc792807c8b07a63128e5a",
"YouTube/[2019] YouTube Rewind 2019 For the Record #YouTubeRewind/folder.jpg": "50ee47c80f679029f5d3503bb91b045a" "YouTube/[2019] YouTube Rewind 2019 For the Record #YouTubeRewind/folder.jpg": "50ee47c80f679029f5d3503bb91b045a"
} }

View file

@ -1,5 +0,0 @@
{
".ytdl-sub-single_song_test-download-archive.json": "c8ff22ec3304c9f8dab18cedaed4e8b4",
"YouTube/[2019] YouTube Rewind 2019 For the Record #YouTubeRewind/01 - YouTube Rewind 2019 For the Record #YouTubeRewind.mp3": "11376667a11bb71565b520f8ce5fa303",
"YouTube/[2019] YouTube Rewind 2019 For the Record #YouTubeRewind/folder.jpg": "50ee47c80f679029f5d3503bb91b045a"
}

View file

@ -1,6 +1,6 @@
{ {
".ytdl-sub-sponsorblock_with_embedded_subs_test-download-archive.json": "2cb4b9586fd5bb7f1fed76ed9195e6e4", ".ytdl-sub-sponsorblock_with_embedded_subs_test-download-archive.json": "2cb4b9586fd5bb7f1fed76ed9195e6e4",
"JMC/This GPU SLIDES into this Case! - Silverstone SUGO 16 ITX Case.jpg": "b5353a824a4800cc26f884e3025ed969", "JMC/This GPU SLIDES into this Case! - Silverstone SUGO 16 ITX Case.jpg": "b5353a824a4800cc26f884e3025ed969",
"JMC/This GPU SLIDES into this Case! - Silverstone SUGO 16 ITX Case.mp4": "8c12640f0c5c280c7a77423431b4ecb1", "JMC/This GPU SLIDES into this Case! - Silverstone SUGO 16 ITX Case.mp4": "526b6df52a8aaf11dfe56f25ac35a567",
"JMC/This GPU SLIDES into this Case! - Silverstone SUGO 16 ITX Case.nfo": "ae73ec18a9f0e5a54c90061ccd32e7f4" "JMC/This GPU SLIDES into this Case! - Silverstone SUGO 16 ITX Case.nfo": "ae73ec18a9f0e5a54c90061ccd32e7f4"
} }

View file

@ -18,7 +18,7 @@
"Project Zombie/Season 2011/s2011.e022701 - Jesse's Minecraft Server [Trailer - Feb.27].nfo": "20af231e30a035fb2bc0c946f4b4026d", "Project Zombie/Season 2011/s2011.e022701 - Jesse's Minecraft Server [Trailer - Feb.27].nfo": "20af231e30a035fb2bc0c946f4b4026d",
"Project Zombie/Season 2011/s2011.e032101 - Jesse's Minecraft Server [Trailer - Mar.21]-thumb.jpg": "e7830aa8a64b0cde65ba3f7e5fc56530", "Project Zombie/Season 2011/s2011.e032101 - Jesse's Minecraft Server [Trailer - Mar.21]-thumb.jpg": "e7830aa8a64b0cde65ba3f7e5fc56530",
"Project Zombie/Season 2011/s2011.e032101 - Jesse's Minecraft Server [Trailer - Mar.21].info.json": "INFO_JSON", "Project Zombie/Season 2011/s2011.e032101 - Jesse's Minecraft Server [Trailer - Mar.21].info.json": "INFO_JSON",
"Project Zombie/Season 2011/s2011.e032101 - Jesse's Minecraft Server [Trailer - Mar.21].mp4": "5f12b36e5ce717fa9550000237e2e5b8", "Project Zombie/Season 2011/s2011.e032101 - Jesse's Minecraft Server [Trailer - Mar.21].mp4": "eac3dfce44c2a723f2fc74e9552aa510",
"Project Zombie/Season 2011/s2011.e032101 - Jesse's Minecraft Server [Trailer - Mar.21].nfo": "f8bdc463c0cb2ffdc82aba2bcc27ad5d", "Project Zombie/Season 2011/s2011.e032101 - Jesse's Minecraft Server [Trailer - Mar.21].nfo": "f8bdc463c0cb2ffdc82aba2bcc27ad5d",
"Project Zombie/Season 2011/s2011.e052901 - Project Zombie Official Trailer (IP mc.projectzombie.beastnode.net)-thumb.jpg": "c956192a379b3661595c9920972d4819", "Project Zombie/Season 2011/s2011.e052901 - Project Zombie Official Trailer (IP mc.projectzombie.beastnode.net)-thumb.jpg": "c956192a379b3661595c9920972d4819",
"Project Zombie/Season 2011/s2011.e052901 - Project Zombie Official Trailer (IP mc.projectzombie.beastnode.net).info.json": "INFO_JSON", "Project Zombie/Season 2011/s2011.e052901 - Project Zombie Official Trailer (IP mc.projectzombie.beastnode.net).info.json": "INFO_JSON",
@ -30,7 +30,7 @@
"Project Zombie/Season 2011/s2011.e063001 - Project Zombie Fin.nfo": "54ea4a48116aa98480a79495036c25e9", "Project Zombie/Season 2011/s2011.e063001 - Project Zombie Fin.nfo": "54ea4a48116aa98480a79495036c25e9",
"Project Zombie/Season 2011/s2011.e112101 - Skyrim 'Ultra HD wMods' [PC]-thumb.jpg": "1718599d5189c65f7d8cf6acfa5ea851", "Project Zombie/Season 2011/s2011.e112101 - Skyrim 'Ultra HD wMods' [PC]-thumb.jpg": "1718599d5189c65f7d8cf6acfa5ea851",
"Project Zombie/Season 2011/s2011.e112101 - Skyrim 'Ultra HD wMods' [PC].info.json": "INFO_JSON", "Project Zombie/Season 2011/s2011.e112101 - Skyrim 'Ultra HD wMods' [PC].info.json": "INFO_JSON",
"Project Zombie/Season 2011/s2011.e112101 - Skyrim 'Ultra HD wMods' [PC].mp4": "2a18cc4baa198edecdc5154082f73eff", "Project Zombie/Season 2011/s2011.e112101 - Skyrim 'Ultra HD wMods' [PC].mp4": "554d46311112228c22ca9532007803cd",
"Project Zombie/Season 2011/s2011.e112101 - Skyrim 'Ultra HD wMods' [PC].nfo": "04f6aad56d85b1f65b81b6b8000f6479", "Project Zombie/Season 2011/s2011.e112101 - Skyrim 'Ultra HD wMods' [PC].nfo": "04f6aad56d85b1f65b81b6b8000f6479",
"Project Zombie/Season 2012/s2012.e012301 - Project Zombie Map Trailer-thumb.jpg": "54ebe9df801b278fdd17b21afa8373a6", "Project Zombie/Season 2012/s2012.e012301 - Project Zombie Map Trailer-thumb.jpg": "54ebe9df801b278fdd17b21afa8373a6",
"Project Zombie/Season 2012/s2012.e012301 - Project Zombie Map Trailer.info.json": "INFO_JSON", "Project Zombie/Season 2012/s2012.e012301 - Project Zombie Map Trailer.info.json": "INFO_JSON",

View file

@ -10,8 +10,10 @@
"JMC/Season 01/s01.e11022701 - Jesse's Minecraft Server [Trailer - Feb.27].nfo": "da7645e8826586388ae0d8278ef6a1c1", "JMC/Season 01/s01.e11022701 - Jesse's Minecraft Server [Trailer - Feb.27].nfo": "da7645e8826586388ae0d8278ef6a1c1",
"JMC/Season 01/s01.e11032101 - Jesse's Minecraft Server [Trailer - Mar.21]-thumb.jpg": "e7830aa8a64b0cde65ba3f7e5fc56530", "JMC/Season 01/s01.e11032101 - Jesse's Minecraft Server [Trailer - Mar.21]-thumb.jpg": "e7830aa8a64b0cde65ba3f7e5fc56530",
"JMC/Season 01/s01.e11032101 - Jesse's Minecraft Server [Trailer - Mar.21].info.json": "INFO_JSON", "JMC/Season 01/s01.e11032101 - Jesse's Minecraft Server [Trailer - Mar.21].info.json": "INFO_JSON",
"JMC/Season 01/s01.e11032101 - Jesse's Minecraft Server [Trailer - Mar.21].mp4": "7f1a1d5c1d94938a9b6f565e8159c3d6", "JMC/Season 01/s01.e11032101 - Jesse's Minecraft Server [Trailer - Mar.21].mp4": "dc04853870bbb811f6b312e20b58253b",
"JMC/Season 01/s01.e11032101 - Jesse's Minecraft Server [Trailer - Mar.21].nfo": "c56083e2f3545fa2cafc4d67cbfdacf8", "JMC/Season 01/s01.e11032101 - Jesse's Minecraft Server [Trailer - Mar.21].nfo": "c56083e2f3545fa2cafc4d67cbfdacf8",
"JMC/fanart.jpg": "129c6639b47299bc48062f0365e670ee",
"JMC/poster.jpg": "5de28eea5a921a041452ab3ce1041f73",
"JMC/season01-poster.jpg": "e7830aa8a64b0cde65ba3f7e5fc56530", "JMC/season01-poster.jpg": "e7830aa8a64b0cde65ba3f7e5fc56530",
"JMC/tvshow.nfo": "e92e4a2c01522dd9a9c3423f0f9304dc" "JMC/tvshow.nfo": "e92e4a2c01522dd9a9c3423f0f9304dc"
} }

View file

@ -10,8 +10,10 @@
"JMC/Season 01/s01.e11022701 - Jesse's Minecraft Server [Trailer - Feb.27].nfo": "da7645e8826586388ae0d8278ef6a1c1", "JMC/Season 01/s01.e11022701 - Jesse's Minecraft Server [Trailer - Feb.27].nfo": "da7645e8826586388ae0d8278ef6a1c1",
"JMC/Season 01/s01.e11032101 - Jesse's Minecraft Server [Trailer - Mar.21]-thumb.jpg": "e7830aa8a64b0cde65ba3f7e5fc56530", "JMC/Season 01/s01.e11032101 - Jesse's Minecraft Server [Trailer - Mar.21]-thumb.jpg": "e7830aa8a64b0cde65ba3f7e5fc56530",
"JMC/Season 01/s01.e11032101 - Jesse's Minecraft Server [Trailer - Mar.21].info.json": "INFO_JSON", "JMC/Season 01/s01.e11032101 - Jesse's Minecraft Server [Trailer - Mar.21].info.json": "INFO_JSON",
"JMC/Season 01/s01.e11032101 - Jesse's Minecraft Server [Trailer - Mar.21].mp4": "7f1a1d5c1d94938a9b6f565e8159c3d6", "JMC/Season 01/s01.e11032101 - Jesse's Minecraft Server [Trailer - Mar.21].mp4": "dc04853870bbb811f6b312e20b58253b",
"JMC/Season 01/s01.e11032101 - Jesse's Minecraft Server [Trailer - Mar.21].nfo": "c56083e2f3545fa2cafc4d67cbfdacf8", "JMC/Season 01/s01.e11032101 - Jesse's Minecraft Server [Trailer - Mar.21].nfo": "c56083e2f3545fa2cafc4d67cbfdacf8",
"JMC/fanart.jpg": "129c6639b47299bc48062f0365e670ee",
"JMC/poster.jpg": "5de28eea5a921a041452ab3ce1041f73",
"JMC/season01-poster.jpg": "e7830aa8a64b0cde65ba3f7e5fc56530", "JMC/season01-poster.jpg": "e7830aa8a64b0cde65ba3f7e5fc56530",
"JMC/tvshow.nfo": "e92e4a2c01522dd9a9c3423f0f9304dc" "JMC/tvshow.nfo": "e92e4a2c01522dd9a9c3423f0f9304dc"
} }

View file

@ -3,7 +3,7 @@ Files created:
{output_directory} {output_directory}
.ytdl-sub-single_song_best_test-download-archive.json .ytdl-sub-single_song_best_test-download-archive.json
{output_directory}/YouTube/[2019] YouTube Rewind 2019 For the Record #YouTubeRewind {output_directory}/YouTube/[2019] YouTube Rewind 2019 For the Record #YouTubeRewind
01 - YouTube Rewind 2019 For the Record #YouTubeRewind.m4a 01 - YouTube Rewind 2019 For the Record #YouTubeRewind.opus
Music Tags: Music Tags:
album: YouTube Rewind 2019: For the Record | #YouTubeRewind album: YouTube Rewind 2019: For the Record | #YouTubeRewind
albumartist: YouTube albumartist: YouTube

View file

@ -1,18 +0,0 @@
Files created:
----------------------------------------
{output_directory}
.ytdl-sub-single_song_test-download-archive.json
{output_directory}/YouTube/[2019] YouTube Rewind 2019 For the Record #YouTubeRewind
01 - YouTube Rewind 2019 For the Record #YouTubeRewind.mp3
Embedded Thumbnail, Music Tags:
album: YouTube Rewind 2019: For the Record | #YouTubeRewind
albumartist: YouTube
albumartists: YouTube
artist: YouTube
artists: YouTube
genres: Unset
title: YouTube Rewind 2019: For the Record | #YouTubeRewind
track: 1
tracktotal: 1
year: 2019
folder.jpg

View file

@ -2,6 +2,8 @@ Files created:
---------------------------------------- ----------------------------------------
{output_directory} {output_directory}
.ytdl-sub-music_video_playlist_test-download-archive.json .ytdl-sub-music_video_playlist_test-download-archive.json
fanart.jpg
poster.jpg
season01-poster.jpg season01-poster.jpg
tvshow.nfo tvshow.nfo
NFO tags: NFO tags:

View file

@ -2,6 +2,7 @@ import re
import sys import sys
from pathlib import Path from pathlib import Path
from typing import Callable from typing import Callable
from typing import List
from unittest.mock import patch from unittest.mock import patch
import pytest import pytest
@ -22,6 +23,7 @@ from ytdl_sub.utils.exceptions import ExperimentalFeatureNotEnabled
@pytest.mark.parametrize("dry_run", [True, False]) @pytest.mark.parametrize("dry_run", [True, False])
@pytest.mark.parametrize("mock_success_output", [True, False]) @pytest.mark.parametrize("mock_success_output", [True, False])
@pytest.mark.parametrize("keep_successful_logs", [True, False]) @pytest.mark.parametrize("keep_successful_logs", [True, False])
@pytest.mark.parametrize("match", [[], ["Rick", "Michael"]])
def test_subscription_logs_write_to_file( def test_subscription_logs_write_to_file(
persist_logs_directory: str, persist_logs_directory: str,
persist_logs_config_factory: Callable, persist_logs_config_factory: Callable,
@ -30,8 +32,11 @@ def test_subscription_logs_write_to_file(
dry_run: bool, dry_run: bool,
mock_success_output: bool, mock_success_output: bool,
keep_successful_logs: bool, keep_successful_logs: bool,
match: List[str],
): ):
subscripton_names = ["Rick Astley", "Michael Jackson", "Eric Clapton"] subscription_names = ["Rick Astley", "Michael Jackson", "Eric Clapton"]
if match:
subscription_names = ["Rick Astley", "Michael Jackson"]
num_runs = 2 num_runs = 2
config = persist_logs_config_factory(keep_successful_logs=keep_successful_logs) config = persist_logs_config_factory(keep_successful_logs=keep_successful_logs)
@ -47,6 +52,8 @@ def test_subscription_logs_write_to_file(
_download_subscriptions_from_yaml_files( _download_subscriptions_from_yaml_files(
config=config, config=config,
subscription_paths=subscription_paths, subscription_paths=subscription_paths,
subscription_matches=match,
subscription_override_dict={},
update_with_info_json=False, update_with_info_json=False,
dry_run=dry_run, dry_run=dry_run,
) )
@ -61,8 +68,8 @@ def test_subscription_logs_write_to_file(
return return
# If not success, expect 2 log files for both sub errors # If not success, expect 2 log files for both sub errors
elif not mock_success_output: elif not mock_success_output:
assert len(log_directory_files) == (num_runs * len(subscripton_names)) assert len(log_directory_files) == (num_runs * len(subscription_names))
for log_path, subscription_name in zip(log_directory_files, subscripton_names): for log_path, subscription_name in zip(log_directory_files, subscription_names):
subscription_log_file_name = subscription_name.lower().replace(" ", "_") subscription_log_file_name = subscription_name.lower().replace(" ", "_")
assert bool(re.match(rf"\d\.{subscription_log_file_name}\.error\.log", log_path.name)) assert bool(re.match(rf"\d\.{subscription_log_file_name}\.error\.log", log_path.name))
@ -74,9 +81,9 @@ def test_subscription_logs_write_to_file(
) )
# If success and success logging, expect 3 log files # If success and success logging, expect 3 log files
else: else:
assert len(log_directory_files) == (num_runs * len(subscripton_names)) assert len(log_directory_files) == (num_runs * len(subscription_names))
for log_file_path, subscription_name in zip( for log_file_path, subscription_name in zip(
log_directory_files, subscripton_names * num_runs log_directory_files, subscription_names * num_runs
): ):
subscription_log_file_name = subscription_name.lower().replace(" ", "_") subscription_log_file_name = subscription_name.lower().replace(" ", "_")

View file

@ -17,9 +17,6 @@ class TestPreset:
"youtube.com/watch?v=123abc", # single string "youtube.com/watch?v=123abc", # single string
["youtube.com/watch?v=123abc", "youtube.com/watch?v=123xyz"], # list of strings ["youtube.com/watch?v=123abc", "youtube.com/watch?v=123xyz"], # list of strings
[{"url": "youtube.com/watch?v=123abc"}, "youtube.com/watch?v=123abc"], # dict and str [{"url": "youtube.com/watch?v=123abc"}, "youtube.com/watch?v=123abc"], # dict and str
# OLD download_strategy format
{"download_strategy": "url", "url": "youtube.com/watch?v=123abc"},
{"download_strategy": "multi-url", "urls": [{"url": "youtube.com/watch?v=123abc"}]},
], ],
) )
def test_bare_minimum_preset(self, config_file, output_options, download_value): def test_bare_minimum_preset(self, config_file, output_options, download_value):
@ -129,7 +126,7 @@ class TestPreset:
): ):
with pytest.raises( with pytest.raises(
StringFormattingVariableNotFoundException, StringFormattingVariableNotFoundException,
match="Variable dne_var does not exist.", match="contains the following variables that do not exist: dne_var",
): ):
_ = Preset( _ = Preset(
config=config_file, config=config_file,
@ -145,7 +142,7 @@ class TestPreset:
): ):
with pytest.raises( with pytest.raises(
StringFormattingVariableNotFoundException, StringFormattingVariableNotFoundException,
match="Variable dne_var does not exist", match="contains the following variables that do not exist: dne_var",
): ):
_ = Preset( _ = Preset(
config=config_file, config=config_file,
@ -161,7 +158,7 @@ class TestPreset:
): ):
with pytest.raises( with pytest.raises(
StringFormattingVariableNotFoundException, StringFormattingVariableNotFoundException,
match="Variable dne_var does not exist", match="contains the following variables that do not exist: dne_var",
): ):
_ = Preset( _ = Preset(
config=config_file, config=config_file,
@ -182,7 +179,7 @@ class TestPreset:
): ):
with pytest.raises( with pytest.raises(
StringFormattingVariableNotFoundException, StringFormattingVariableNotFoundException,
match="Variable dne_var does not exist", match="contains the following variables that do not exist: dne_var",
): ):
_ = Preset( _ = Preset(
config=config_file, config=config_file,
@ -198,6 +195,26 @@ class TestPreset:
}, },
) )
def test_preset_error__dict_override_variable_not_static(
self, config_file, output_options, youtube_video
):
with pytest.raises(
StringFormattingVariableNotFoundException,
match="static formatters must contain variables that "
"have no dependency to entry variables",
):
_ = Preset(
config=config_file,
name="test",
value={
"download": youtube_video,
"output_options": {
"output_directory": "{title}",
"file_name": "{uid}",
},
},
)
def test_preset_with_multi_url__contains_empty_url(self, config_file, output_options): def test_preset_with_multi_url__contains_empty_url(self, config_file, output_options):
_ = Preset( _ = Preset(
config=config_file, config=config_file,

View file

@ -47,8 +47,8 @@ def test_main_exit_code(mock_sys_exit, return_code: int):
main() main()
assert mock_logger_cleanup.call_count == 1 assert mock_logger_cleanup.call_count == 1
assert mock_logger_cleanup.call_args.kwargs["cleanup_error_log"] == ( assert mock_logger_cleanup.call_args.kwargs["has_error"] == (
True if return_code == 0 else False True if return_code != 0 else False
) )
@ -107,6 +107,58 @@ def test_args_after_sub_work(mock_sys_exit, tv_show_config_path):
assert mock_sub.call_count == 1 assert mock_sub.call_count == 1
assert mock_sub.call_args.kwargs["subscription_paths"] == ["subscriptions.yaml"] assert mock_sub.call_args.kwargs["subscription_paths"] == ["subscriptions.yaml"]
assert mock_sub.call_args.kwargs["config"]._name == tv_show_config_path assert mock_sub.call_args.kwargs["config"]._name == tv_show_config_path
assert mock_sub.call_args.kwargs["subscription_matches"] == []
assert Logger._LOGGER_LEVEL == LoggerLevels.VERBOSE
def test_sub_match_arguments_before(mock_sys_exit, tv_show_config_path):
with mock_sys_exit(expected_exit_code=0), patch.object(
sys,
"argv",
[
"ytdl-sub",
"--match",
"testA",
"testB",
"-c",
tv_show_config_path,
"sub",
"--log-level",
"verbose",
],
), patch("ytdl_sub.cli.entrypoint._download_subscriptions_from_yaml_files") as mock_sub:
main()
assert mock_sub.call_count == 1
assert mock_sub.call_args.kwargs["subscription_paths"] == ["subscriptions.yaml"]
assert mock_sub.call_args.kwargs["config"]._name == tv_show_config_path
assert mock_sub.call_args.kwargs["subscription_matches"] == ["testA", "testB"]
assert Logger._LOGGER_LEVEL == LoggerLevels.VERBOSE
def test_sub_match_arguments_after_many(mock_sys_exit, tv_show_config_path):
with mock_sys_exit(expected_exit_code=0), patch.object(
sys,
"argv",
[
"ytdl-sub",
"-c",
tv_show_config_path,
"sub",
"--log-level",
"verbose",
"--match",
"testA",
"--match",
"testB",
],
), patch("ytdl_sub.cli.entrypoint._download_subscriptions_from_yaml_files") as mock_sub:
main()
assert mock_sub.call_count == 1
assert mock_sub.call_args.kwargs["subscription_paths"] == ["subscriptions.yaml"]
assert mock_sub.call_args.kwargs["config"]._name == tv_show_config_path
assert mock_sub.call_args.kwargs["subscription_matches"] == ["testA", "testB"]
assert Logger._LOGGER_LEVEL == LoggerLevels.VERBOSE assert Logger._LOGGER_LEVEL == LoggerLevels.VERBOSE

View file

@ -1,3 +1,4 @@
import pytest
from conftest import assert_logs from conftest import assert_logs
from ytdl_sub.plugins.throttle_protection import logger as throttle_protection_logger from ytdl_sub.plugins.throttle_protection import logger as throttle_protection_logger
@ -57,3 +58,59 @@ class TestThrottleProtectionPlugin:
expected_occurrences=1, expected_occurrences=1,
): ):
_ = subscription.download(dry_run=False) _ = subscription.download(dry_run=False)
@pytest.mark.parametrize(
"disable_value",
[
"",
False,
"{tp_bool_string}",
"{tp_empty_string}",
],
)
def test_disabled(
self,
config,
subscription_name,
output_directory,
mock_download_collection_entries,
disable_value,
):
preset_dict = {
"preset": [
"Kodi Music Videos",
],
"overrides": {
"url": "https://your.name.here",
"music_video_directory": output_directory,
"tp_bool_string": "{ %bool(False) }",
"tp_empty_string": "",
},
"throttle_protection": {
"enable": disable_value,
"sleep_per_download_s": {
"min": 0.01,
"max": 0.01,
},
"sleep_per_subscription_s": {
"min": 0.02,
"max": 0.02,
},
},
}
subscription = Subscription.from_dict(
config=config,
preset_name=subscription_name,
preset_dict=preset_dict,
)
with mock_download_collection_entries(
is_youtube_channel=False, num_urls=1, is_extracted_audio=False
), assert_logs(
logger=throttle_protection_logger,
expected_message="Sleeping between downloads for %0.2f seconds",
log_level="debug",
expected_occurrences=0,
):
_ = subscription.download(dry_run=False)

View file

@ -26,6 +26,14 @@ class TestArrayFunctions:
output = single_variable_output("{%array_at(['a', 'b', 'c'], 1)}") output = single_variable_output("{%array_at(['a', 'b', 'c'], 1)}")
assert output == "b" assert output == "b"
def test_array_at_default(self):
output = single_variable_output("{%array_at(['a', 'b', 'c'], 30, 'd')}")
assert output == "d"
def test_array_at_error(self):
with pytest.raises(FunctionRuntimeException):
single_variable_output("{%array_at(['a', 'b', 'c'], 30)}")
def test_array_flatten(self): def test_array_flatten(self):
output = single_variable_output("{%array_flatten(['a', ['b'], [['c']]])}") output = single_variable_output("{%array_flatten(['a', ['b'], [['c']]])}")
assert output == ["a", "b", "c"] assert output == ["a", "b", "c"]

View file

@ -1,6 +1,10 @@
import re
import pytest import pytest
from unit.script.conftest import single_variable_output from unit.script.conftest import single_variable_output
from ytdl_sub.script.utils.exceptions import FunctionRuntimeException
class TestConditionalFunction: class TestConditionalFunction:
@pytest.mark.parametrize( @pytest.mark.parametrize(
@ -33,3 +37,67 @@ class TestConditionalFunction:
}""" }"""
) )
assert output == "winner" assert output == "winner"
def test_elif_function(self):
output = single_variable_output(
"""{
%elif(
False,
"nope",
False,
"still nope",
True,
"yes",
"default value"
)
}"""
)
assert output == "yes"
def test_elif_function_default_value(self):
output = single_variable_output(
"""{
%elif(
False,
"nope",
False,
"still nope",
False,
"will be default",
"default value"
)
}"""
)
assert output == "default value"
def test_elif_function_errors_lt3(self):
with pytest.raises(
FunctionRuntimeException,
match=re.escape("elif requires at least 3 arguments"),
):
single_variable_output(
"""
{
%elif(
False,
"only two args"
)
}"""
)
def test_elif_function_errors_odd(self):
with pytest.raises(
FunctionRuntimeException,
match=re.escape("elif must have an odd number of arguments"),
):
single_variable_output(
"""
{
%elif(
False,
"1",
False,
"even number args bad"
)
}"""
)

View file

@ -1,3 +1,6 @@
from typing import List
from typing import Optional
import pytest import pytest
from unit.script.conftest import single_variable_output from unit.script.conftest import single_variable_output
@ -114,3 +117,21 @@ class TestNumericFunctions:
def test_contains(self, value, expected_output): def test_contains(self, value, expected_output):
output = single_variable_output(f"{{%contains('a brown dog', '{value}')}}") output = single_variable_output(f"{{%contains('a brown dog', '{value}')}}")
assert output == expected_output assert output == expected_output
@pytest.mark.parametrize(
"input_string, split, max_split, expected_output",
[
("no splits", " | ", None, ["no splits"]),
("one | split", " | ", None, ["one", "split"]),
("max | split | one", " | ", 1, ["max", "split | one"]),
],
)
def test_split(
self, input_string: str, split: str, max_split: Optional[int], expected_output: List[str]
):
if max_split:
output = single_variable_output(f"{{%split('{input_string}', '{split}', {max_split})}}")
else:
output = single_variable_output(f"{{%split('{input_string}', '{split}')}}")
assert output == expected_output

View file

@ -49,7 +49,11 @@ class TestFunction:
@pytest.mark.parametrize( @pytest.mark.parametrize(
"function_str, expected_types, received_types", "function_str, expected_types, received_types",
[ [
("{%array_at({'a': 'dict?'}, 1)}", "array: Array, idx: Integer", "Map, Integer"), (
"{%array_at({'a': 'dict?'}, 1)}",
"array: Array, idx: Integer, default: Optional[AnyArgument]",
"Map, Integer",
),
("{%array_extend('not', 'array')}", "arrays: Array, ...", "String, String"), ("{%array_extend('not', 'array')}", "arrays: Array, ...", "String, String"),
( (
"{%replace('hi mom', 'mom', 'dad', 1, 0)}", "{%replace('hi mom', 'mom', 'dad', 1, 0)}",

View file

@ -111,8 +111,8 @@ class TestLogger:
Logger.cleanup() Logger.cleanup()
assert not os.path.isfile(Logger._DEBUG_LOGGER_FILE.name) assert not os.path.isfile(Logger._DEBUG_LOGGER_FILE.name)
@pytest.mark.parametrize("clean_error_log", [True, False]) @pytest.mark.parametrize("has_error", [True, False])
def test_logger_can_be_cleaned_during_execution(self, clean_error_log: bool): def test_logger_can_be_cleaned_during_execution(self, has_error: bool):
Logger._LOGGER_LEVEL = LoggerLevels.INFO Logger._LOGGER_LEVEL = LoggerLevels.INFO
logger = Logger.get(name="name_test") logger = Logger.get(name="name_test")
@ -133,11 +133,11 @@ class TestLogger:
except ValueError as exc: except ValueError as exc:
Logger.log_exception(exception=exc) Logger.log_exception(exception=exc)
Logger.cleanup(cleanup_error_log=clean_error_log) Logger.cleanup(has_error=has_error)
assert not os.path.isfile(Logger.debug_log_filename()) assert not os.path.isfile(Logger.debug_log_filename())
assert clean_error_log == (not os.path.isfile(Logger.error_log_filename())) assert not has_error == (not os.path.isfile(Logger.error_log_filename()))
if not clean_error_log: if has_error:
with open(Logger.error_log_filename(), mode="r", encoding="utf-8") as err_file: with open(Logger.error_log_filename(), mode="r", encoding="utf-8") as err_file:
err_logs = err_file.readlines() err_logs = err_file.readlines()
expected = [ expected = [

View file

@ -1,5 +1,6 @@
import copy import copy
import pytest
from unit.script.conftest import single_variable_output from unit.script.conftest import single_variable_output
from ytdl_sub.utils.script import ScriptUtils from ytdl_sub.utils.script import ScriptUtils
@ -31,3 +32,22 @@ class TestScriptUtils:
output = single_variable_output(ScriptUtils.to_script(json_dict)) output = single_variable_output(ScriptUtils.to_script(json_dict))
assert output == expected_output assert output == expected_output
@pytest.mark.parametrize(
"input_str, expected_output",
[
("", False),
("true", True),
("false", False),
("[ ]", False),
("{ }", False),
("True", True),
("False", False),
("lol not False", True),
("0", False),
("-1", True),
("1", True),
],
)
def test_bool_formatter_output(self, input_str: str, expected_output: bool):
assert ScriptUtils.bool_formatter_output(input_str) == expected_output