diff --git a/docs/source/config_reference/plugins.rst b/docs/source/config_reference/plugins.rst index 69512e8f..6aa05162 100644 --- a/docs/source/config_reference/plugins.rst +++ b/docs/source/config_reference/plugins.rst @@ -22,6 +22,15 @@ Extracts audio from a video file. opus, vorbis, wav, and best to grab the best possible format at runtime. +``enable`` + +:expected type: Optional[OverridesFormatter] +:description: + Can typically be left undefined to always default to enable. For preset convenience, + this field can be set using an override variable to easily toggle whether this plugin + is enabled or not via Boolean. + + ``quality`` :expected type: Float @@ -76,6 +85,15 @@ chapters and remove specific ones. Can also remove chapters using regex. Defaults to True. Embed chapters into the file. +``enable`` + +:expected type: Optional[OverridesFormatter] +:description: + Can typically be left undefined to always default to enable. For preset convenience, + this field can be set using an override variable to easily toggle whether this plugin + is enabled or not via Boolean. + + ``force_key_frames`` :expected type: Optional[Boolean] @@ -148,6 +166,23 @@ granularity possible. Only download videos before this datetime. +``breaks`` + +:expected type: Optional[OverridesFormatter] +:description: + Toggle to enable breaking subsequent metadata downloads if an entry's upload date + is out of range. Defaults to True. + + +``enable`` + +:expected type: Optional[OverridesFormatter] +:description: + Can typically be left undefined to always default to enable. For preset convenience, + this field can be set using an override variable to easily toggle whether this plugin + is enabled or not via Boolean. + + ---------------------------------------------------------------------------------------------------- download @@ -259,6 +294,15 @@ Also supports custom ffmpeg conversions: with ``ffmpeg_post_process_args``. Defaults to ``yt-dlp``. +``enable`` + +:expected type: Optional[OverridesFormatter] +:description: + Can typically be left undefined to always default to enable. For preset convenience, + this field can be set using an override variable to easily toggle whether this plugin + is enabled or not via Boolean. + + ``ffmpeg_post_process_args`` :expected type: Optional[OverridesFormatter] @@ -395,6 +439,15 @@ with a ``.nfo`` extension. You can add any values into the NFO. episode: "{upload_month}{upload_day_padded}" kodi_safe: False +``enable`` + +:expected type: Optional[OverridesFormatter] +:description: + Can typically be left undefined to always default to enable. For preset convenience, + this field can be set using an override variable to easily toggle whether this plugin + is enabled or not via Boolean. + + ``kodi_safe`` :expected type: Optional[Boolean] @@ -484,6 +537,15 @@ Usage: # optional kodi_safe: False +``enable`` + +:expected type: Optional[OverridesFormatter] +:description: + Can typically be left undefined to always default to enable. For preset convenience, + this field can be set using an override variable to easily toggle whether this plugin + is enabled or not via Boolean. + + ``kodi_safe`` :expected type: Optional[Boolean] @@ -800,6 +862,15 @@ and using ``title_and_description`` can regex match/exclude from either ``title` - "{upload_month}" - "{upload_day}" +``enable`` + +:expected type: Optional[OverridesFormatter] +:description: + Can typically be left undefined to always default to enable. For preset convenience, + this field can be set using an override variable to easily toggle whether this plugin + is enabled or not via Boolean. + + ``skip_if_match_fails`` :expected type: Optional[Boolean] @@ -880,6 +951,15 @@ It will set the respective language to the correct subtitle file. webm files can only embed "vtt" subtitle types. +``enable`` + +:expected type: Optional[OverridesFormatter] +:description: + Can typically be left undefined to always default to enable. For preset convenience, + this field can be set using an override variable to easily toggle whether this plugin + is enabled or not via Boolean. + + ``languages`` :expected type: Optional[List[String]] @@ -931,6 +1011,15 @@ scripted. max: 36 subscription_download_probability: 1.0 +``enable`` + +:expected type: Optional[OverridesFormatter] +:description: + Can typically be left undefined to always default to enable. For preset convenience, + this field can be set using an override variable to easily toggle whether this plugin + is enabled or not via Boolean. + + ``max_downloads_per_subscription`` :expected type: Optional[Range] @@ -998,9 +1087,6 @@ for more details. # Stop downloading additional metadata/videos if it # exists in your download archive break_on_existing: True - # Stop downloading additional metadata/videos if it - # is out of your date range - break_on_reject: True # Path to your YouTube cookies file to download 18+ restricted content cookiefile: "/path/to/cookies/file.txt" # Only download this number of videos/audio diff --git a/docs/source/config_reference/scripting/scripting_functions.rst b/docs/source/config_reference/scripting/scripting_functions.rst index c271669e..1e62d137 100644 --- a/docs/source/config_reference/scripting/scripting_functions.rst +++ b/docs/source/config_reference/scripting/scripting_functions.rst @@ -38,10 +38,11 @@ array_apply_fixed array_at ~~~~~~~~ -:spec: ``array_at(array: Array, idx: Integer) -> AnyArgument`` +:spec: ``array_at(array: Array, idx: Integer, default: Optional[AnyArgument]) -> AnyArgument`` :description: - Return the element in the Array at index ``idx``. + Return the element in the Array at index ``idx``. If ``idx`` exceeds the array length, + either return ``default`` if provided or throw an error. array_contains ~~~~~~~~~~~~~~ @@ -225,6 +226,27 @@ xor Conditional Functions --------------------- +elif +~~~~ +:spec: ``elif(if_elif_else: AnyArgument, ...) -> AnyArgument`` + +:description: + Conditional ``if`` statement that is capable of doing else-ifs (``elif``) via + adjacent arguments. It is expected for there to be an odd number of arguments >= 3 to + supply at least one conditional and an else. +:usage: + + .. code-block:: python + + %elif( + condition1, + return1, + condition2, + return2, + ... + else_return + ) + if ~~ :spec: ``if(condition: Boolean, true: ReturnableArgumentA, false: ReturnableArgumentB) -> Union[ReturnableArgumentA, ReturnableArgumentB]`` @@ -543,6 +565,13 @@ slice :description: Returns the slice of the Array. +split +~~~~~ +:spec: ``split(string: String, sep: String, max_split: Optional[Integer]) -> Array`` + +:description: + Splits the input string into multiple strings. + string ~~~~~~ :spec: ``string(value: AnyArgument) -> String`` diff --git a/docs/source/requirements.txt b/docs/source/requirements.txt index c8203d0f..4e9484fa 100644 --- a/docs/source/requirements.txt +++ b/docs/source/requirements.txt @@ -1,3 +1,3 @@ -sphinx-book-theme==1.0.1 +sphinx-book-theme==1.1.0 sphinx-copybutton==0.5.2 sphinx-design==0.5.0 \ No newline at end of file diff --git a/docs/source/usage.rst b/docs/source/usage.rst index 0a9f1d3e..b9865cce 100644 --- a/docs/source/usage.rst +++ b/docs/source/usage.rst @@ -25,6 +25,8 @@ General options must be specified before the command (i.e. ``sub``). path to store the transaction log output of all files added, modified, deleted -st, --suppress-transaction-log do not output transaction logs to console or file + -m MATCH [MATCH ...], --match MATCH [MATCH ...] + match subscription names to one or more substrings, and only run those subscriptions Sub Options ----------- @@ -37,6 +39,14 @@ Download all subscriptions specified in each ``SUBPATH``. ``SUBPATH`` is one or more paths to subscription files, uses ``subscriptions.yaml`` if not provided. It will use the config specified by ``--config``, or ``config.yaml`` if not provided. +.. code-block:: text + :caption: Additional Options + + -u, --update-with-info-json + update all subscriptions with the current config using info.json files + -o DL_OVERRIDE, --dl-override DL_OVERRIDE + override all subscription config values using `dl` syntax, i.e. --dl-override='--ytdl_options.max_downloads 3' + Download Options ----------------- Download a single subscription in the form of CLI arguments. @@ -67,7 +77,7 @@ Using the command: --overrides.url: "https://www.youtube.com/channel/UCuAXFkgsw1L7xaCfnd5JJOw" See how to shorten commands using -`download aliases `_. +`download aliases `_. View Options ----------------- @@ -76,6 +86,7 @@ View Options ytdl-sub view [-sc] [URL] .. code-block:: text + :caption: Additional Options -sc, --split-chapters View source variables after splitting by chapters diff --git a/setup.cfg b/setup.cfg index 5692d3f1..2aa2a99e 100644 --- a/setup.cfg +++ b/setup.cfg @@ -50,8 +50,9 @@ lint = isort==5.10.1 pylint==2.13.5 docs = - sphinx==4.5.0 - sphinx-rtd-theme==1.0.0 + sphinx==7.2.6 + sphinx-rtd-theme==2.0.0 + sphinx-book-theme==1.1.0 build = build twine diff --git a/src/ytdl_sub/cli/entrypoint.py b/src/ytdl_sub/cli/entrypoint.py index bc7fc04e..86a98e57 100644 --- a/src/ytdl_sub/cli/entrypoint.py +++ b/src/ytdl_sub/cli/entrypoint.py @@ -3,6 +3,7 @@ import os import sys from datetime import datetime from pathlib import Path +from typing import Dict from typing import List from typing import Optional @@ -67,7 +68,12 @@ def _maybe_write_subscription_log_file( def _download_subscriptions_from_yaml_files( - config: ConfigFile, subscription_paths: List[str], update_with_info_json: bool, dry_run: bool + config: ConfigFile, + subscription_paths: List[str], + subscription_matches: List[str], + subscription_override_dict: Dict, + update_with_info_json: bool, + dry_run: bool, ) -> List[Subscription]: """ Downloads all subscriptions from one or many subscription yaml files. @@ -78,6 +84,8 @@ def _download_subscriptions_from_yaml_files( Configuration file subscription_paths Path to subscription files to download + subscription_matches + Optional list of substrings to match subscription names to (only run if matched) update_with_info_json Whether to actually download or update using existing info json dry_run @@ -96,7 +104,12 @@ def _download_subscriptions_from_yaml_files( # Load all the subscriptions first to perform all validation before downloading for path in subscription_paths: - subscriptions += Subscription.from_file_path(config=config, subscription_path=path) + subscriptions += Subscription.from_file_path( + config=config, + subscription_path=path, + subscription_matches=subscription_matches, + subscription_override_dict=subscription_override_dict, + ) for subscription in subscriptions: with subscription.exception_handling(): @@ -119,7 +132,7 @@ def _download_subscriptions_from_yaml_files( exception=subscription.exception, ) - Logger.cleanup(cleanup_error_log=False) + Logger.cleanup(has_error=False) gc.collect() # Garbage collect after each subscription download return subscriptions @@ -221,10 +234,18 @@ def main() -> List[Subscription]: "full backup before usage. You have been warned!", ) + subscription_override_dict = {} + if args.dl_override: + subscription_override_dict = DownloadArgsParser.from_dl_override( + override=args.dl_override, config=config + ).to_subscription_dict() + logger.info("Validating subscriptions...") subscriptions = _download_subscriptions_from_yaml_files( config=config, subscription_paths=args.subscription_paths, + subscription_matches=args.match, + subscription_override_dict=subscription_override_dict, update_with_info_json=args.update_with_info_json, dry_run=args.dry_run, ) diff --git a/src/ytdl_sub/cli/parsers/dl.py b/src/ytdl_sub/cli/parsers/dl.py index a3ada3b3..d9e1eb1a 100644 --- a/src/ytdl_sub/cli/parsers/dl.py +++ b/src/ytdl_sub/cli/parsers/dl.py @@ -9,6 +9,7 @@ from typing import Tuple from mergedeep import mergedeep from ytdl_sub.cli.parsers.main import MainArguments +from ytdl_sub.config.config_file import ConfigFile from ytdl_sub.config.config_validator import ConfigOptions from ytdl_sub.utils.exceptions import InvalidDlArguments @@ -247,3 +248,12 @@ class DownloadArgsParser: """ hash_string = str(sorted(self._unknown_arguments)) return hashlib.sha256(hash_string.encode()).hexdigest()[-8:] + + @classmethod + def from_dl_override(cls, override: str, config: ConfigFile) -> "DownloadArgsParser": + """ + Create a DownloadArgsParser from a sub --override argument value + """ + return DownloadArgsParser( + extra_arguments=override.split(), config_options=config.config_options + ) diff --git a/src/ytdl_sub/cli/parsers/main.py b/src/ytdl_sub/cli/parsers/main.py index ccb50734..99f6c231 100644 --- a/src/ytdl_sub/cli/parsers/main.py +++ b/src/ytdl_sub/cli/parsers/main.py @@ -40,6 +40,10 @@ class MainArguments: long="--suppress-transaction-log", is_positional=True, ) + MATCH = CLIArgument( + short="-m", + long="--match", + ) @classmethod def all(cls) -> List[CLIArgument]: @@ -54,6 +58,7 @@ class MainArguments: cls.LOG_LEVEL, cls.TRANSACTION_LOG, cls.SUPPRESS_TRANSACTION_LOG, + cls.MATCH, ] @classmethod @@ -124,6 +129,16 @@ def _add_shared_arguments(arg_parser: argparse.ArgumentParser, suppress_defaults help="do not output transaction logs to console or file", default=argparse.SUPPRESS if suppress_defaults else False, ) + arg_parser.add_argument( + MainArguments.MATCH.short, + MainArguments.MATCH.long, + dest="match", + nargs="+", + action="extend", + type=str, + help="match subscription names to one or more substrings, and only run those subscriptions", + default=argparse.SUPPRESS if suppress_defaults else [], + ) ################################################################################################### @@ -142,6 +157,10 @@ class SubArguments: short="-u", long="--update-with-info-json", ) + OVERRIDE = CLIArgument( + short="-o", + long="--dl-override", + ) subscription_parser = subparsers.add_parser("sub") @@ -160,6 +179,13 @@ subscription_parser.add_argument( help="update all subscriptions with the current config using info.json files", default=False, ) +subscription_parser.add_argument( + SubArguments.OVERRIDE.short, + SubArguments.OVERRIDE.long, + type=str, + help="override all subscription config values using `dl` syntax, " + "i.e. --dl-override='--ytdl_options.max_downloads 3'", +) ################################################################################################### # DOWNLOAD PARSER diff --git a/src/ytdl_sub/config/plugin/plugin.py b/src/ytdl_sub/config/plugin/plugin.py index 24d616f6..33e98a1b 100644 --- a/src/ytdl_sub/config/plugin/plugin.py +++ b/src/ytdl_sub/config/plugin/plugin.py @@ -1,5 +1,6 @@ from abc import ABC from abc import abstractmethod +from functools import cached_property from typing import Dict from typing import Generic from typing import List @@ -8,9 +9,11 @@ from typing import Tuple from typing import Type from ytdl_sub.config.overrides import Overrides +from ytdl_sub.config.validators.options import ToggleableOptionsDictValidator from ytdl_sub.config.validators.options import TOptionsValidator from ytdl_sub.entries.entry import Entry from ytdl_sub.utils.file_handler import FileMetadata +from ytdl_sub.utils.script import ScriptUtils from ytdl_sub.ytdl_additions.enhanced_download_archive import DownloadArchiver from ytdl_sub.ytdl_additions.enhanced_download_archive import EnhancedDownloadArchive @@ -40,6 +43,17 @@ class Plugin(BasePlugin[TOptionsValidator], Generic[TOptionsValidator], ABC): Class to define the new plugin functionality """ + @cached_property + def is_enabled(self) -> bool: + """ + Returns True if enabled, False if disabled. + """ + if isinstance(self.plugin_options, ToggleableOptionsDictValidator): + return ScriptUtils.bool_formatter_output( + self.overrides.apply_formatter(self.plugin_options.enable) + ) + return True + def ytdl_options_match_filters(self) -> Tuple[List[str], List[str]]: """ Returns diff --git a/src/ytdl_sub/config/preset_options.py b/src/ytdl_sub/config/preset_options.py index 8cc4697c..a6737a64 100644 --- a/src/ytdl_sub/config/preset_options.py +++ b/src/ytdl_sub/config/preset_options.py @@ -32,9 +32,6 @@ class YTDLOptions(LiteralDictValidator): # Stop downloading additional metadata/videos if it # exists in your download archive break_on_existing: True - # Stop downloading additional metadata/videos if it - # is out of your date range - break_on_reject: True # Path to your YouTube cookies file to download 18+ restricted content cookiefile: "/path/to/cookies/file.txt" # Only download this number of videos/audio diff --git a/src/ytdl_sub/config/validators/options.py b/src/ytdl_sub/config/validators/options.py index c35defa2..324b39e1 100644 --- a/src/ytdl_sub/config/validators/options.py +++ b/src/ytdl_sub/config/validators/options.py @@ -6,6 +6,7 @@ from typing import TypeVar from ytdl_sub.config.plugin.plugin_operation import PluginOperation from ytdl_sub.utils.exceptions import ValidationException from ytdl_sub.validators.strict_dict_validator import StrictDictValidator +from ytdl_sub.validators.string_formatter_validators import OverridesBooleanFormatterValidator from ytdl_sub.validators.validators import Validator # pylint: disable=no-self-use @@ -57,3 +58,28 @@ TOptionsValidator = TypeVar("TOptionsValidator", bound=OptionsValidator) class OptionsDictValidator(StrictDictValidator, OptionsValidator, ABC): pass + + +class ToggleableOptionsDictValidator(OptionsDictValidator): + _optional_keys = {"enable"} + + def __init__(self, name, value): + assert ( + "enable" in self._optional_keys + ), f"{self.__class__.__name__} does not have enable as an optional field" + super().__init__(name, value) + + self._enable = self._validate_key( + key="enable", validator=OverridesBooleanFormatterValidator, default="True" + ) + + @property + def enable(self) -> OverridesBooleanFormatterValidator: + """ + :expected type: Optional[OverridesFormatter] + :description: + Can typically be left undefined to always default to enable. For preset convenience, + this field can be set using an override variable to easily toggle whether this plugin + is enabled or not via Boolean. + """ + return self._enable diff --git a/src/ytdl_sub/config/validators/variable_validation.py b/src/ytdl_sub/config/validators/variable_validation.py index cc554a70..f5de3ed5 100644 --- a/src/ytdl_sub/config/validators/variable_validation.py +++ b/src/ytdl_sub/config/validators/variable_validation.py @@ -13,9 +13,10 @@ from ytdl_sub.config.plugin.preset_plugins import PresetPlugins from ytdl_sub.config.preset_options import OutputOptions from ytdl_sub.config.validators.options import OptionsValidator from ytdl_sub.downloaders.url.validators import MultiUrlValidator -from ytdl_sub.entries.script.variable_definitions import VARIABLE_SCRIPTS from ytdl_sub.entries.variables.override_variables import SubscriptionVariables from ytdl_sub.script.script import Script +from ytdl_sub.utils.scriptable import BASE_SCRIPT +from ytdl_sub.validators.string_formatter_validators import to_variable_dependency_format_string from ytdl_sub.validators.string_formatter_validators import validate_formatters @@ -32,10 +33,10 @@ def _add_dummy_overrides(overrides: Overrides) -> Dict[str, str]: # Have the dummy override variable contain all variable deps that it uses in the string dummy_overrides: Dict[str, str] = {} for override_name in _override_variables(overrides): - dummy_overrides[override_name] = "" # pylint: disable=protected-access - for variable_dependency in overrides.script._variables[override_name].variables: - dummy_overrides[override_name] += f"{{ {variable_dependency.name } }}" + dummy_overrides[override_name] = to_variable_dependency_format_string( + script=overrides.script, parsed_format_string=overrides.script._variables[override_name] + ) # pylint: enable=protected-access return dummy_overrides @@ -72,8 +73,15 @@ def _override_variables(overrides: Overrides) -> Set[str]: } -def _entry_variables() -> Set[str]: - return set(list(VARIABLE_SCRIPTS.keys())) +_DUMMY_ENTRY_VARIABLES: Dict[str, str] = { + name: to_variable_dependency_format_string( + # pylint: disable=protected-access + script=BASE_SCRIPT, + parsed_format_string=BASE_SCRIPT._variables[name] + # pylint: enable=protected-access + ) + for name in BASE_SCRIPT.variable_names +} class VariableValidation: @@ -97,12 +105,12 @@ class VariableValidation: """ Do some gymnastics to initialize the Overrides script. """ - entry_variables = _entry_variables() override_variables = _override_variables(overrides) # Set resolved variables as all entry + override variables # at this point to generate every possible added/modified variable - self.resolved_variables = entry_variables | override_variables + self.resolved_variables = set(_DUMMY_ENTRY_VARIABLES.keys()) | override_variables + plugin_variables: Set[str] = set() for ( plugin_options, @@ -125,6 +133,7 @@ class VariableValidation: # Set unresolved as variables that are added but do not exist as # entry/override variables since they are created at run-time self.unresolved_variables |= added_variables | modified_variables + plugin_variables |= added_variables | modified_variables # Then update resolved variables to reflect that self.resolved_variables -= self.unresolved_variables @@ -137,10 +146,11 @@ class VariableValidation: ) # copy the script and mock entry variables - self.script = copy.deepcopy(overrides.script).add(_add_dummy_variables(entry_variables)) + self.script = copy.deepcopy(overrides.script) self.script.add( - variables=_add_dummy_overrides(overrides=overrides), - unresolvable=self.unresolved_variables, + variables=_add_dummy_overrides(overrides=overrides) + | _add_dummy_variables(variables=plugin_variables) + | _DUMMY_ENTRY_VARIABLES ) return self @@ -158,7 +168,6 @@ class VariableValidation: resolved_variables = added_variables | modified_variables - self.script.add(_add_dummy_variables(resolved_variables)) self.resolved_variables |= resolved_variables self.unresolved_variables -= resolved_variables @@ -177,13 +186,10 @@ class VariableValidation: ): self._add_variables(PluginOperation.MODIFY_ENTRY_METADATA, options=plugin_options) - self._update_script() for plugin_options in PluginMapping.order_options_by( self.plugins.zipped(), PluginOperation.MODIFY_ENTRY ): - added = self._add_variables(PluginOperation.MODIFY_ENTRY, options=plugin_options) - if added: - self._update_script() + self._add_variables(PluginOperation.MODIFY_ENTRY, options=plugin_options) # Validate that any formatter in the plugin options can resolve validate_formatters( diff --git a/src/ytdl_sub/downloaders/url/downloader.py b/src/ytdl_sub/downloaders/url/downloader.py index b1be016b..a0ceb535 100644 --- a/src/ytdl_sub/downloaders/url/downloader.py +++ b/src/ytdl_sub/downloaders/url/downloader.py @@ -112,13 +112,6 @@ class UrlDownloaderThumbnailPlugin(SourcePluginExtension): directory, run this function. This lets the downloader add any extra files directly to the output directory, for things like YT channel image, banner. """ - if playlist_metadata := entry.get(v.playlist_metadata, dict): - self._download_parent_thumbnails( - thumbnail_list_info=collection_url.playlist_thumbnails, - entry=entry, - parent=EntryParent(playlist_metadata, working_directory=self.working_directory), - ) - if source_metadata := entry.get(v.source_metadata, dict): self._download_parent_thumbnails( thumbnail_list_info=collection_url.source_thumbnails, @@ -126,6 +119,13 @@ class UrlDownloaderThumbnailPlugin(SourcePluginExtension): parent=EntryParent(source_metadata, working_directory=self.working_directory), ) + if playlist_metadata := entry.get(v.playlist_metadata, dict): + self._download_parent_thumbnails( + thumbnail_list_info=collection_url.playlist_thumbnails, + entry=entry, + parent=EntryParent(playlist_metadata, working_directory=self.working_directory), + ) + def modify_entry(self, entry: Entry) -> Optional[Entry]: """ Use the entry to download thumbnails (or move if LATEST_ENTRY). diff --git a/src/ytdl_sub/downloaders/url/validators.py b/src/ytdl_sub/downloaders/url/validators.py index 631ee979..53330306 100644 --- a/src/ytdl_sub/downloaders/url/validators.py +++ b/src/ytdl_sub/downloaders/url/validators.py @@ -1,4 +1,3 @@ -import copy from typing import Any from typing import Dict from typing import Optional @@ -276,17 +275,11 @@ class MultiUrlValidator(OptionsValidator): def __init__(self, name, value): super().__init__(name, value) - # Copy since we're popping things - value_copy = copy.deepcopy(value) - if isinstance(value, dict): - # Pop old required field in case it's still there - value_copy.pop("download_strategy", None) - # Deal with old multi-url download strategy - if isinstance(value, dict) and "urls" in value_copy: - self._urls = UrlListValidator(name=name, value=value_copy["urls"]) + if isinstance(value, dict) and "urls" in value: + self._urls = UrlListValidator(name=name, value=value["urls"]) else: - self._urls = UrlListValidator(name=name, value=value_copy) + self._urls = UrlListValidator(name=name, value=value) @property def urls(self) -> UrlListValidator: diff --git a/src/ytdl_sub/downloaders/ytdlp.py b/src/ytdl_sub/downloaders/ytdlp.py index a7912232..c0ac01e0 100644 --- a/src/ytdl_sub/downloaders/ytdlp.py +++ b/src/ytdl_sub/downloaders/ytdlp.py @@ -102,7 +102,7 @@ class YTDLP: ) if is_downloaded and is_thumbnail_downloaded: - return entry_dict + return entry_dict or {} # in-case yt-dlp returns None # Always add check_formats # See https://github.com/yt-dlp/yt-dlp/issues/502 @@ -220,11 +220,11 @@ class YTDLP: with cls._listen_and_log_downloaded_info_json( working_directory=working_directory, log_prefix=log_prefix_on_info_json_dl ): - _ = cls.extract_info(ytdl_options_overrides=ytdl_options_overrides, **kwargs) + cls.extract_info(ytdl_options_overrides=ytdl_options_overrides, **kwargs) except RejectedVideoReached: cls.logger.debug( "RejectedVideoReached, stopping additional downloads " - "(Can be disable by setting `ytdl_options.break_on_reject` to False)." + "(Can be disable by setting `date_range.breaking` to False)." ) except ExistingVideoReached: cls.logger.debug( @@ -234,4 +234,34 @@ class YTDLP: except MaxDownloadsReached: cls.logger.info("MaxDownloadsReached, stopping additional downloads.") - return cls._get_entry_dicts_from_info_json_files(working_directory=working_directory) + parent_dicts: List[Dict] = [] + entry_dicts = cls._get_entry_dicts_from_info_json_files(working_directory=working_directory) + entry_ids = {entry_dict.get("id") for entry_dict in entry_dicts} + + # Try to get additional uploader (source) metadata that yt-dlp does not fetch + # in a single request + for entry_dict in entry_dicts: + if not (uploader_id := entry_dict.get("uploader_id")): + continue + + if uploader_id in entry_ids or not (uploader_url := entry_dict.get("uploader_url")): + continue + + cls.logger.debug("Attempting to get parent metadata from URL %s", uploader_url) + try: + parent_dict = cls.extract_info( + ytdl_options_overrides=ytdl_options_overrides | {"playlist_items": "0:0"}, + url=uploader_url, + ) + except Exception: # pylint: disable=broad-except + # Do not try this uploader_id again + entry_ids.add(uploader_id) + break + + if isinstance(parent_dict, dict): + parent_id = parent_dict.get("id") + parent_dicts.append(parent_dict) + entry_ids |= {uploader_id, parent_id} + cls.logger.debug("Adding parent metadata with ids [%s, %s]", uploader_id, parent_id) + + return entry_dicts + parent_dicts diff --git a/src/ytdl_sub/entries/entry_parent.py b/src/ytdl_sub/entries/entry_parent.py index 6cf7dcb1..232b2ab4 100644 --- a/src/ytdl_sub/entries/entry_parent.py +++ b/src/ytdl_sub/entries/entry_parent.py @@ -157,13 +157,16 @@ class EntryParent(BaseEntry): def _uid_is_uploader_id(parent: "EntryParent"): return parent.uid == parent.uploader_id - top_level_parents = [ - parent for parent in parents if parent.num_children() == 0 and _url_matches(parent) - ] + top_level_parents = [parent for parent in parents if parent.num_children() == 0] # If more than 1 parent exists, assume the uploader_id is the root parent if len(top_level_parents) > 1: - top_level_parents = [parent for parent in parents if _uid_is_uploader_id(parent)] + top_level_parents = [ + parent for parent in top_level_parents if _uid_is_uploader_id(parent) + ] + + if len(top_level_parents) > 1: + top_level_parents = [parent for parent in top_level_parents if _url_matches(parent)] match len(top_level_parents): case 0: diff --git a/src/ytdl_sub/main.py b/src/ytdl_sub/main.py index b4e7e776..44258a5c 100644 --- a/src/ytdl_sub/main.py +++ b/src/ytdl_sub/main.py @@ -27,7 +27,7 @@ def main(): """ try: return_code = _main() - Logger.cleanup(cleanup_error_log=return_code == 0) + Logger.cleanup(has_error=return_code != 0) sys.exit(return_code) except Exception as exc: # pylint: disable=broad-except Logger.log_exception(exception=exc) diff --git a/src/ytdl_sub/plugins/audio_extract.py b/src/ytdl_sub/plugins/audio_extract.py index 644fa1c1..2fbd1d6b 100644 --- a/src/ytdl_sub/plugins/audio_extract.py +++ b/src/ytdl_sub/plugins/audio_extract.py @@ -6,7 +6,7 @@ from typing import Set from ytdl_sub.config.plugin.plugin import Plugin from ytdl_sub.config.plugin.plugin_operation import PluginOperation -from ytdl_sub.config.validators.options import OptionsDictValidator +from ytdl_sub.config.validators.options import ToggleableOptionsDictValidator from ytdl_sub.downloaders.ytdl_options_builder import YTDLOptionsBuilder from ytdl_sub.entries.entry import Entry from ytdl_sub.entries.script.variable_definitions import VARIABLES @@ -21,7 +21,7 @@ from ytdl_sub.validators.validators import FloatValidator v: VariableDefinitions = VARIABLES -class AudioExtractOptions(OptionsDictValidator): +class AudioExtractOptions(ToggleableOptionsDictValidator): """ Extracts audio from a video file. @@ -35,7 +35,7 @@ class AudioExtractOptions(OptionsDictValidator): """ _required_keys = {"codec"} - _optional_keys = {"quality"} + _optional_keys = {"enable", "quality"} @classmethod def partial_validate(cls, name: str, value: Any) -> None: @@ -102,6 +102,8 @@ class AudioExtractPlugin(Plugin[AudioExtractOptions]): return ytdl_options_builder.add( { "postprocessors": [postprocessor_dict], + "format": "bestaudio/best", + "keepvideo": False, } ).to_dict() diff --git a/src/ytdl_sub/plugins/chapters.py b/src/ytdl_sub/plugins/chapters.py index fbd11c09..198f91b5 100644 --- a/src/ytdl_sub/plugins/chapters.py +++ b/src/ytdl_sub/plugins/chapters.py @@ -7,7 +7,7 @@ from typing import Set from ytdl_sub.config.plugin.plugin import Plugin from ytdl_sub.config.plugin.plugin_operation import PluginOperation -from ytdl_sub.config.validators.options import OptionsDictValidator +from ytdl_sub.config.validators.options import ToggleableOptionsDictValidator from ytdl_sub.downloaders.ytdl_options_builder import YTDLOptionsBuilder from ytdl_sub.entries.entry import Entry from ytdl_sub.entries.entry import ytdl_sub_chapters_from_comments @@ -59,7 +59,7 @@ class SponsorBlockCategoryListValidator(ListValidator[SponsorBlockCategoriesVali _inner_list_type = SponsorBlockCategoriesValidator -class ChaptersOptions(OptionsDictValidator): +class ChaptersOptions(ToggleableOptionsDictValidator): """ Embeds chapters to video files if they are present. Additional options to add SponsorBlock chapters and remove specific ones. Can also remove chapters using regex. @@ -90,6 +90,7 @@ class ChaptersOptions(OptionsDictValidator): """ _optional_keys = { + "enable", "embed_chapters", "allow_chapters_from_comments", "sponsorblock_categories", diff --git a/src/ytdl_sub/plugins/date_range.py b/src/ytdl_sub/plugins/date_range.py index 3948cb1e..e01ce1ff 100644 --- a/src/ytdl_sub/plugins/date_range.py +++ b/src/ytdl_sub/plugins/date_range.py @@ -3,12 +3,14 @@ from typing import Optional from typing import Tuple from ytdl_sub.config.plugin.plugin import Plugin -from ytdl_sub.config.validators.options import OptionsDictValidator +from ytdl_sub.config.validators.options import ToggleableOptionsDictValidator from ytdl_sub.utils.datetime import to_date_str +from ytdl_sub.utils.script import ScriptUtils from ytdl_sub.validators.string_datetime import StringDatetimeValidator +from ytdl_sub.validators.string_formatter_validators import OverridesBooleanFormatterValidator -class DateRangeOptions(OptionsDictValidator): +class DateRangeOptions(ToggleableOptionsDictValidator): """ Only download files uploaded within the specified date range. Dates must adhere to a yt-dlp datetime. From their docs: @@ -31,12 +33,15 @@ class DateRangeOptions(OptionsDictValidator): after: "today-2weeks" """ - _optional_keys = {"before", "after"} + _optional_keys = {"enable", "before", "after", "breaks"} def __init__(self, name, value): super().__init__(name, value) self._before = self._validate_key_if_present("before", StringDatetimeValidator) self._after = self._validate_key_if_present("after", StringDatetimeValidator) + self._breaks = self._validate_key_if_present( + "breaks", OverridesBooleanFormatterValidator, default="True" + ) @property def before(self) -> Optional[StringDatetimeValidator]: @@ -56,6 +61,16 @@ class DateRangeOptions(OptionsDictValidator): """ return self._after + @property + def breaks(self) -> OverridesBooleanFormatterValidator: + """ + :expected type: Optional[OverridesFormatter] + :description: + Toggle to enable breaking subsequent metadata downloads if an entry's upload date + is out of range. Defaults to True. + """ + return self._breaks + class DateRangePlugin(Plugin[DateRangeOptions]): plugin_options_type = DateRangeOptions @@ -79,6 +94,12 @@ class DateRangePlugin(Plugin[DateRangeOptions]): after_str = to_date_str( date_validator=self.plugin_options.after, overrides=self.overrides ) - breaking_match_filters.append(f"upload_date >= {after_str}") + after_filter = f"upload_date >= {after_str}" + if ScriptUtils.bool_formatter_output( + self.overrides.apply_formatter(self.plugin_options.breaks) + ): + breaking_match_filters.append(after_filter) + else: + match_filters.append(after_filter) return match_filters, breaking_match_filters diff --git a/src/ytdl_sub/plugins/file_convert.py b/src/ytdl_sub/plugins/file_convert.py index 8fccaedb..00182d69 100644 --- a/src/ytdl_sub/plugins/file_convert.py +++ b/src/ytdl_sub/plugins/file_convert.py @@ -7,7 +7,7 @@ from typing import Set from ytdl_sub.config.overrides import Overrides from ytdl_sub.config.plugin.plugin import Plugin from ytdl_sub.config.plugin.plugin_operation import PluginOperation -from ytdl_sub.config.validators.options import OptionsDictValidator +from ytdl_sub.config.validators.options import ToggleableOptionsDictValidator from ytdl_sub.entries.entry import Entry from ytdl_sub.entries.script.variable_definitions import VARIABLES from ytdl_sub.entries.script.variable_definitions import VariableDefinitions @@ -28,7 +28,7 @@ class FileConvertWithValidator(StringSelectValidator): _select_values = {"yt-dlp", "ffmpeg"} -class FileConvertOptions(OptionsDictValidator): +class FileConvertOptions(ToggleableOptionsDictValidator): """ Converts video files from one extension to another. @@ -56,7 +56,7 @@ class FileConvertOptions(OptionsDictValidator): """ _required_keys = {"convert_to"} - _optional_keys = {"convert_with", "ffmpeg_post_process_args"} + _optional_keys = {"enable", "convert_with", "ffmpeg_post_process_args"} @classmethod def partial_validate(cls, name: str, value: Any) -> None: diff --git a/src/ytdl_sub/plugins/filter_exclude.py b/src/ytdl_sub/plugins/filter_exclude.py index 72ad6c88..08f13939 100644 --- a/src/ytdl_sub/plugins/filter_exclude.py +++ b/src/ytdl_sub/plugins/filter_exclude.py @@ -1,4 +1,3 @@ -import json from typing import Dict from typing import Optional @@ -8,6 +7,7 @@ from ytdl_sub.config.validators.options import OptionsValidator from ytdl_sub.entries.entry import Entry from ytdl_sub.utils.exceptions import StringFormattingException from ytdl_sub.utils.logger import Logger +from ytdl_sub.utils.script import ScriptUtils from ytdl_sub.validators.string_formatter_validators import ListFormatterValidator from ytdl_sub.ytdl_additions.enhanced_download_archive import EnhancedDownloadArchive @@ -53,7 +53,9 @@ class FilterExcludePlugin(Plugin[FilterExcludeOptions]): return entry for formatter in self.plugin_options.list: - out = json.loads(self.overrides.apply_formatter(formatter=formatter, entry=entry)) + out = ScriptUtils.bool_formatter_output( + self.overrides.apply_formatter(formatter=formatter, entry=entry) + ) if bool(out): logger.info( "Filtering '%s' from the filter %s evaluating to True", diff --git a/src/ytdl_sub/plugins/filter_include.py b/src/ytdl_sub/plugins/filter_include.py index 5a54e8ec..5f7fcc4a 100644 --- a/src/ytdl_sub/plugins/filter_include.py +++ b/src/ytdl_sub/plugins/filter_include.py @@ -1,4 +1,3 @@ -import json from typing import Dict from typing import Optional @@ -8,6 +7,7 @@ from ytdl_sub.config.validators.options import OptionsValidator from ytdl_sub.entries.entry import Entry from ytdl_sub.utils.exceptions import StringFormattingException from ytdl_sub.utils.logger import Logger +from ytdl_sub.utils.script import ScriptUtils from ytdl_sub.validators.string_formatter_validators import ListFormatterValidator from ytdl_sub.ytdl_additions.enhanced_download_archive import EnhancedDownloadArchive @@ -61,7 +61,9 @@ class FilterIncludePlugin(Plugin[FilterIncludeOptions]): return entry for formatter in self.plugin_options.list: - out = json.loads(self.overrides.apply_formatter(formatter=formatter, entry=entry)) + out = ScriptUtils.bool_formatter_output( + self.overrides.apply_formatter(formatter=formatter, entry=entry) + ) if not bool(out): logger.info( "Filtering '%s' from the filter %s evaluating to False", diff --git a/src/ytdl_sub/plugins/match_filters.py b/src/ytdl_sub/plugins/match_filters.py index 90211e2c..6a777054 100644 --- a/src/ytdl_sub/plugins/match_filters.py +++ b/src/ytdl_sub/plugins/match_filters.py @@ -4,7 +4,7 @@ from typing import List from typing import Tuple from ytdl_sub.config.plugin.plugin import Plugin -from ytdl_sub.config.validators.options import OptionsDictValidator +from ytdl_sub.config.validators.options import ToggleableOptionsDictValidator from ytdl_sub.utils.logger import Logger from ytdl_sub.validators.validators import StringListValidator @@ -44,19 +44,17 @@ def combine_filters(filters: List[str], to_combine: List[str]) -> List[str]: if not filters: return copy.deepcopy(to_combine) - if len(to_combine) > 1: - raise ValueError("Match-filters to combine only supports 1 at this time") - output_filters: List[str] = [] - filter_to_combine: str = to_combine[0] for match_filter in filters: - output_filters.append(f"{match_filter} & {filter_to_combine}") + output_filters.append(match_filter) + for filter_combine in to_combine: + output_filters[-1] += f" & {filter_combine}" return output_filters -class MatchFiltersOptions(OptionsDictValidator): +class MatchFiltersOptions(ToggleableOptionsDictValidator): """ Set ``--match-filters`` to pass into yt-dlp to filter entries from being downloaded. Uses the same syntax as yt-dlp. An entry will be downloaded if any one of the filters are met. @@ -74,7 +72,7 @@ class MatchFiltersOptions(OptionsDictValidator): # - "availability=?public" """ - _optional_keys = {"filters"} + _optional_keys = {"enable", "filters"} @classmethod def partial_validate(cls, name: str, value: Any) -> None: diff --git a/src/ytdl_sub/plugins/music_tags.py b/src/ytdl_sub/plugins/music_tags.py index 6ad24e37..b9542810 100644 --- a/src/ytdl_sub/plugins/music_tags.py +++ b/src/ytdl_sub/plugins/music_tags.py @@ -1,6 +1,4 @@ -import copy from collections import defaultdict -from typing import Any from typing import Dict from typing import List @@ -14,10 +12,8 @@ from ytdl_sub.entries.script.variable_definitions import VariableDefinitions from ytdl_sub.utils.file_handler import FileMetadata from ytdl_sub.utils.logger import Logger from ytdl_sub.validators.audo_codec_validator import AUDIO_CODEC_EXTS -from ytdl_sub.validators.strict_dict_validator import StrictDictValidator from ytdl_sub.validators.string_formatter_validators import ListFormatterValidator from ytdl_sub.validators.string_formatter_validators import StringFormatterValidator -from ytdl_sub.validators.validators import BoolValidator v: VariableDefinitions = VARIABLES @@ -40,31 +36,6 @@ def _is_multi_field(tag_name: str) -> bool: } -class MusicTagsValidator(StrictDictValidator): - """ - Validator for the music_tag's `tags` field. Treat each value as a list. - Can still specify it like a single value but under-the-hood it's a list of a single element. - """ - - _optional_keys = set(list(mediafile.MediaFile.sorted_fields())) - - def __init__(self, name, value): - super().__init__(name, value) - - self._tags: Dict[str, List[StringFormatterValidator]] = {} - for key in self._keys: - self._tags[key] = self._validate_key(key=key, validator=ListFormatterValidator).list - - @property - def as_lists(self) -> Dict[str, List[StringFormatterValidator]]: - """ - Returns - ------- - Tag formatter(s) as a list - """ - return self._tags - - class MusicTagsOptions(OptionsDictValidator): """ Adds tags to every download audio file using @@ -93,40 +64,24 @@ class MusicTagsOptions(OptionsDictValidator): - "ytdl-sub" """ - _optional_keys = {"tags", "embed_thumbnail"} - _allow_extra_keys = True + _optional_keys = set(list(mediafile.MediaFile.sorted_fields())) def __init__(self, name, value): super().__init__(name, value) - self._embed_thumbnail = self._validate_key_if_present( - key="embed_thumbnail", validator=BoolValidator - ) - - new_tags_dict: Dict[str, Any] = copy.deepcopy(value) - old_tags_dict = new_tags_dict.pop("tags", {}) - new_tags_dict.pop("embed_thumbnail", None) - - self._is_old_format = len(old_tags_dict) > 0 or self._embed_thumbnail is not None - self._tags = MusicTagsValidator(name=name, value=dict(old_tags_dict, **new_tags_dict)) + self._tags: Dict[str, List[StringFormatterValidator]] = {} + for key in self._keys: + self._tags[key] = self._validate_key(key=key, validator=ListFormatterValidator).list @property - def tags(self) -> MusicTagsValidator: + def as_lists(self) -> Dict[str, List[StringFormatterValidator]]: """ - Key, values of tag names, tag values. Supports source and override variables. - Supports lists which will get written to MP3s as id3v2.4 multi-tags. + Returns + ------- + Tag formatter(s) as a list """ return self._tags - @property - def embed_thumbnail(self) -> bool: - """ - Optional. Whether to embed the thumbnail into the audio file. - """ - if self._embed_thumbnail is None: - return False - return self._embed_thumbnail.value - class MusicTagsPlugin(Plugin[MusicTagsOptions]): plugin_options_type = MusicTagsOptions @@ -142,23 +97,9 @@ class MusicTagsPlugin(Plugin[MusicTagsOptions]): f"to audio using the audio_extract plugin." ) - # pylint: disable=protected-access - if self.plugin_options._is_old_format: - logger.warning( - "music_tags.tags is now deprecated. Place your tags directly under music_tags " - "instead. The old format will be removed in October of 2023. See " - "https://ytdl-sub.readthedocs.io/en/latest/deprecation_notices.html#music-tags " - "for more details." - ) - if self.plugin_options.embed_thumbnail: - logger.warning( - "music_tags.embed_thumbnail is also deprecated. Use the dedicated " - "embed_thumbnail plugin instead. This will be removed in October of 2023." - ) - # Resolve the tags into this dict tags_to_write: Dict[str, List[str]] = defaultdict(list) - for tag_name, tag_formatters in self.plugin_options.tags.as_lists.items(): + for tag_name, tag_formatters in self.plugin_options.as_lists.items(): for tag_formatter in tag_formatters: tag_value = self.overrides.apply_formatter(formatter=tag_formatter, entry=entry) tags_to_write[tag_name].append(tag_value) @@ -180,19 +121,10 @@ class MusicTagsPlugin(Plugin[MusicTagsOptions]): ) setattr(audio_file, tag_name, tag_value[0]) - if self.plugin_options.embed_thumbnail and entry.is_thumbnail_downloaded(): - with open(entry.get_download_thumbnail_path(), "rb") as thumb: - mediafile_img = mediafile.Image( - data=thumb.read(), desc="cover", type=mediafile.ImageType.front - ) - - audio_file.images = [mediafile_img] - audio_file.save() # report the tags written - title = f"{'Embedded Thumbnail, ' if self.plugin_options.embed_thumbnail else ''}Music Tags" return FileMetadata.from_dict( + title="Music Tags", value_dict=tags_to_write, - title=title, ) diff --git a/src/ytdl_sub/plugins/nfo_tags.py b/src/ytdl_sub/plugins/nfo_tags.py index 728259e1..86fe6568 100644 --- a/src/ytdl_sub/plugins/nfo_tags.py +++ b/src/ytdl_sub/plugins/nfo_tags.py @@ -8,7 +8,7 @@ from typing import List from typing import Optional from ytdl_sub.config.plugin.plugin import Plugin -from ytdl_sub.config.validators.options import OptionsDictValidator +from ytdl_sub.config.validators.options import ToggleableOptionsDictValidator from ytdl_sub.entries.entry import Entry from ytdl_sub.utils.file_handler import FileHandler from ytdl_sub.utils.file_handler import FileMetadata @@ -23,13 +23,13 @@ from ytdl_sub.validators.string_formatter_validators import StringFormatterValid from ytdl_sub.validators.validators import BoolValidator -class SharedNfoTagsOptions(OptionsDictValidator): +class SharedNfoTagsOptions(ToggleableOptionsDictValidator): """ Shared code between NFO tags and Ouptut Directory NFO Tags """ _required_keys = {"nfo_name", "nfo_root", "tags"} - _optional_keys = {"kodi_safe"} + _optional_keys = {"enable", "kodi_safe"} @classmethod def partial_validate(cls, name: str, value: Any) -> None: diff --git a/src/ytdl_sub/plugins/output_directory_nfo_tags.py b/src/ytdl_sub/plugins/output_directory_nfo_tags.py index f754ec44..aae44103 100644 --- a/src/ytdl_sub/plugins/output_directory_nfo_tags.py +++ b/src/ytdl_sub/plugins/output_directory_nfo_tags.py @@ -32,7 +32,7 @@ class OutputDirectoryNfoTagsOptions(SharedNfoTagsOptions): # Hack to make it so collection named seasons do not error # when adding output_directory_nfo info for plex _required_keys = set() - _optional_keys = {"kodi_safe", "nfo_name", "nfo_root", "tags"} + _optional_keys = {"enable", "kodi_safe", "nfo_name", "nfo_root", "tags"} @property def nfo_root(self) -> StringFormatterValidator: diff --git a/src/ytdl_sub/plugins/regex.py b/src/ytdl_sub/plugins/regex.py index 47b34262..0874b7f7 100644 --- a/src/ytdl_sub/plugins/regex.py +++ b/src/ytdl_sub/plugins/regex.py @@ -8,7 +8,7 @@ from typing import Set from ytdl_sub.config.overrides import Overrides from ytdl_sub.config.plugin.plugin import Plugin from ytdl_sub.config.plugin.plugin_operation import PluginOperation -from ytdl_sub.config.validators.options import OptionsDictValidator +from ytdl_sub.config.validators.options import ToggleableOptionsDictValidator from ytdl_sub.entries.entry import Entry from ytdl_sub.script.parser import parse from ytdl_sub.script.utils.exceptions import RuntimeException @@ -120,7 +120,7 @@ class FromSourceVariablesRegex(DictValidator): } -class RegexOptions(OptionsDictValidator): +class RegexOptions(ToggleableOptionsDictValidator): r""" .. attention:: @@ -218,7 +218,7 @@ class RegexOptions(OptionsDictValidator): """ _required_keys = {"from"} - _optional_keys = {"skip_if_match_fails"} + _optional_keys = {"enable", "skip_if_match_fails"} @classmethod def partial_validate(cls, name: str, value: Any) -> None: diff --git a/src/ytdl_sub/plugins/subtitles.py b/src/ytdl_sub/plugins/subtitles.py index 0e74d448..7be3d8f3 100644 --- a/src/ytdl_sub/plugins/subtitles.py +++ b/src/ytdl_sub/plugins/subtitles.py @@ -6,7 +6,7 @@ from typing import Set from ytdl_sub.config.plugin.plugin import Plugin from ytdl_sub.config.plugin.plugin_operation import PluginOperation -from ytdl_sub.config.validators.options import OptionsDictValidator +from ytdl_sub.config.validators.options import ToggleableOptionsDictValidator from ytdl_sub.downloaders.ytdl_options_builder import YTDLOptionsBuilder from ytdl_sub.entries.entry import Entry from ytdl_sub.entries.script.variable_definitions import VARIABLES @@ -31,7 +31,7 @@ class SubtitlesTypeValidator(StringSelectValidator): _select_values = SUBTITLE_EXTENSIONS -class SubtitleOptions(OptionsDictValidator): +class SubtitleOptions(ToggleableOptionsDictValidator): """ Defines how to download and store subtitles. Using this plugin creates two new variables: ``lang`` and ``subtitles_ext``. ``lang`` is dynamic since you can download multiple subtitles. @@ -52,6 +52,7 @@ class SubtitleOptions(OptionsDictValidator): """ _optional_keys = { + "enable", "subtitles_name", "subtitles_type", "embed_subtitles", diff --git a/src/ytdl_sub/plugins/throttle_protection.py b/src/ytdl_sub/plugins/throttle_protection.py index 5295c4f2..9fb102db 100644 --- a/src/ytdl_sub/plugins/throttle_protection.py +++ b/src/ytdl_sub/plugins/throttle_protection.py @@ -6,7 +6,7 @@ from typing import Tuple from ytdl_sub.config.overrides import Overrides from ytdl_sub.config.plugin.plugin import Plugin -from ytdl_sub.config.validators.options import OptionsDictValidator +from ytdl_sub.config.validators.options import ToggleableOptionsDictValidator from ytdl_sub.entries.entry import Entry from ytdl_sub.utils.file_handler import FileMetadata from ytdl_sub.utils.logger import Logger @@ -59,7 +59,7 @@ class RandomizedRangeValidator(StrictDictValidator): return int(self.randomized_float()) -class ThrottleProtectionOptions(OptionsDictValidator): +class ThrottleProtectionOptions(ToggleableOptionsDictValidator): """ Provides options to make ytdl-sub look more 'human-like' to protect from throttling. For range-based values, a random number will be chosen within the range to avoid sleeps looking @@ -85,6 +85,7 @@ class ThrottleProtectionOptions(OptionsDictValidator): """ _optional_keys = { + "enable", "sleep_per_download_s", "sleep_per_subscription_s", "max_downloads_per_subscription", diff --git a/src/ytdl_sub/plugins/video_tags.py b/src/ytdl_sub/plugins/video_tags.py index 0d4614a8..16b7d949 100644 --- a/src/ytdl_sub/plugins/video_tags.py +++ b/src/ytdl_sub/plugins/video_tags.py @@ -1,9 +1,7 @@ -import copy -from typing import Any from typing import Dict from ytdl_sub.config.plugin.plugin import Plugin -from ytdl_sub.config.validators.options import OptionsDictValidator +from ytdl_sub.config.validators.options import OptionsValidator from ytdl_sub.entries.entry import Entry from ytdl_sub.utils.ffmpeg import add_ffmpeg_metadata_key_values from ytdl_sub.utils.file_handler import FileMetadata @@ -13,7 +11,7 @@ from ytdl_sub.validators.string_formatter_validators import DictFormatterValidat logger = Logger.get("video-tags") -class VideoTagsOptions(OptionsDictValidator): +class VideoTagsOptions(DictFormatterValidator, OptionsValidator): """ Adds tags to every downloaded video file using ffmpeg ``-metadata key=value`` args. @@ -27,34 +25,6 @@ class VideoTagsOptions(OptionsDictValidator): description: "{description}" """ - _optional_keys = {"tags"} - _allow_extra_keys = True - - @classmethod - def partial_validate(cls, name: str, value: Any) -> None: - """ - Partially validate video tags - """ - if isinstance(value, dict): - value["tags"] = value.get("tags", {}) - _ = cls(name, value) - - def __init__(self, name, value): - super().__init__(name, value) - - new_tags_dict: Dict[str, Any] = copy.deepcopy(value) - old_tags_dict = new_tags_dict.pop("tags", {}) - - self._is_old_format = len(old_tags_dict) > 0 - self._tags = DictFormatterValidator(name=name, value=dict(old_tags_dict, **new_tags_dict)) - - @property - def tags(self) -> DictFormatterValidator: - """ - Key/values of tag names/values. Supports source and override variables. - """ - return self._tags - class VideoTagsPlugin(Plugin[VideoTagsOptions]): plugin_options_type = VideoTagsOptions @@ -63,17 +33,8 @@ class VideoTagsPlugin(Plugin[VideoTagsOptions]): """ Tags the entry's audio file using values defined in the metadata options """ - # pylint: disable=protected-access - if self.plugin_options._is_old_format: - logger.warning( - "video_tags.tags is now deprecated. Place your tags directly under video_tags " - "instead. The old format will be removed in October of 2023. See " - "https://ytdl-sub.readthedocs.io/en/latest/deprecation_notices.html#video-tags " - "for more details." - ) - tags_to_write: Dict[str, str] = {} - for tag_name, tag_formatter in self.plugin_options.tags.dict.items(): + for tag_name, tag_formatter in self.plugin_options.dict.items(): tag_value = self.overrides.apply_formatter(formatter=tag_formatter, entry=entry) tags_to_write[tag_name] = tag_value diff --git a/src/ytdl_sub/prebuilt_presets/helpers/common.yaml b/src/ytdl_sub/prebuilt_presets/helpers/download_deletion_options.yaml similarity index 67% rename from src/ytdl_sub/prebuilt_presets/helpers/common.yaml rename to src/ytdl_sub/prebuilt_presets/helpers/download_deletion_options.yaml index 9278d012..ffb0f596 100644 --- a/src/ytdl_sub/prebuilt_presets/helpers/common.yaml +++ b/src/ytdl_sub/prebuilt_presets/helpers/download_deletion_options.yaml @@ -1,36 +1,36 @@ - - presets: - best_video_quality: - format: "bestvideo+bestaudio/best" - ytdl_options: - merge_output_format: "mp4" - - max_1080p: - format: "(bv*[height<=1080]+bestaudio/best[height<=1080])" - ytdl_options: - merge_output_format: "mp4" - - chunk_initial_download: - ytdl_options: - max_downloads: 20 - playlistreverse: True - break_on_existing: False - break_on_reject: True - "Only Recent": + preset: + - "Only Recent Archive" # Only fetch videos after today minus date_range - date_range: - after: "today-{only_recent_date_range}" # Only keep files uploaded after date_range output_options: keep_files_after: "today-{only_recent_date_range}" keep_max_files: "{only_recent_max_files}" + overrides: + only_recent_max_files: 0 + + + "Only Recent Archive": + # Only fetch videos after today minus date_range + date_range: + after: "today-{only_recent_date_range}" + # Set the default date_range to 2 months overrides: date_range: "2months" # keep for legacy-reasons only_recent_date_range: "{date_range}" - only_recent_max_files: 0 \ No newline at end of file + + + chunk_initial_download: + ytdl_options: + max_downloads: 20 + playlistreverse: True + break_on_existing: False + + "Download in Chunks": + preset: + - chunk_initial_download \ No newline at end of file diff --git a/src/ytdl_sub/prebuilt_presets/helpers/media_quality.yaml b/src/ytdl_sub/prebuilt_presets/helpers/media_quality.yaml new file mode 100644 index 00000000..7bb5e53d --- /dev/null +++ b/src/ytdl_sub/prebuilt_presets/helpers/media_quality.yaml @@ -0,0 +1,40 @@ +presets: + + best_video_quality: + format: "bestvideo+bestaudio/best" + ytdl_options: + merge_output_format: "mp4" + + "Best Video Quality": + preset: + - best_video_quality + + + "Max 2160p": + format: "(bv*[height<=2160]+bestaudio/best[height<=2160])" + ytdl_options: + merge_output_format: "mp4" + + "Max 1440p": + format: "(bv*[height<=1440]+bestaudio/best[height<=1440])" + ytdl_options: + merge_output_format: "mp4" + + max_1080p: + format: "(bv*[height<=1080]+bestaudio/best[height<=1080])" + ytdl_options: + merge_output_format: "mp4" + + "Max 1080p": + preset: + - max_1080p + + "Max 720p": + format: "(bv*[height<=720]+bestaudio/best[height<=720])" + ytdl_options: + merge_output_format: "mp4" + + "Max 480p": + format: "(bv*[height<=480]+bestaudio/best[height<=480])" + ytdl_options: + merge_output_format: "mp4" \ No newline at end of file diff --git a/src/ytdl_sub/prebuilt_presets/helpers/url.yaml b/src/ytdl_sub/prebuilt_presets/helpers/url.yaml index 746578ea..787e0713 100644 --- a/src/ytdl_sub/prebuilt_presets/helpers/url.yaml +++ b/src/ytdl_sub/prebuilt_presets/helpers/url.yaml @@ -20,6 +20,11 @@ presets: uid: "avatar_uncropped" - name: "{banner_uncropped_thumbnail_file_name}" uid: "banner_uncropped" + source_thumbnails: + - name: "{avatar_uncropped_thumbnail_file_name}" + uid: "avatar_uncropped" + - name: "{banner_uncropped_thumbnail_file_name}" + uid: "banner_uncropped" - url: "{url2}" - url: "{url3}" - url: "{url4}" diff --git a/src/ytdl_sub/prebuilt_presets/music/albums_from_chapters.yaml b/src/ytdl_sub/prebuilt_presets/music/albums_from_chapters.yaml new file mode 100644 index 00000000..0c629516 --- /dev/null +++ b/src/ytdl_sub/prebuilt_presets/music/albums_from_chapters.yaml @@ -0,0 +1,21 @@ +presets: + + # assumes that each chapter in an entry is a song + _albums_from_chapters: + preset: "Single" + + chapters: + embed_chapters: True + split_by_chapters: + when_no_chapters: "pass" + + overrides: + track_title: "{chapter_title}" # Chapter title is the track title + track_album: "{title}" # Video's title is the album title + track_number: "{chapter_index}" + track_number_padded: "{chapter_index_padded}" + track_total: "{chapter_count}" + + "YouTube Full Albums": + preset: + - "_albums_from_chapters" \ No newline at end of file diff --git a/src/ytdl_sub/prebuilt_presets/music/albums_from_playlists.yaml b/src/ytdl_sub/prebuilt_presets/music/albums_from_playlists.yaml new file mode 100644 index 00000000..ab0955a0 --- /dev/null +++ b/src/ytdl_sub/prebuilt_presets/music/albums_from_playlists.yaml @@ -0,0 +1,36 @@ +presets: + + # assumes that each entry is a song in one album + _albums_from_playlists: + preset: + - "_music_base" + + download: + - url: "{url}" + include_sibling_metadata: True + + overrides: + track_album: "{playlist_title}" + track_number: "{playlist_index}" + track_number_padded: "{playlist_index_padded}" + track_total: "{playlist_count}" + track_year: "{playlist_max_upload_year}" + + "YouTube Releases": + preset: + - "_albums_from_playlists" + + "Bandcamp": + preset: + - "_albums_from_playlists" + regex: + from: + title: + match: + - ".*? - (.*)" # Captures 'Some - Song' from 'Emily Hopkins - Some - Song' + capture_group_names: + - "captured_track_title" + capture_group_defaults: + - "{title}" + overrides: + track_title: "{captured_track_title}" \ No newline at end of file diff --git a/src/ytdl_sub/prebuilt_presets/music/music.yaml b/src/ytdl_sub/prebuilt_presets/music/music.yaml deleted file mode 100644 index 7c2f23a3..00000000 --- a/src/ytdl_sub/prebuilt_presets/music/music.yaml +++ /dev/null @@ -1,163 +0,0 @@ -presets: - - _music_base: - output_options: - output_directory: "{music_directory}" - file_name: "{track_full_path}" - thumbnail_name: "{album_cover_path}" - maintain_download_archive: True - - ytdl_options: - break_on_existing: True - - format: "ba[ext=webm]/ba" - - audio_extract: - codec: "best" - - music_tags: - artist: "{track_artist}" - albumartist: "{track_album_artist}" - title: "{track_title}" - album: "{track_album}" - track: "{track_number}" - tracktotal: "{track_total}" - year: "{track_year}" - # multi-tags - artists: - - "{track_artist}" - albumartists: - - "{track_album_artist}" - genres: - - "{track_genre}" - - overrides: -# MUST DEFINE: -# music_directory - - # Defaults - track_genre_default: "Unset" - - # Subscription overrides - subscription_indent_1: "{track_genre_default}" - subscription_value: "" - url: "{subscription_value}" - - # Track Overrides - track_title: "{title}" - track_album: "{title}" - track_artist: "{subscription_name}" - track_album_artist: "{track_artist}" - track_number: "1" - track_number_padded: "01" - track_total: "1" - track_year: "{upload_year}" - track_genre: "{subscription_indent_1}" - - # Directory Overrides - artist_dir: "{track_artist_sanitized}" - album_dir: "[{track_year}] {track_album_sanitized}" - track_file_name: "{track_number_padded} - {track_title_sanitized}.{ext}" - track_full_path: "{artist_dir}/{album_dir}/{track_file_name}" - album_cover_path: "{artist_dir}/{album_dir}/folder.{thumbnail_ext}" - - - "Single": - preset: - - "_music_base" - - download: - - url: "{url}" - include_sibling_metadata: False - - - _albums_from_playlists: - preset: - - "_music_base" - - download: - - url: "{url}" - include_sibling_metadata: True - - overrides: - track_album: "{playlist_title}" - track_number: "{playlist_index}" - track_number_padded: "{playlist_index_padded}" - track_total: "{playlist_count}" - track_year: "{playlist_max_upload_year}" - - - _albums_from_chapters: - preset: "Single" - - chapters: - embed_chapters: True - split_by_chapters: - when_no_chapters: "pass" - - overrides: - track_title: "{chapter_title}" # Chapter title is the track title - track_album: "{title}" # Video's title is the album title - track_number: "{chapter_index}" - track_number_padded: "{chapter_index_padded}" - track_total: "{chapter_count}" - - "SoundCloud Discography": - preset: "_music_base" - - # Download using the multi_url strategy - download: - download_strategy: "multi_url" - urls: - # The first URL will be all the artist's tracks. - # Treat these as singles - an album with a single track - - url: "{url}/tracks" - include_sibling_metadata: False - variables: - sc_track_album: "{title}" - sc_track_number: "1" - sc_track_number_padded: "01" - sc_track_total: "1" - sc_track_year: "{upload_year}" - # Set the second URL to the artist's albums. If a track belongs to both - # to an album and tracks (in the URL above), it will resolve to this - # URL and include the album metadata we set below. - - url: "{url}/albums" - include_sibling_metadata: True - variables: - sc_track_album: "{playlist_title}" - sc_track_number: "{playlist_index}" - sc_track_number_padded: "{playlist_index_padded}" - sc_track_total: "{playlist_count}" - sc_track_year: "{playlist_max_upload_year}" - - # Override various track properties using playlist variables. - overrides: - track_album: "{sc_track_album}" - track_number: "{sc_track_number}" - track_number_padded: "{sc_track_number_padded}" - track_total: "{sc_track_total}" - track_year: "{sc_track_year}" - - "YouTube Releases": - preset: - - "_albums_from_playlists" - - "YouTube Full Albums": - preset: - - "_albums_from_chapters" - - "Bandcamp": - preset: - - "_albums_from_playlists" - regex: - from: - title: - match: - - ".*? - (.*)" # Captures 'Some - Song' from 'Emily Hopkins - Some - Song' - capture_group_names: - - "captured_track_title" - capture_group_defaults: - - "{title}" - overrides: - track_title: "{captured_track_title}" diff --git a/src/ytdl_sub/prebuilt_presets/music/other_websites.yaml b/src/ytdl_sub/prebuilt_presets/music/other_websites.yaml new file mode 100644 index 00000000..14780bea --- /dev/null +++ b/src/ytdl_sub/prebuilt_presets/music/other_websites.yaml @@ -0,0 +1,38 @@ +presets: + + "SoundCloud Discography": + preset: "_music_base" + + # Download using the multi_url strategy + download: + download_strategy: "multi_url" + urls: + # The first URL will be all the artist's tracks. + # Treat these as singles - an album with a single track + - url: "{url}/tracks" + include_sibling_metadata: False + variables: + sc_track_album: "{title}" + sc_track_number: "1" + sc_track_number_padded: "01" + sc_track_total: "1" + sc_track_year: "{upload_year}" + # Set the second URL to the artist's albums. If a track belongs to both + # to an album and tracks (in the URL above), it will resolve to this + # URL and include the album metadata we set below. + - url: "{url}/albums" + include_sibling_metadata: True + variables: + sc_track_album: "{playlist_title}" + sc_track_number: "{playlist_index}" + sc_track_number_padded: "{playlist_index_padded}" + sc_track_total: "{playlist_count}" + sc_track_year: "{playlist_max_upload_year}" + + # Override various track properties using playlist variables. + overrides: + track_album: "{sc_track_album}" + track_number: "{sc_track_number}" + track_number_padded: "{sc_track_number_padded}" + track_total: "{sc_track_total}" + track_year: "{sc_track_year}" \ No newline at end of file diff --git a/src/ytdl_sub/prebuilt_presets/music/singles.yaml b/src/ytdl_sub/prebuilt_presets/music/singles.yaml new file mode 100644 index 00000000..634fd43f --- /dev/null +++ b/src/ytdl_sub/prebuilt_presets/music/singles.yaml @@ -0,0 +1,72 @@ +presets: + + + _music_base: + output_options: + output_directory: "{music_directory}" + file_name: "{track_full_path}" + thumbnail_name: "{album_cover_path}" + maintain_download_archive: True + + ytdl_options: + break_on_existing: True + + format: "ba[ext=webm]/ba" + + audio_extract: + codec: "best" + + music_tags: + artist: "{track_artist}" + albumartist: "{track_album_artist}" + title: "{track_title}" + album: "{track_album}" + track: "{track_number}" + tracktotal: "{track_total}" + year: "{track_year}" + # multi-tags + artists: + - "{track_artist}" + albumartists: + - "{track_album_artist}" + genres: + - "{track_genre}" + + overrides: +# MUST DEFINE: +# music_directory + + # Defaults + track_genre_default: "Unset" + + # Subscription overrides + subscription_indent_1: "{track_genre_default}" + subscription_value: "" + url: "{subscription_value}" + + # Track Overrides + track_title: "{title}" + track_album: "{title}" + track_artist: "{subscription_name}" + track_album_artist: "{track_artist}" + track_number: "1" + track_number_padded: "01" + track_total: "1" + track_year: "{upload_year}" + track_genre: "{subscription_indent_1}" + + # Directory Overrides + artist_dir: "{track_artist_sanitized}" + album_dir: "[{track_year}] {track_album_sanitized}" + track_file_name: "{track_number_padded} - {track_title_sanitized}.{ext}" + track_full_path: "{artist_dir}/{album_dir}/{track_file_name}" + album_cover_path: "{artist_dir}/{album_dir}/folder.{thumbnail_ext}" + + + "Single": + preset: + - "_music_base" + + download: + - url: "{url}" + include_sibling_metadata: False \ No newline at end of file diff --git a/src/ytdl_sub/prebuilt_presets/music_videos/music_video_base.yaml b/src/ytdl_sub/prebuilt_presets/music_videos/music_video_base.yaml new file mode 100644 index 00000000..d040a7c9 --- /dev/null +++ b/src/ytdl_sub/prebuilt_presets/music_videos/music_video_base.yaml @@ -0,0 +1,33 @@ +presets: + _music_video_base: + output_options: + output_directory: "{music_video_directory}" + file_name: "{music_video_file_name}.{ext}" + thumbnail_name: "{music_video_file_name}.jpg" + maintain_download_archive: True + + ytdl_options: + break_on_existing: True + + overrides: +# MUST DEFINE: +# music_video_directory + + # Defaults + music_video_genre_default: "ytdl-sub" + music_video_album_default: "Music Videos" + + # Subscription overrides + subscription_indent_1: "{music_video_genre_default}" + + # Music Video Overrides + music_video_title: "{title}" + music_video_album: "{music_video_album_default}" + music_video_artist: "{subscription_name}" + music_video_year: "{upload_year}" + music_video_date: "{upload_date_standardized}" + music_video_genre: "{subscription_indent_1}" + + # Directory Overrides + music_video_file_name_suffix: "" + music_video_file_name: "{music_video_artist_sanitized}/{music_video_title_sanitized}{music_video_file_name_suffix}" diff --git a/src/ytdl_sub/prebuilt_presets/music_videos/music_videos.yaml b/src/ytdl_sub/prebuilt_presets/music_videos/music_videos.yaml index f535bc6d..47aeb2ba 100644 --- a/src/ytdl_sub/prebuilt_presets/music_videos/music_videos.yaml +++ b/src/ytdl_sub/prebuilt_presets/music_videos/music_videos.yaml @@ -1,58 +1,4 @@ presets: - _music_video_base: - output_options: - output_directory: "{music_video_directory}" - file_name: "{music_video_file_name}.{ext}" - thumbnail_name: "{music_video_file_name}.jpg" - maintain_download_archive: True - - ytdl_options: - break_on_existing: True - - overrides: -# MUST DEFINE: -# music_video_directory - - # Defaults - music_video_genre_default: "ytdl-sub" - music_video_album_default: "Music Videos" - - # Subscription overrides - subscription_indent_1: "{music_video_genre_default}" - - # Music Video Overrides - music_video_title: "{title}" - music_video_album: "{music_video_album_default}" - music_video_artist: "{subscription_name}" - music_video_year: "{upload_year}" - music_video_date: "{upload_date_standardized}" - music_video_genre: "{subscription_indent_1}" - - # Directory Overrides - music_video_file_name_suffix: "" - music_video_file_name: "{music_video_artist_sanitized}/{music_video_title_sanitized}{music_video_file_name_suffix}" - - _music_video_nfo: - nfo_tags: - nfo_name: "{music_video_file_name}.nfo" - nfo_root: "musicvideo" - tags: - artist: "{music_video_artist}" - title: "{music_video_title}" - album: "{music_video_album}" - genre: - - "{music_video_genre}" - # Kodi v20 to not use year, so removing - premiered: "{music_video_date}" - - _music_video_tags: - video_tags: - artist: "{music_video_artist}" - title: "{music_video_title}" - album: "{music_video_album}" - genre: "{music_video_genre}" - year: "{music_video_year}" - premiered: "{music_video_date}" "Jellyfin Music Videos": preset: @@ -75,4 +21,26 @@ presets: - "_multi_url" - "_plex_video_base" - "_music_video_base" - - "_music_video_tags" \ No newline at end of file + - "_music_video_tags" + + _music_video_nfo: + nfo_tags: + nfo_name: "{music_video_file_name}.nfo" + nfo_root: "musicvideo" + tags: + artist: "{music_video_artist}" + title: "{music_video_title}" + album: "{music_video_album}" + genre: + - "{music_video_genre}" + # Kodi v20 to not use year, so removing + premiered: "{music_video_date}" + + _music_video_tags: + video_tags: + artist: "{music_video_artist}" + title: "{music_video_title}" + album: "{music_video_album}" + genre: "{music_video_genre}" + year: "{music_video_year}" + premiered: "{music_video_date}" \ No newline at end of file diff --git a/src/ytdl_sub/prebuilt_presets/tv_show/episode.yaml b/src/ytdl_sub/prebuilt_presets/tv_show/episode.yaml index bb1c9047..b6bacd27 100644 --- a/src/ytdl_sub/prebuilt_presets/tv_show/episode.yaml +++ b/src/ytdl_sub/prebuilt_presets/tv_show/episode.yaml @@ -6,7 +6,7 @@ presets: output_options: output_directory: "{tv_show_directory}/{tv_show_name_sanitized}" file_name: "{episode_file_path}.{ext}" - thumbnail_name: "{episode_file_path}-thumb.jpg" + thumbnail_name: "{thumbnail_file_name}" info_json_name: "{episode_file_path}.{info_json_ext}" maintain_download_archive: True @@ -43,102 +43,7 @@ presets: episode_date_standardized: "{upload_date_standardized}" episode_file_name: "s{season_number_padded}.e{episode_number_padded} - {file_title}" episode_file_path: "{season_directory_name_sanitized}/{episode_file_name_sanitized}" + thumbnail_file_name: "{episode_file_path}-thumb.jpg" - _episode_video_tags: - video_tags: - show: "{tv_show_name}" - genre: "{tv_show_genre}" - episode_id: "{episode_number}" - title: "{episode_title}" - synopsis: "{episode_plot}" - year: "{episode_year}" - date: "{episode_date_standardized}" - contentRating: "{episode_content_rating}" - _episode_nfo_tags: - nfo_tags: - nfo_name: "{episode_file_path}.nfo" - nfo_root: "episodedetails" - tags: - genre: - - "{tv_show_genre}" - season: "{season_number}" - episode: "{episode_number}" - title: "{episode_title}" - plot: "{episode_plot}" - year: "{episode_year}" - aired: "{episode_date_standardized}" - mpaa: "{episode_content_rating}" - output_directory_nfo_tags: - nfo_name: "tvshow.nfo" - nfo_root: "tvshow" - tags: - title: "{tv_show_name}" - genre: - - "{tv_show_genre}" - mpaa: "{tv_show_content_rating}" - -#################################################################################################### - - _season_by_year: - overrides: - season_number: "{upload_year}" - season_number_padded: "{season_number}" - - _season_by_year_month: - overrides: - season_number: "{upload_year}{upload_month_padded}" - season_number_padded: "{season_number}" - -#################################################################################################### - - season_by_year_month__episode_by_day: - preset: - - "_season_by_year_month" - overrides: - episode_number: "{upload_day}{upload_date_index_padded}" - episode_number_padded: "{upload_day_padded}{upload_date_index_padded}" - - season_by_year__episode_by_month_day: - preset: - - "_season_by_year" - overrides: - episode_number: "{upload_month}{upload_day_padded}{upload_date_index_padded}" - episode_number_padded: "{upload_month_padded}{upload_day_padded}{upload_date_index_padded}" - - season_by_year__episode_by_month_day_reversed: - preset: - - "_season_by_year" - overrides: - episode_number: "{upload_day_of_year_reversed}{upload_date_index_reversed_padded}" - episode_number_padded: "{upload_day_of_year_reversed_padded}{upload_date_index_reversed_padded}" - - season_by_year__episode_by_download_index: - preset: - - "_season_by_year" - overrides: - episode_number: "{download_index}" - episode_number_padded: "{download_index_padded6}" - - ############### - - season_by_collection__episode_by_year_month_day: - overrides: - episode_number: "{upload_year_truncated}{upload_month_padded}{upload_day_padded}{upload_date_index_padded}" - episode_number_padded: "{episode_number}" - - season_by_collection__episode_by_year_month_day_reversed: - overrides: - episode_number: "{upload_year_truncated_reversed}{upload_month_reversed_padded}{upload_day_reversed_padded}{upload_date_index_reversed_padded}" - episode_number_padded: "{episode_number}" - - season_by_collection__episode_by_playlist_index: - overrides: - episode_number: "{playlist_index}" - episode_number_padded: "{playlist_index_padded6}" - - season_by_collection__episode_by_playlist_index_reversed: - overrides: - episode_number: "{playlist_index_reversed}" - episode_number_padded: "{playlist_index_reversed_padded6}" diff --git a/src/ytdl_sub/prebuilt_presets/tv_show/tv_show.yaml b/src/ytdl_sub/prebuilt_presets/tv_show/tv_show.yaml index 308162e2..edf64ec5 100644 --- a/src/ytdl_sub/prebuilt_presets/tv_show/tv_show.yaml +++ b/src/ytdl_sub/prebuilt_presets/tv_show/tv_show.yaml @@ -46,4 +46,39 @@ presets: overrides: collection_season_number_padded: "{ %pad_zero(%int(collection_season_number), 2) }" season_number: "{collection_season_number}" - season_number_padded: "{collection_season_number_padded}" \ No newline at end of file + season_number_padded: "{collection_season_number_padded}" + + _episode_video_tags: + video_tags: + show: "{tv_show_name}" + genre: "{tv_show_genre}" + episode_id: "{episode_number}" + title: "{episode_title}" + synopsis: "{episode_plot}" + year: "{episode_year}" + date: "{episode_date_standardized}" + contentRating: "{episode_content_rating}" + + _episode_nfo_tags: + nfo_tags: + nfo_name: "{episode_file_path}.nfo" + nfo_root: "episodedetails" + tags: + genre: + - "{tv_show_genre}" + season: "{season_number}" + episode: "{episode_number}" + title: "{episode_title}" + plot: "{episode_plot}" + year: "{episode_year}" + aired: "{episode_date_standardized}" + mpaa: "{episode_content_rating}" + + output_directory_nfo_tags: + nfo_name: "tvshow.nfo" + nfo_root: "tvshow" + tags: + title: "{tv_show_name}" + genre: + - "{tv_show_genre}" + mpaa: "{tv_show_content_rating}" \ No newline at end of file diff --git a/src/ytdl_sub/prebuilt_presets/tv_show/tv_show_by_date.yaml b/src/ytdl_sub/prebuilt_presets/tv_show/tv_show_by_date.yaml index 275216fe..30027444 100644 --- a/src/ytdl_sub/prebuilt_presets/tv_show/tv_show_by_date.yaml +++ b/src/ytdl_sub/prebuilt_presets/tv_show/tv_show_by_date.yaml @@ -29,4 +29,46 @@ presets: "Plex TV Show by Date": preset: - "plex_tv_show_by_date" - - "season_by_year__episode_by_month_day" \ No newline at end of file + - "season_by_year__episode_by_month_day" + +#################################################################################################### + + _season_by_year: + overrides: + season_number: "{upload_year}" + season_number_padded: "{season_number}" + + _season_by_year_month: + overrides: + season_number: "{upload_year}{upload_month_padded}" + season_number_padded: "{season_number}" + +#################################################################################################### + + season_by_year_month__episode_by_day: + preset: + - "_season_by_year_month" + overrides: + episode_number: "{upload_day}{upload_date_index_padded}" + episode_number_padded: "{upload_day_padded}{upload_date_index_padded}" + + season_by_year__episode_by_month_day: + preset: + - "_season_by_year" + overrides: + episode_number: "{upload_month}{upload_day_padded}{upload_date_index_padded}" + episode_number_padded: "{upload_month_padded}{upload_day_padded}{upload_date_index_padded}" + + season_by_year__episode_by_month_day_reversed: + preset: + - "_season_by_year" + overrides: + episode_number: "{upload_day_of_year_reversed}{upload_date_index_reversed_padded}" + episode_number_padded: "{upload_day_of_year_reversed_padded}{upload_date_index_reversed_padded}" + + season_by_year__episode_by_download_index: + preset: + - "_season_by_year" + overrides: + episode_number: "{download_index}" + episode_number_padded: "{download_index_padded6}" \ No newline at end of file diff --git a/src/ytdl_sub/prebuilt_presets/tv_show/tv_show_collection.yaml b/src/ytdl_sub/prebuilt_presets/tv_show/tv_show_collection.yaml index 1b374d33..0999161c 100644 --- a/src/ytdl_sub/prebuilt_presets/tv_show/tv_show_collection.yaml +++ b/src/ytdl_sub/prebuilt_presets/tv_show/tv_show_collection.yaml @@ -1,6 +1,5 @@ presets: - kodi_tv_show_collection: preset: - "_kodi_tv_show" @@ -34,6 +33,11 @@ presets: uid: "avatar_uncropped" - name: "{tv_show_fanart_file_name}" uid: "banner_uncropped" + source_thumbnails: + - name: "{tv_show_poster_file_name}" + uid: "avatar_uncropped" + - name: "{tv_show_fanart_file_name}" + uid: "banner_uncropped" output_directory_nfo_tags: tags: @@ -703,4 +707,27 @@ presets: namedseason: - tag: "{collection_season_40_name}" attributes: - number: "40" \ No newline at end of file + number: "40" + + + ############### + + season_by_collection__episode_by_year_month_day: + overrides: + episode_number: "{upload_year_truncated}{upload_month_padded}{upload_day_padded}{upload_date_index_padded}" + episode_number_padded: "{episode_number}" + + season_by_collection__episode_by_year_month_day_reversed: + overrides: + episode_number: "{upload_year_truncated_reversed}{upload_month_reversed_padded}{upload_day_reversed_padded}{upload_date_index_reversed_padded}" + episode_number_padded: "{episode_number}" + + season_by_collection__episode_by_playlist_index: + overrides: + episode_number: "{playlist_index}" + episode_number_padded: "{playlist_index_padded6}" + + season_by_collection__episode_by_playlist_index_reversed: + overrides: + episode_number: "{playlist_index_reversed}" + episode_number_padded: "{playlist_index_reversed_padded6}" \ No newline at end of file diff --git a/src/ytdl_sub/script/functions/array_functions.py b/src/ytdl_sub/script/functions/array_functions.py index 4af6f63c..a07585e7 100644 --- a/src/ytdl_sub/script/functions/array_functions.py +++ b/src/ytdl_sub/script/functions/array_functions.py @@ -73,12 +73,18 @@ class ArrayFunctions: return Array(output) @staticmethod - def array_at(array: Array, idx: Integer) -> AnyArgument: + def array_at(array: Array, idx: Integer, default: Optional[AnyArgument] = None) -> AnyArgument: """ :description: - Return the element in the Array at index ``idx``. + Return the element in the Array at index ``idx``. If ``idx`` exceeds the array length, + either return ``default`` if provided or throw an error. """ - return array.value[idx.value] + try: + return array.value[idx.value] + except IndexError: + if default is not None: + return default + raise @staticmethod def array_first(array: Array, fallback: AnyArgument) -> AnyArgument: diff --git a/src/ytdl_sub/script/functions/conditional_functions.py b/src/ytdl_sub/script/functions/conditional_functions.py index e532031b..8b9da74b 100644 --- a/src/ytdl_sub/script/functions/conditional_functions.py +++ b/src/ytdl_sub/script/functions/conditional_functions.py @@ -1,8 +1,10 @@ from typing import Union +from ytdl_sub.script.types.resolvable import AnyArgument from ytdl_sub.script.types.resolvable import Boolean from ytdl_sub.script.types.resolvable import ReturnableArgumentA from ytdl_sub.script.types.resolvable import ReturnableArgumentB +from ytdl_sub.script.utils.exceptions import FunctionRuntimeException class ConditionalFunctions: @@ -19,6 +21,39 @@ class ConditionalFunctions: return true return false + @staticmethod + def elif_(*if_elif_else: AnyArgument) -> AnyArgument: + """ + :description: + Conditional ``if`` statement that is capable of doing else-ifs (``elif``) via + adjacent arguments. It is expected for there to be an odd number of arguments >= 3 to + supply at least one conditional and an else. + :usage: + + .. code-block:: python + + %elif( + condition1, + return1, + condition2, + return2, + ... + else_return + ) + """ + arguments = list(if_elif_else) + if len(arguments) < 3: + raise FunctionRuntimeException("elif requires at least 3 arguments") + + if len(arguments) % 2 == 0: + raise FunctionRuntimeException("elif must have an odd number of arguments") + + for idx in range(0, len(arguments) - 1, 2): + if bool(arguments[idx].value): + return arguments[idx + 1] + + return arguments[-1] + @staticmethod def if_passthrough( maybe_true_arg: ReturnableArgumentA, else_arg: ReturnableArgumentB diff --git a/src/ytdl_sub/script/functions/string_functions.py b/src/ytdl_sub/script/functions/string_functions.py index 1f0280b5..fd0cac9f 100644 --- a/src/ytdl_sub/script/functions/string_functions.py +++ b/src/ytdl_sub/script/functions/string_functions.py @@ -1,5 +1,6 @@ from typing import Optional +from ytdl_sub.script.types.array import Array from ytdl_sub.script.types.resolvable import AnyArgument from ytdl_sub.script.types.resolvable import Boolean from ytdl_sub.script.types.resolvable import Integer @@ -80,6 +81,22 @@ class StringFunctions: return String(string.value.replace(old.value, new.value)) + @staticmethod + def split(string: String, sep: String, max_split: Optional[Integer] = None) -> Array: + """ + :description: + Splits the input string into multiple strings. + """ + if max_split is not None: + return Array( + [ + String(split_val) + for split_val in string.value.split(sep=sep.value, maxsplit=max_split.value) + ] + ) + + return Array([String(split_val) for split_val in string.value.split(sep=sep.value)]) + @staticmethod def concat(*values: String) -> String: """ diff --git a/src/ytdl_sub/script/script.py b/src/ytdl_sub/script/script.py index 0560984b..3f3b42ff 100644 --- a/src/ytdl_sub/script/script.py +++ b/src/ytdl_sub/script/script.py @@ -248,6 +248,52 @@ class Script: for variable_name, resolved in resolved_variables.items(): self._variables[variable_name] = SyntaxTree(ast=[resolved]) + def _recursive_get_unresolved_output_filter_variables( + self, current_var: SyntaxTree, subset_to_resolve: Set[str], unresolvable: Set[Variable] + ) -> Set[str]: + for var_dep in current_var.variables: + if var_dep in unresolvable: + raise ScriptVariableNotResolved( + f"Output filter variable contains the variable {var_dep} " + f"which is set as unresolvable" + ) + subset_to_resolve.add(var_dep.name) + subset_to_resolve |= self._recursive_get_unresolved_output_filter_variables( + current_var=self._variables[var_dep.name], + subset_to_resolve=subset_to_resolve, + unresolvable=unresolvable, + ) + + return subset_to_resolve + + def _get_unresolved_output_filter( + self, + unresolved: Dict[Variable, SyntaxTree], + output_filter: Set[str], + unresolvable: Set[Variable], + ) -> Dict[Variable, SyntaxTree]: + """ + When an output filter is applied, only a subset of variables that the filter + depends on need to be resolved. + """ + subset_to_resolve: Set[str] = set() + + for output_filter_variable in output_filter: + subset_to_resolve.add(output_filter_variable) + + if output_filter_variable not in self._variables: + raise ScriptVariableNotResolved( + "Tried to specify an output filter variable that does not exist" + ) + + subset_to_resolve |= self._recursive_get_unresolved_output_filter_variables( + current_var=self._variables[output_filter_variable], + subset_to_resolve=subset_to_resolve, + unresolvable=unresolvable, + ) + + return {var: syntax for var, syntax in unresolved.items() if var.name in subset_to_resolve} + def _resolve( self, pre_resolved: Optional[Dict[str, Resolvable]] = None, @@ -288,6 +334,13 @@ class Script: if Variable(name) not in unresolved_filter } + if output_filter: + unresolved = self._get_unresolved_output_filter( + unresolved=unresolved, + output_filter=output_filter, + unresolvable=unresolvable, + ) + while unresolved: unresolved_count: int = len(unresolved) diff --git a/src/ytdl_sub/subscriptions/subscription.py b/src/ytdl_sub/subscriptions/subscription.py index 50b2f319..e40c2663 100644 --- a/src/ytdl_sub/subscriptions/subscription.py +++ b/src/ytdl_sub/subscriptions/subscription.py @@ -3,6 +3,9 @@ from pathlib import Path from typing import Any from typing import Dict from typing import List +from typing import Optional + +from mergedeep import mergedeep from ytdl_sub.config.config_file import ConfigFile from ytdl_sub.config.preset import Preset @@ -69,7 +72,11 @@ class Subscription(SubscriptionDownload): @classmethod def from_file_path( - cls, config: ConfigFile, subscription_path: str | Path + cls, + config: ConfigFile, + subscription_path: str | Path, + subscription_matches: Optional[List[str]] = None, + subscription_override_dict: Optional[Dict] = None, ) -> List["Subscription"]: """ Loads subscriptions from a file. @@ -80,6 +87,10 @@ class Subscription(SubscriptionDownload): Validated instance of the config subscription_path: File path to the subscription yaml file + subscription_matches: + Optional list, only output subscriptions that match one or more of these values + subscription_override_dict: + Optional dict containing overrides to every subscription Returns ------- @@ -91,16 +102,16 @@ class Subscription(SubscriptionDownload): If subscription file is misconfigured """ subscriptions: List["Subscription"] = [] - subscription_dict = load_yaml(file_path=subscription_path) + subscription_object = load_yaml(file_path=subscription_path) - has_file_preset = FILE_PRESET_APPLY_KEY in subscription_dict + has_file_preset = FILE_PRESET_APPLY_KEY in subscription_object # If a file preset is present... if has_file_preset: # Validate it (make sure it is a dict) file_preset = LiteralDictValidator( name=f"{subscription_path}.{FILE_PRESET_APPLY_KEY}", - value=subscription_dict[FILE_PRESET_APPLY_KEY], + value=subscription_object[FILE_PRESET_APPLY_KEY], ) # Deep copy the config and add this file preset to its preset list @@ -108,7 +119,9 @@ class Subscription(SubscriptionDownload): config.presets.dict[FILE_PRESET_APPLY_KEY] = file_preset.dict subscriptions_dict: Dict[str, Any] = { - key: obj for key, obj in subscription_dict.items() if key not in [FILE_PRESET_APPLY_KEY] + key: obj + for key, obj in subscription_object.items() + if key not in [FILE_PRESET_APPLY_KEY] } subscriptions_dicts = SubscriptionValidator( @@ -121,11 +134,27 @@ class Subscription(SubscriptionDownload): global_presets_to_apply=[FILE_PRESET_APPLY_KEY] if has_file_preset else [] ) - for subscription_key, subscription_object in subscriptions_dicts.items(): + if subscriptions_dicts and subscription_matches: + logger.info("Filtering subscriptions by name based on --match arguments") + subscriptions_dicts = { + subscription_name: subscription_object + for subscription_name, subscription_object in subscriptions_dicts.items() + if any(match in subscription_name for match in subscription_matches) + } + + for subscription_name, subscription_object in subscriptions_dicts.items(): + + # Hard-override subscriptions here + mergedeep.merge( + subscription_object, + subscription_override_dict or {}, + strategy=mergedeep.Strategy.ADDITIVE, + ) + subscriptions.append( cls.from_dict( config=config, - preset_name=subscription_key, + preset_name=subscription_name, preset_dict=subscription_object, ) ) diff --git a/src/ytdl_sub/subscriptions/subscription_download.py b/src/ytdl_sub/subscriptions/subscription_download.py index 56e93954..06ce1bd2 100644 --- a/src/ytdl_sub/subscriptions/subscription_download.py +++ b/src/ytdl_sub/subscriptions/subscription_download.py @@ -190,7 +190,7 @@ class SubscriptionDownload(BaseSubscription, ABC): ------- List of plugins defined in the subscription, initialized and ready to use. """ - return [ + plugins = [ plugin_type( options=plugin_options, overrides=self.overrides, @@ -198,6 +198,7 @@ class SubscriptionDownload(BaseSubscription, ABC): ) for plugin_type, plugin_options in self.plugins.zipped() ] + return [plugin for plugin in plugins if plugin.is_enabled] @classmethod def _cleanup_entry_files(cls, entry: Entry): diff --git a/src/ytdl_sub/subscriptions/subscription_ytdl_options.py b/src/ytdl_sub/subscriptions/subscription_ytdl_options.py index 77ec1c6b..230134db 100644 --- a/src/ytdl_sub/subscriptions/subscription_ytdl_options.py +++ b/src/ytdl_sub/subscriptions/subscription_ytdl_options.py @@ -84,10 +84,6 @@ class SubscriptionYTDLOptions: "extract_flat": "discard", # do not store info.json in mem since its in file } - @property - def _download_only_options(self) -> Dict: - return {"break_on_reject": True} - @property def _output_options(self) -> Dict: ytdl_options = {} @@ -95,10 +91,12 @@ class SubscriptionYTDLOptions: if self._preset.output_options.maintain_download_archive: ytdl_options["download_archive"] = self._enhanced_download_archive.working_file_path if self._preset.output_options.keep_max_files: - # yt-dlp has a weird bug with max_downloads=1, set to 2 for safe measure - ytdl_options["max_downloads"] = max( - int(self._overrides.apply_formatter(self._preset.output_options.keep_max_files)), 2 + keep_max_files = int( + self._overrides.apply_formatter(self._preset.output_options.keep_max_files) ) + if keep_max_files > 0: + # yt-dlp has a weird bug with max_downloads=1, set to 2 for safe measure + ytdl_options["max_downloads"] = max(keep_max_files, 2) return ytdl_options @@ -176,6 +174,7 @@ class SubscriptionYTDLOptions: self._output_options, self._plugin_match_filters, self._plugin_ytdl_options(FormatPlugin), + self._plugin_ytdl_options(AudioExtractPlugin), # will override format self._user_ytdl_options, # user ytdl options... self._info_json_only_options, # then info_json_only options ) @@ -193,10 +192,9 @@ class SubscriptionYTDLOptions: self._plugin_ytdl_options(FileConvertPlugin), self._plugin_ytdl_options(SubtitlesPlugin), self._plugin_ytdl_options(ChaptersPlugin), - self._plugin_ytdl_options(AudioExtractPlugin), self._plugin_ytdl_options(FormatPlugin), + self._plugin_ytdl_options(AudioExtractPlugin), # will override format self._user_ytdl_options, # user ytdl options... - self._download_only_options, # then download_only options ) # Add dry run options last if enabled if self._dry_run: diff --git a/src/ytdl_sub/utils/logger.py b/src/ytdl_sub/utils/logger.py index 0b531a32..62e40ba1 100644 --- a/src/ytdl_sub/utils/logger.py +++ b/src/ytdl_sub/utils/logger.py @@ -210,6 +210,14 @@ class Logger: finally: redirect_stream.flush() + @classmethod + def _append_to_error_log(cls): + # Any time an exception occurs, dump all debug logs into the error log + with open(cls.debug_log_filename(), mode="r", encoding="utf-8") as debug_logs, open( + cls.error_log_filename(), mode="a", encoding="utf-8" + ) as error_logs: + error_logs.writelines(debug_logs.readlines()) + @classmethod def log_exception(cls, exception: Exception, log_filepath: Optional[Path] = None): """ @@ -248,14 +256,10 @@ class Logger: log_filepath if log_filepath else Logger.error_log_filename(), ) - # Any time an exception occurs, dump all debug logs into the error log - with open(cls.debug_log_filename(), mode="r", encoding="utf-8") as debug_logs, open( - cls.error_log_filename(), mode="a", encoding="utf-8" - ) as error_logs: - error_logs.writelines(debug_logs.readlines()) + cls._append_to_error_log() @classmethod - def cleanup(cls, cleanup_error_log: bool = False): + def cleanup(cls, has_error: bool = False): """ Cleans up debug log file left behind """ @@ -263,9 +267,11 @@ class Logger: for handler in logger.handlers: handler.close() - cls._DEBUG_LOGGER_FILE.close() - FileHandler.delete(cls.debug_log_filename()) - - if cleanup_error_log: + if has_error: + cls._append_to_error_log() + else: cls._ERROR_LOG_FILE.close() FileHandler.delete(cls.error_log_filename()) + + cls._DEBUG_LOGGER_FILE.close() + FileHandler.delete(cls.debug_log_filename()) diff --git a/src/ytdl_sub/utils/script.py b/src/ytdl_sub/utils/script.py index d5d36b87..5653bc74 100644 --- a/src/ytdl_sub/utils/script.py +++ b/src/ytdl_sub/utils/script.py @@ -38,3 +38,15 @@ class ScriptUtils: out = f"{{%from_json('''{dumped_json}''')}}" return out + + @classmethod + def bool_formatter_output(cls, output: str) -> bool: + """ + Translate formatter output to a boolean + """ + if not output or output.lower() == "false": + return False + try: + return bool(json.loads(output)) + except Exception: # pylint: disable=broad-except + return True diff --git a/src/ytdl_sub/utils/scriptable.py b/src/ytdl_sub/utils/scriptable.py index b1dc20e7..6aff1db3 100644 --- a/src/ytdl_sub/utils/scriptable.py +++ b/src/ytdl_sub/utils/scriptable.py @@ -14,10 +14,8 @@ from ytdl_sub.script.utils.exceptions import RuntimeException from ytdl_sub.utils.exceptions import StringFormattingException from ytdl_sub.utils.script import ScriptUtils -_BASE_SCRIPT: Script = Script( - ScriptUtils.add_sanitized_variables( - dict(copy.deepcopy(VARIABLE_SCRIPTS), **copy.deepcopy(CUSTOM_FUNCTION_SCRIPTS)) - ) +BASE_SCRIPT: Script = Script( + dict(ScriptUtils.add_sanitized_variables(VARIABLE_SCRIPTS), **CUSTOM_FUNCTION_SCRIPTS) ) @@ -37,7 +35,7 @@ class Scriptable(ABC): """ Initializes with base values """ - self._script = copy.deepcopy(_BASE_SCRIPT) + self._script = copy.deepcopy(BASE_SCRIPT) self._unresolvable = copy.deepcopy(UNRESOLVED_VARIABLES) @property diff --git a/src/ytdl_sub/validators/string_formatter_validators.py b/src/ytdl_sub/validators/string_formatter_validators.py index 7957169d..c561e47e 100644 --- a/src/ytdl_sub/validators/string_formatter_validators.py +++ b/src/ytdl_sub/validators/string_formatter_validators.py @@ -6,8 +6,10 @@ from typing import final from ytdl_sub.entries.script.variable_definitions import VARIABLES from ytdl_sub.script.parser import parse from ytdl_sub.script.script import Script +from ytdl_sub.script.types.syntax_tree import SyntaxTree +from ytdl_sub.script.utils.exceptions import RuntimeException +from ytdl_sub.script.utils.exceptions import ScriptVariableNotResolved from ytdl_sub.script.utils.exceptions import UserException -from ytdl_sub.script.utils.exceptions import VariableDoesNotExist from ytdl_sub.utils.exceptions import StringFormattingVariableNotFoundException from ytdl_sub.validators.validators import DictValidator from ytdl_sub.validators.validators import ListValidator @@ -90,7 +92,7 @@ class OverridesStringFormatterValidator(StringFormatterValidator): # pylint: enable=line-too-long -class OverridesIntegerFormatterValidator(StringFormatterValidator): +class OverridesIntegerFormatterValidator(OverridesStringFormatterValidator): _expected_value_type_name = "integer" def post_process(self, resolved: str) -> str: @@ -104,6 +106,10 @@ class OverridesIntegerFormatterValidator(StringFormatterValidator): return resolved +class OverridesBooleanFormatterValidator(OverridesStringFormatterValidator): + _expected_value_type_name = "boolean" + + class ListFormatterValidator(ListValidator[StringFormatterValidator]): _inner_list_type = StringFormatterValidator @@ -142,21 +148,67 @@ class OverridesDictFormatterValidator(DictFormatterValidator): _key_validator = OverridesStringFormatterValidator +def to_variable_dependency_format_string(script: Script, parsed_format_string: SyntaxTree) -> str: + """ + Create a dummy format string that contains all variable deps as a string. + """ + dummy_format_string = "" + for var in parsed_format_string.variables: + dummy_format_string += f"{{ {var.name} }}" + # pylint: disable=protected-access + for variable_dependency in script._variables[var.name].variables: + dummy_format_string += f"{{ {variable_dependency.name} }}" + # pylint: enable=protected-access + return dummy_format_string + + def _validate_formatter( mock_script: Script, unresolved_variables: Set[str], formatter_validator: Union[StringFormatterValidator, OverridesStringFormatterValidator], ) -> None: - try: - unresolvable = unresolved_variables - if isinstance(formatter_validator, OverridesStringFormatterValidator): - unresolvable = unresolved_variables.union({VARIABLES.entry_metadata.variable_name}) + is_static_formatter = False + unresolvable = unresolved_variables + if isinstance(formatter_validator, OverridesStringFormatterValidator): + is_static_formatter = True + unresolvable = unresolved_variables.union({VARIABLES.entry_metadata.variable_name}) + parsed = parse( + text=formatter_validator.format_string, + ) + variable_names = {var.name for var in parsed.variables} + custom_function_names = {f"%{func.name}" for func in parsed.custom_functions} + + if not variable_names.issubset(mock_script.variable_names): + raise StringFormattingVariableNotFoundException( + "contains the following variables that do not exist: " + f"{', '.join(sorted(variable_names - mock_script.variable_names))}" + ) + if not custom_function_names.issubset(mock_script.function_names): + raise StringFormattingVariableNotFoundException( + "contains the following custom functions that do not exist: " + f"{', '.join(sorted(custom_function_names - mock_script.function_names))}" + ) + if unresolved := variable_names.intersection(unresolvable): + raise StringFormattingVariableNotFoundException( + "contains the following variables that are unresolved when executing this " + f"formatter: {', '.join(sorted(unresolved))}" + ) + try: mock_script.resolve_once( - {"tmp_var": formatter_validator.format_string}, + { + "tmp_var": to_variable_dependency_format_string( + script=mock_script, parsed_format_string=parsed + ) + }, unresolvable=unresolvable, ) - except VariableDoesNotExist as exc: + except RuntimeException as exc: + if isinstance(exc, ScriptVariableNotResolved) and is_static_formatter: + raise StringFormattingVariableNotFoundException( + "static formatters must contain variables that have no dependency to " + "entry variables" + ) from exc raise StringFormattingVariableNotFoundException(exc) from exc diff --git a/tests/conftest.py b/tests/conftest.py index b7ec75ff..522ddb3b 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -113,7 +113,7 @@ def assert_logs( for call_args in patched_debug.call_args_list: occurrences += int(expected_message in call_args.args[0]) - if expected_occurrences: + if expected_occurrences is not None: assert ( occurrences == expected_occurrences ), f"{expected_message} was expected {expected_occurrences} times, got {occurrences}" diff --git a/tests/e2e/plugins/test_audio_extract.py b/tests/e2e/plugins/test_audio_extract.py index 08315486..90be1603 100644 --- a/tests/e2e/plugins/test_audio_extract.py +++ b/tests/e2e/plugins/test_audio_extract.py @@ -5,25 +5,6 @@ from expected_transaction_log import assert_transaction_log_matches from ytdl_sub.subscriptions.subscription import Subscription -@pytest.fixture -def single_preset_dict_old_format(output_directory): - return { - "preset": "Single", - # test multi-tags - "music_tags": {"embed_thumbnail": True, "tags": {"genres": ["multi_tag_1", "multi_tag_2"]}}, - "format": "worst[ext=mp4]", - "audio_extract": {"codec": "mp3", "quality": 320}, - "ytdl_options": { - "postprocessor_args": {"ffmpeg": ["-bitexact"]}, # Must add this for reproducibility - }, - "overrides": { - "track_artist": "YouTube", - "url": "https://www.youtube.com/watch?v=2lAe1cqCOXo", - "music_directory": output_directory, - }, - } - - @pytest.fixture def single_preset_dict(output_directory): return { @@ -32,7 +13,6 @@ def single_preset_dict(output_directory): "music_tags": {"genres": ["multi_tag_1", "multi_tag_2"]}, # test the new embed_thumbnail plugin "embed_thumbnail": True, - "format": "worst[ext=mp4]", "audio_extract": {"codec": "mp3", "quality": 320}, "ytdl_options": { "postprocessor_args": {"ffmpeg": ["-bitexact"]}, # Must add this for reproducibility @@ -55,7 +35,6 @@ def youtube_release_preset_dict(output_directory): return { "preset": "YouTube Releases", "audio_extract": {"codec": "vorbis", "quality": 140}, - "format": "worst[ext=mp4]", # download the worst format so it is fast "ytdl_options": { "postprocessor_args": {"ffmpeg": ["-bitexact"]}, # Must add this for reproducibility }, @@ -68,32 +47,6 @@ def youtube_release_preset_dict(output_directory): class TestAudioExtract: - @pytest.mark.parametrize("dry_run", [True, False]) - def test_audio_extract_single_song_old_format( - self, - default_config, - single_preset_dict_old_format, - output_directory, - dry_run, - ): - subscription = Subscription.from_dict( - config=default_config, - preset_name="single_song_test", - preset_dict=single_preset_dict_old_format, - ) - - transaction_log = subscription.download(dry_run=dry_run) - assert_transaction_log_matches( - output_directory=output_directory, - transaction_log=transaction_log, - transaction_log_summary_file_name="plugins/test_audio_extract_single_old_format.txt", - ) - assert_expected_downloads( - output_directory=output_directory, - dry_run=dry_run, - expected_download_summary_file_name="plugins/test_audio_extract_single_old_format.json", - ) - @pytest.mark.parametrize("dry_run", [False]) def test_audio_extract_single_song( self, diff --git a/tests/e2e/plugins/test_date_range.py b/tests/e2e/plugins/test_date_range.py index 4196d567..e803fa36 100644 --- a/tests/e2e/plugins/test_date_range.py +++ b/tests/e2e/plugins/test_date_range.py @@ -40,20 +40,30 @@ def rolling_recent_channel_preset_dict(recent_preset_dict): class TestDateRange: @pytest.mark.parametrize("dry_run", [True, False]) + @pytest.mark.parametrize("date_range_breaks", [True, False]) def test_recent_channel_download( self, recent_preset_dict, tv_show_config, output_directory, - dry_run, + dry_run: bool, + date_range_breaks: bool, ): + recent_preset_dict["date_range"]["breaks"] = date_range_breaks recent_channel_subscription = Subscription.from_dict( config=tv_show_config, preset_name="recent", preset_dict=recent_preset_dict, ) - transaction_log = recent_channel_subscription.download(dry_run=dry_run) + with assert_logs( + logger=YTDLP.logger, + expected_message="RejectedVideoReached, stopping additional downloads", + log_level="debug", + expected_occurrences=1 if date_range_breaks else 0, + ): + transaction_log = recent_channel_subscription.download(dry_run=dry_run) + assert_transaction_log_matches( output_directory=output_directory, transaction_log=transaction_log, diff --git a/tests/e2e/youtube/test_channel.py b/tests/e2e/youtube/test_channel.py index 2b36cf9b..4ed9fe20 100644 --- a/tests/e2e/youtube/test_channel.py +++ b/tests/e2e/youtube/test_channel.py @@ -18,7 +18,6 @@ def channel_preset_dict(output_directory): "format": "worst[ext=mp4]", # download the worst format so it is fast "ytdl_options": { "max_views": 100000, # do not download the popular PJ concert - "break_on_reject": False, # do not break from max views }, "subtitles": { "subtitles_name": "{episode_file_path}.{lang}.{subtitles_ext}", diff --git a/tests/e2e/youtube/test_playlist.py b/tests/e2e/youtube/test_playlist.py index 7d8eca5e..3c8e142f 100644 --- a/tests/e2e/youtube/test_playlist.py +++ b/tests/e2e/youtube/test_playlist.py @@ -199,3 +199,26 @@ class TestPlaylist: dry_run=dry_run, expected_download_summary_file_name="youtube/test_playlist.json", ) + + def test_playlist_download_from_cli_sub_with_override_arg( + self, + preset_dict_to_subscription_yaml_generator, + playlist_preset_dict, + output_directory, + ): + # TODO: Fix CLI parsing on windows when dealing with spaces + if IS_WINDOWS: + return + + # No config needed when using only prebuilt presets + with preset_dict_to_subscription_yaml_generator( + subscription_name="music_video_playlist_test", preset_dict=playlist_preset_dict + ) as subscription_path: + args = ( + f"--dry-run sub '{subscription_path}' --dl-override '--date_range.after 20240101'" + ) + + subscriptions = mock_run_from_cli(args=args) + + assert len(subscriptions) == 1 + assert subscriptions[0].transaction_log.is_empty diff --git a/tests/e2e/youtube/test_video.py b/tests/e2e/youtube/test_video.py index f2c78bb8..9faf44b2 100644 --- a/tests/e2e/youtube/test_video.py +++ b/tests/e2e/youtube/test_video.py @@ -14,30 +14,6 @@ from ytdl_sub.utils.system import IS_WINDOWS from ytdl_sub.utils.thumbnail import try_convert_download_thumbnail -@pytest.fixture -def single_video_preset_dict_old_video_tags_format(output_directory): - return { - "preset": "Jellyfin Music Videos", - "download": "https://youtube.com/watch?v=HKTNxEqsN3Q", - # override the output directory with our fixture-generated dir - "output_options": { - "maintain_download_archive": False, - }, - "embed_thumbnail": True, # embed thumb into the video - "format": "worst[ext=mp4]", # download the worst format so it is fast - # also test video tags - "video_tags": { - "tags": { - "title": "{title}", - } - }, - "overrides": { - "music_video_artist": "JMC", - "music_video_directory": output_directory, - }, - } - - @pytest.fixture def single_video_preset_dict(output_directory): return { @@ -100,25 +76,6 @@ def single_video_preset_dict_dl_args(single_video_preset_dict): class TestYoutubeVideo: - def test_single_video_old_video_tags_format_download( - self, - default_config, - single_video_preset_dict_old_video_tags_format, - output_directory, - ): - single_video_subscription = Subscription.from_dict( - config=default_config, - preset_name="music_video_single_video_test", - preset_dict=single_video_preset_dict_old_video_tags_format, - ) - - transaction_log = single_video_subscription.download(dry_run=True) - assert_transaction_log_matches( - output_directory=output_directory, - transaction_log=transaction_log, - transaction_log_summary_file_name="youtube/test_video.txt", - ) - @pytest.mark.parametrize("dry_run", [True, False]) def test_single_video_download( self, diff --git a/tests/resources/expected_downloads_summaries/plugins/chapters/test_chapters_from_comments.json b/tests/resources/expected_downloads_summaries/plugins/chapters/test_chapters_from_comments.json index 1ff13886..49314662 100644 --- a/tests/resources/expected_downloads_summaries/plugins/chapters/test_chapters_from_comments.json +++ b/tests/resources/expected_downloads_summaries/plugins/chapters/test_chapters_from_comments.json @@ -1,6 +1,6 @@ { ".ytdl-sub-chapters_from_comments-download-archive.json": "2510b2ff3c54aa4813a4f23ea079e1ec", "JMC/Move 78 - Automated Improvisation [Full Album].jpg": "c12e6a6f242680d1096a1a99d74a62c6", - "JMC/Move 78 - Automated Improvisation [Full Album].mp4": "f401b98c332b76ee1c87065e195d73ce", + "JMC/Move 78 - Automated Improvisation [Full Album].mp4": "068526b2d8f85fdcf914df3e23d0b1fa", "JMC/Move 78 - Automated Improvisation [Full Album].nfo": "039268e97673a6f2b391772ec3b52fac" } \ No newline at end of file diff --git a/tests/resources/expected_downloads_summaries/plugins/match_filters/test_match_filters_partial.json b/tests/resources/expected_downloads_summaries/plugins/match_filters/test_match_filters_partial.json index 15b2036d..2a6691fb 100644 --- a/tests/resources/expected_downloads_summaries/plugins/match_filters/test_match_filters_partial.json +++ b/tests/resources/expected_downloads_summaries/plugins/match_filters/test_match_filters_partial.json @@ -1,6 +1,6 @@ { ".ytdl-sub-match_filter_test-download-archive.json": "30f10646149d9eea4eb970749f352f7d", "match_filter_test/Jesse's Minecraft Server [Trailer - Mar.21].jpg": "e7830aa8a64b0cde65ba3f7e5fc56530", - "match_filter_test/Jesse's Minecraft Server [Trailer - Mar.21].mp4": "70204418e9af11a696611aa19571cf3a", + "match_filter_test/Jesse's Minecraft Server [Trailer - Mar.21].mp4": "f17a540070964a199b35f981561d94e4", "match_filter_test/Jesse's Minecraft Server [Trailer - Mar.21].nfo": "d85f4500bb5d8a2425d734a23b5a944c" } \ No newline at end of file diff --git a/tests/resources/expected_downloads_summaries/plugins/split_by_chapters_video.json b/tests/resources/expected_downloads_summaries/plugins/split_by_chapters_video.json index 5ef02a47..07adaebb 100644 --- a/tests/resources/expected_downloads_summaries/plugins/split_by_chapters_video.json +++ b/tests/resources/expected_downloads_summaries/plugins/split_by_chapters_video.json @@ -1,15 +1,15 @@ { ".ytdl-sub-Proved Records-download-archive.json": "c3fb0b4f31caaa10ac7954ea93da33c4", - "Proved Records/[2017] Alfa Mist - Nocturne [Full Album]/01 - 01. Intro (Feat. Racheal Ofori & Barney Artist).mp3": "29d09da874133ce5c5f7459c2fa6cd85", - "Proved Records/[2017] Alfa Mist - Nocturne [Full Album]/02 - 02. Answers (Feat. Rick David & Kaya Thomas - Dyke).mp3": "d5432e6a4f7af9810b0e7c8eebe4898a", - "Proved Records/[2017] Alfa Mist - Nocturne [Full Album]/03 - 03. Blaze (Feat. Kaya Thomas - Dyke).mp3": "cf03da6688a73373f9295dc595156e11", - "Proved Records/[2017] Alfa Mist - Nocturne [Full Album]/04 - 04. What If (Interlude).mp3": "b7b943b6f3395c05433b8532364d63d6", - "Proved Records/[2017] Alfa Mist - Nocturne [Full Album]/05 - 05. No Peace (Feat. Tom Misch).mp3": "0d9719268900d4abcd8c0f51ad3af92c", - "Proved Records/[2017] Alfa Mist - Nocturne [Full Album]/06 - 06. Closer (Feat. Lester Duval).mp3": "bbd9ec616a0f7d3c5d13c04bb118681e", - "Proved Records/[2017] Alfa Mist - Nocturne [Full Album]/07 - 07. Delusions： Rumination (Interlude) (Feat. Racheal Ofori).mp3": "2e59ecdc0f8050bcf190a7898d7ae968", - "Proved Records/[2017] Alfa Mist - Nocturne [Full Album]/08 - 08. Dreams (Feat. Carmody).mp3": "e58966f3326b876f0ab97459b681f76f", - "Proved Records/[2017] Alfa Mist - Nocturne [Full Album]/09 - 09. Dreaming (Interlude) (Feat. Racheal Ofori).mp3": "d6939215b73457fce4e026f8c9b57ecc", - "Proved Records/[2017] Alfa Mist - Nocturne [Full Album]/10 - 10. Hopeful (Feat. Jordan Rakei).mp3": "82d0e7d8070fd15d71c2e11c0285d426", - "Proved Records/[2017] Alfa Mist - Nocturne [Full Album]/11 - 11. Sunrise (Pillows) (Feat. Emmavie).mp3": "3edb11e8cd5f270dd42ee62d9a9aa4ff", + "Proved Records/[2017] Alfa Mist - Nocturne [Full Album]/01 - 01. Intro (Feat. Racheal Ofori & Barney Artist).mp3": "1e3583c9c1dc166b7baf98b81b4ca106", + "Proved Records/[2017] Alfa Mist - Nocturne [Full Album]/02 - 02. Answers (Feat. Rick David & Kaya Thomas - Dyke).mp3": "bb4c9085a4345515e6dffcf2c09f0f0f", + "Proved Records/[2017] Alfa Mist - Nocturne [Full Album]/03 - 03. Blaze (Feat. Kaya Thomas - Dyke).mp3": "c6766f80d0f9993ec6051555cd37dc32", + "Proved Records/[2017] Alfa Mist - Nocturne [Full Album]/04 - 04. What If (Interlude).mp3": "2139e6eedc4d1bc2025f8ef78b0bd2af", + "Proved Records/[2017] Alfa Mist - Nocturne [Full Album]/05 - 05. No Peace (Feat. Tom Misch).mp3": "8194d6a4018121e40b1733a0fa7a382e", + "Proved Records/[2017] Alfa Mist - Nocturne [Full Album]/06 - 06. Closer (Feat. Lester Duval).mp3": "e244183723e23c65156076b5f7ffcf56", + "Proved Records/[2017] Alfa Mist - Nocturne [Full Album]/07 - 07. Delusions： Rumination (Interlude) (Feat. Racheal Ofori).mp3": "692224c169be4735c0892e05726cb335", + "Proved Records/[2017] Alfa Mist - Nocturne [Full Album]/08 - 08. Dreams (Feat. Carmody).mp3": "561c339d4d1232c15ad38a5afe8d61c3", + "Proved Records/[2017] Alfa Mist - Nocturne [Full Album]/09 - 09. Dreaming (Interlude) (Feat. Racheal Ofori).mp3": "bdef7ad26848b45552891b9ea722659e", + "Proved Records/[2017] Alfa Mist - Nocturne [Full Album]/10 - 10. Hopeful (Feat. Jordan Rakei).mp3": "ffff4506702e708288eb54eff8fbaca4", + "Proved Records/[2017] Alfa Mist - Nocturne [Full Album]/11 - 11. Sunrise (Pillows) (Feat. Emmavie).mp3": "41a88d6c07047bb4215fed0e71443f17", "Proved Records/[2017] Alfa Mist - Nocturne [Full Album]/folder.jpg": "bd3685acc53072e591bae2505ecb0648" } \ No newline at end of file diff --git a/tests/resources/expected_downloads_summaries/plugins/split_by_chapters_with_regex_no_chapters_video_pass.txt b/tests/resources/expected_downloads_summaries/plugins/split_by_chapters_with_regex_no_chapters_video_pass.txt index 4b7fb2b5..97f6d053 100644 --- a/tests/resources/expected_downloads_summaries/plugins/split_by_chapters_with_regex_no_chapters_video_pass.txt +++ b/tests/resources/expected_downloads_summaries/plugins/split_by_chapters_with_regex_no_chapters_video_pass.txt @@ -1,5 +1,5 @@ { ".ytdl-sub-split_by_chapters_with_regex_video_no_chapters-download-archive.json": "4008e43668447f1a3a6a55520a6ff475", - "Project Zombie/[2010] Oblivion Mod ＂Falcor＂ p.1/01 - Oblivion Mod ＂Falcor＂ p.1.mp3": "d53121df33ac8c4a4699ec8919196552", + "Project Zombie/[2010] Oblivion Mod ＂Falcor＂ p.1/01 - Oblivion Mod ＂Falcor＂ p.1.mp3": "b886f268a2a9b3b62f528fcf46699082", "Project Zombie/[2010] Oblivion Mod ＂Falcor＂ p.1/folder.jpg": "fb95b510681676e81c321171fc23143e" } \ No newline at end of file diff --git a/tests/resources/expected_downloads_summaries/plugins/split_by_chapters_with_regex_video.json b/tests/resources/expected_downloads_summaries/plugins/split_by_chapters_with_regex_video.json index 653c1f0d..c57034be 100644 --- a/tests/resources/expected_downloads_summaries/plugins/split_by_chapters_with_regex_video.json +++ b/tests/resources/expected_downloads_summaries/plugins/split_by_chapters_with_regex_video.json @@ -1,15 +1,15 @@ { ".ytdl-sub-split_by_chapters_with_regex_video_preset-download-archive.json": "9798e8289742586d0efd295a97c6c906", - "Alfa Mist/[2017] Nocturne/01 - Intro (Feat. Racheal Ofori & Barney Artist).mp3": "7be57c4dde9ba2c3fa72e9cc65b0f586", - "Alfa Mist/[2017] Nocturne/02 - Answers (Feat. Rick David & Kaya Thomas - Dyke).mp3": "5ee032da9965c51913fcfa0319d061bd", - "Alfa Mist/[2017] Nocturne/03 - Blaze (Feat. Kaya Thomas - Dyke).mp3": "82c1f35bdccfb4ec146f33661ac94d6b", - "Alfa Mist/[2017] Nocturne/04 - What If (Interlude).mp3": "040b38c249e478d8832dcea8bfeca17f", - "Alfa Mist/[2017] Nocturne/05 - No Peace (Feat. Tom Misch).mp3": "06f7e57fecdc7b4d2e37c7652791ce19", - "Alfa Mist/[2017] Nocturne/06 - Closer (Feat. Lester Duval).mp3": "91a03449c33bf9dcff5a9654a8525626", - "Alfa Mist/[2017] Nocturne/07 - Delusions： Rumination (Interlude) (Feat. Racheal Ofori).mp3": "54c26621b8d4e74f37217c836479a077", - "Alfa Mist/[2017] Nocturne/08 - Dreams (Feat. Carmody).mp3": "2704327ac5998086e77a77da164e2afd", - "Alfa Mist/[2017] Nocturne/09 - Dreaming (Interlude) (Feat. Racheal Ofori).mp3": "75b8a26d3099aa2d9ba488b33c4eb1e7", - "Alfa Mist/[2017] Nocturne/10 - Hopeful (Feat. Jordan Rakei).mp3": "0bf0979c99c55ef986efaba55c0dc858", - "Alfa Mist/[2017] Nocturne/11 - Sunrise (Pillows) (Feat. Emmavie).mp3": "78950fbfe459de8e9aa951ab6fa153bf", + "Alfa Mist/[2017] Nocturne/01 - Intro (Feat. Racheal Ofori & Barney Artist).mp3": "d07f90d214416a11736e3f73c0955208", + "Alfa Mist/[2017] Nocturne/02 - Answers (Feat. Rick David & Kaya Thomas - Dyke).mp3": "9057d3d7fcd98701e06d399a507c0e5b", + "Alfa Mist/[2017] Nocturne/03 - Blaze (Feat. Kaya Thomas - Dyke).mp3": "4cc4e7066112388e83bdd9e803e8051a", + "Alfa Mist/[2017] Nocturne/04 - What If (Interlude).mp3": "8d0f04c04f8c2b0565954dac66ee29a3", + "Alfa Mist/[2017] Nocturne/05 - No Peace (Feat. Tom Misch).mp3": "e8166110c85e6b9af8d1067b8ffbb428", + "Alfa Mist/[2017] Nocturne/06 - Closer (Feat. Lester Duval).mp3": "64e39199ddcd05b96c15c82a3c695d25", + "Alfa Mist/[2017] Nocturne/07 - Delusions： Rumination (Interlude) (Feat. Racheal Ofori).mp3": "9a53bee41d1985e48e771a02b052986b", + "Alfa Mist/[2017] Nocturne/08 - Dreams (Feat. Carmody).mp3": "19c50e84c08eacdd1095e461306d9077", + "Alfa Mist/[2017] Nocturne/09 - Dreaming (Interlude) (Feat. Racheal Ofori).mp3": "f28df20fdbadf0ec4985b5f1f30c4e5d", + "Alfa Mist/[2017] Nocturne/10 - Hopeful (Feat. Jordan Rakei).mp3": "d7261e48713426ad4905919915a9b103", + "Alfa Mist/[2017] Nocturne/11 - Sunrise (Pillows) (Feat. Emmavie).mp3": "5d51491d0999e1f29918d4db4ed796d3", "Alfa Mist/[2017] Nocturne/folder.jpg": "bd3685acc53072e591bae2505ecb0648" } \ No newline at end of file diff --git a/tests/resources/expected_downloads_summaries/plugins/test_audio_extract_playlist.json b/tests/resources/expected_downloads_summaries/plugins/test_audio_extract_playlist.json index f077d34c..662e2f86 100644 --- a/tests/resources/expected_downloads_summaries/plugins/test_audio_extract_playlist.json +++ b/tests/resources/expected_downloads_summaries/plugins/test_audio_extract_playlist.json @@ -1,7 +1,7 @@ { ".ytdl-sub-multiple_songs_test-download-archive.json": "54237df5e00d1598dfd39f341ee03d75", - "Project Zombie/[2011] Jesse's Minecraft Server/01 - Jesse's Minecraft Server [Trailer - Mar.21].ogg": "5657c5b92f8980b20d8bbee0fdc7e5d8", - "Project Zombie/[2011] Jesse's Minecraft Server/02 - Jesse's Minecraft Server [Trailer - Feb.27].ogg": "a2a3a34e02e26a6c0265530d4499473b", - "Project Zombie/[2011] Jesse's Minecraft Server/03 - Jesse's Minecraft Server [Trailer - Feb.1].ogg": "0a385da3aa06b994a69b8ab812b44975", + "Project Zombie/[2011] Jesse's Minecraft Server/01 - Jesse's Minecraft Server [Trailer - Mar.21].ogg": "e7a8a94ebe9f02f086f4bbf3df3946f6", + "Project Zombie/[2011] Jesse's Minecraft Server/02 - Jesse's Minecraft Server [Trailer - Feb.27].ogg": "aeea4b086507fd7fde3c09c0bd868950", + "Project Zombie/[2011] Jesse's Minecraft Server/03 - Jesse's Minecraft Server [Trailer - Feb.1].ogg": "80200d21a46c7f521bfb23aef2f87e34", "Project Zombie/[2011] Jesse's Minecraft Server/folder.jpg": "e7830aa8a64b0cde65ba3f7e5fc56530" } \ No newline at end of file diff --git a/tests/resources/expected_downloads_summaries/plugins/test_audio_extract_single.json b/tests/resources/expected_downloads_summaries/plugins/test_audio_extract_single.json index e0035559..180cc879 100644 --- a/tests/resources/expected_downloads_summaries/plugins/test_audio_extract_single.json +++ b/tests/resources/expected_downloads_summaries/plugins/test_audio_extract_single.json @@ -1,5 +1,5 @@ { ".ytdl-sub-single_song_test-download-archive.json": "c8ff22ec3304c9f8dab18cedaed4e8b4", - "YouTube/[2019] YouTube Rewind 2019： For the Record ｜ #YouTubeRewind/01 - YouTube Rewind 2019： For the Record ｜ #YouTubeRewind.mp3": "991b0eb62683c2194c5bdfa9a61ab18e", + "YouTube/[2019] YouTube Rewind 2019： For the Record ｜ #YouTubeRewind/01 - YouTube Rewind 2019： For the Record ｜ #YouTubeRewind.mp3": "829eb7dcc5dcae41240701dec4e1708d", "YouTube/[2019] YouTube Rewind 2019： For the Record ｜ #YouTubeRewind/folder.jpg": "50ee47c80f679029f5d3503bb91b045a" } \ No newline at end of file diff --git a/tests/resources/expected_downloads_summaries/plugins/test_audio_extract_single_best.json b/tests/resources/expected_downloads_summaries/plugins/test_audio_extract_single_best.json index 11201b53..25c1b7e9 100644 --- a/tests/resources/expected_downloads_summaries/plugins/test_audio_extract_single_best.json +++ b/tests/resources/expected_downloads_summaries/plugins/test_audio_extract_single_best.json @@ -1,5 +1,5 @@ { - ".ytdl-sub-single_song_best_test-download-archive.json": "f179ccfe0a9b3a76ae62122a3ccb58fd", - "YouTube/[2019] YouTube Rewind 2019： For the Record ｜ #YouTubeRewind/01 - YouTube Rewind 2019： For the Record ｜ #YouTubeRewind.m4a": "114b35c29df84146cd98207f807b837b", + ".ytdl-sub-single_song_best_test-download-archive.json": "0f9484ed868dcbeef82810a3cf7b0eea", + "YouTube/[2019] YouTube Rewind 2019： For the Record ｜ #YouTubeRewind/01 - YouTube Rewind 2019： For the Record ｜ #YouTubeRewind.opus": "73e4afdda9bc792807c8b07a63128e5a", "YouTube/[2019] YouTube Rewind 2019： For the Record ｜ #YouTubeRewind/folder.jpg": "50ee47c80f679029f5d3503bb91b045a" } \ No newline at end of file diff --git a/tests/resources/expected_downloads_summaries/plugins/test_audio_extract_single_old_format.json b/tests/resources/expected_downloads_summaries/plugins/test_audio_extract_single_old_format.json deleted file mode 100644 index 426abd6c..00000000 --- a/tests/resources/expected_downloads_summaries/plugins/test_audio_extract_single_old_format.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - ".ytdl-sub-single_song_test-download-archive.json": "c8ff22ec3304c9f8dab18cedaed4e8b4", - "YouTube/[2019] YouTube Rewind 2019： For the Record ｜ #YouTubeRewind/01 - YouTube Rewind 2019： For the Record ｜ #YouTubeRewind.mp3": "11376667a11bb71565b520f8ce5fa303", - "YouTube/[2019] YouTube Rewind 2019： For the Record ｜ #YouTubeRewind/folder.jpg": "50ee47c80f679029f5d3503bb91b045a" -} \ No newline at end of file diff --git a/tests/resources/expected_downloads_summaries/plugins/test_chapters_sb_and_embedded_subs.json b/tests/resources/expected_downloads_summaries/plugins/test_chapters_sb_and_embedded_subs.json index f6777ba6..03a1f924 100644 --- a/tests/resources/expected_downloads_summaries/plugins/test_chapters_sb_and_embedded_subs.json +++ b/tests/resources/expected_downloads_summaries/plugins/test_chapters_sb_and_embedded_subs.json @@ -1,6 +1,6 @@ { ".ytdl-sub-sponsorblock_with_embedded_subs_test-download-archive.json": "2cb4b9586fd5bb7f1fed76ed9195e6e4", "JMC/This GPU SLIDES into this Case! - Silverstone SUGO 16 ITX Case.jpg": "b5353a824a4800cc26f884e3025ed969", - "JMC/This GPU SLIDES into this Case! - Silverstone SUGO 16 ITX Case.mp4": "8c12640f0c5c280c7a77423431b4ecb1", + "JMC/This GPU SLIDES into this Case! - Silverstone SUGO 16 ITX Case.mp4": "526b6df52a8aaf11dfe56f25ac35a567", "JMC/This GPU SLIDES into this Case! - Silverstone SUGO 16 ITX Case.nfo": "ae73ec18a9f0e5a54c90061ccd32e7f4" } \ No newline at end of file diff --git a/tests/resources/expected_downloads_summaries/youtube/test_channel_full.json b/tests/resources/expected_downloads_summaries/youtube/test_channel_full.json index 74e55e90..386defa9 100644 --- a/tests/resources/expected_downloads_summaries/youtube/test_channel_full.json +++ b/tests/resources/expected_downloads_summaries/youtube/test_channel_full.json @@ -18,7 +18,7 @@ "Project ⧸ Zombie/Season 2011/s2011.e022701 - Jesse's Minecraft Server [Trailer - Feb.27].nfo": "20af231e30a035fb2bc0c946f4b4026d", "Project ⧸ Zombie/Season 2011/s2011.e032101 - Jesse's Minecraft Server [Trailer - Mar.21]-thumb.jpg": "e7830aa8a64b0cde65ba3f7e5fc56530", "Project ⧸ Zombie/Season 2011/s2011.e032101 - Jesse's Minecraft Server [Trailer - Mar.21].info.json": "INFO_JSON", - "Project ⧸ Zombie/Season 2011/s2011.e032101 - Jesse's Minecraft Server [Trailer - Mar.21].mp4": "5f12b36e5ce717fa9550000237e2e5b8", + "Project ⧸ Zombie/Season 2011/s2011.e032101 - Jesse's Minecraft Server [Trailer - Mar.21].mp4": "eac3dfce44c2a723f2fc74e9552aa510", "Project ⧸ Zombie/Season 2011/s2011.e032101 - Jesse's Minecraft Server [Trailer - Mar.21].nfo": "f8bdc463c0cb2ffdc82aba2bcc27ad5d", "Project ⧸ Zombie/Season 2011/s2011.e052901 - Project Zombie ｜Official Trailer｜ (IP： mc.projectzombie.beastnode.net)-thumb.jpg": "c956192a379b3661595c9920972d4819", "Project ⧸ Zombie/Season 2011/s2011.e052901 - Project Zombie ｜Official Trailer｜ (IP： mc.projectzombie.beastnode.net).info.json": "INFO_JSON", @@ -30,7 +30,7 @@ "Project ⧸ Zombie/Season 2011/s2011.e063001 - Project Zombie ｜Fin｜.nfo": "54ea4a48116aa98480a79495036c25e9", "Project ⧸ Zombie/Season 2011/s2011.e112101 - Skyrim 'Ultra HD w⧸Mods' [PC]-thumb.jpg": "1718599d5189c65f7d8cf6acfa5ea851", "Project ⧸ Zombie/Season 2011/s2011.e112101 - Skyrim 'Ultra HD w⧸Mods' [PC].info.json": "INFO_JSON", - "Project ⧸ Zombie/Season 2011/s2011.e112101 - Skyrim 'Ultra HD w⧸Mods' [PC].mp4": "2a18cc4baa198edecdc5154082f73eff", + "Project ⧸ Zombie/Season 2011/s2011.e112101 - Skyrim 'Ultra HD w⧸Mods' [PC].mp4": "554d46311112228c22ca9532007803cd", "Project ⧸ Zombie/Season 2011/s2011.e112101 - Skyrim 'Ultra HD w⧸Mods' [PC].nfo": "04f6aad56d85b1f65b81b6b8000f6479", "Project ⧸ Zombie/Season 2012/s2012.e012301 - Project Zombie ｜Map Trailer｜-thumb.jpg": "54ebe9df801b278fdd17b21afa8373a6", "Project ⧸ Zombie/Season 2012/s2012.e012301 - Project Zombie ｜Map Trailer｜.info.json": "INFO_JSON", diff --git a/tests/resources/expected_downloads_summaries/youtube/test_playlist.json b/tests/resources/expected_downloads_summaries/youtube/test_playlist.json index c4fb6cca..1aa4510f 100644 --- a/tests/resources/expected_downloads_summaries/youtube/test_playlist.json +++ b/tests/resources/expected_downloads_summaries/youtube/test_playlist.json @@ -10,8 +10,10 @@ "JMC/Season 01/s01.e11022701 - Jesse's Minecraft Server [Trailer - Feb.27].nfo": "da7645e8826586388ae0d8278ef6a1c1", "JMC/Season 01/s01.e11032101 - Jesse's Minecraft Server [Trailer - Mar.21]-thumb.jpg": "e7830aa8a64b0cde65ba3f7e5fc56530", "JMC/Season 01/s01.e11032101 - Jesse's Minecraft Server [Trailer - Mar.21].info.json": "INFO_JSON", - "JMC/Season 01/s01.e11032101 - Jesse's Minecraft Server [Trailer - Mar.21].mp4": "7f1a1d5c1d94938a9b6f565e8159c3d6", + "JMC/Season 01/s01.e11032101 - Jesse's Minecraft Server [Trailer - Mar.21].mp4": "dc04853870bbb811f6b312e20b58253b", "JMC/Season 01/s01.e11032101 - Jesse's Minecraft Server [Trailer - Mar.21].nfo": "c56083e2f3545fa2cafc4d67cbfdacf8", + "JMC/fanart.jpg": "129c6639b47299bc48062f0365e670ee", + "JMC/poster.jpg": "5de28eea5a921a041452ab3ce1041f73", "JMC/season01-poster.jpg": "e7830aa8a64b0cde65ba3f7e5fc56530", "JMC/tvshow.nfo": "e92e4a2c01522dd9a9c3423f0f9304dc" } \ No newline at end of file diff --git a/tests/resources/expected_downloads_summaries/youtube/test_playlist_archive_migrated.json b/tests/resources/expected_downloads_summaries/youtube/test_playlist_archive_migrated.json index c5490aa6..49d30e58 100644 --- a/tests/resources/expected_downloads_summaries/youtube/test_playlist_archive_migrated.json +++ b/tests/resources/expected_downloads_summaries/youtube/test_playlist_archive_migrated.json @@ -10,8 +10,10 @@ "JMC/Season 01/s01.e11022701 - Jesse's Minecraft Server [Trailer - Feb.27].nfo": "da7645e8826586388ae0d8278ef6a1c1", "JMC/Season 01/s01.e11032101 - Jesse's Minecraft Server [Trailer - Mar.21]-thumb.jpg": "e7830aa8a64b0cde65ba3f7e5fc56530", "JMC/Season 01/s01.e11032101 - Jesse's Minecraft Server [Trailer - Mar.21].info.json": "INFO_JSON", - "JMC/Season 01/s01.e11032101 - Jesse's Minecraft Server [Trailer - Mar.21].mp4": "7f1a1d5c1d94938a9b6f565e8159c3d6", + "JMC/Season 01/s01.e11032101 - Jesse's Minecraft Server [Trailer - Mar.21].mp4": "dc04853870bbb811f6b312e20b58253b", "JMC/Season 01/s01.e11032101 - Jesse's Minecraft Server [Trailer - Mar.21].nfo": "c56083e2f3545fa2cafc4d67cbfdacf8", + "JMC/fanart.jpg": "129c6639b47299bc48062f0365e670ee", + "JMC/poster.jpg": "5de28eea5a921a041452ab3ce1041f73", "JMC/season01-poster.jpg": "e7830aa8a64b0cde65ba3f7e5fc56530", "JMC/tvshow.nfo": "e92e4a2c01522dd9a9c3423f0f9304dc" } \ No newline at end of file diff --git a/tests/resources/transaction_log_summaries/plugins/test_audio_extract_single_best.txt b/tests/resources/transaction_log_summaries/plugins/test_audio_extract_single_best.txt index 9b538e8d..07cb8132 100644 --- a/tests/resources/transaction_log_summaries/plugins/test_audio_extract_single_best.txt +++ b/tests/resources/transaction_log_summaries/plugins/test_audio_extract_single_best.txt @@ -3,7 +3,7 @@ Files created: {output_directory} .ytdl-sub-single_song_best_test-download-archive.json {output_directory}/YouTube/[2019] YouTube Rewind 2019： For the Record ｜ #YouTubeRewind - 01 - YouTube Rewind 2019： For the Record ｜ #YouTubeRewind.m4a + 01 - YouTube Rewind 2019： For the Record ｜ #YouTubeRewind.opus Music Tags: album: YouTube Rewind 2019: For the Record | #YouTubeRewind albumartist: YouTube diff --git a/tests/resources/transaction_log_summaries/plugins/test_audio_extract_single_old_format.txt b/tests/resources/transaction_log_summaries/plugins/test_audio_extract_single_old_format.txt deleted file mode 100644 index 9a8b14bd..00000000 --- a/tests/resources/transaction_log_summaries/plugins/test_audio_extract_single_old_format.txt +++ /dev/null @@ -1,18 +0,0 @@ -Files created: ----------------------------------------- -{output_directory} - .ytdl-sub-single_song_test-download-archive.json -{output_directory}/YouTube/[2019] YouTube Rewind 2019： For the Record ｜ #YouTubeRewind - 01 - YouTube Rewind 2019： For the Record ｜ #YouTubeRewind.mp3 - Embedded Thumbnail, Music Tags: - album: YouTube Rewind 2019: For the Record | #YouTubeRewind - albumartist: YouTube - albumartists: YouTube - artist: YouTube - artists: YouTube - genres: Unset - title: YouTube Rewind 2019: For the Record | #YouTubeRewind - track: 1 - tracktotal: 1 - year: 2019 - folder.jpg \ No newline at end of file diff --git a/tests/resources/transaction_log_summaries/youtube/test_playlist.txt b/tests/resources/transaction_log_summaries/youtube/test_playlist.txt index 3213a124..ae53c14d 100644 --- a/tests/resources/transaction_log_summaries/youtube/test_playlist.txt +++ b/tests/resources/transaction_log_summaries/youtube/test_playlist.txt @@ -2,6 +2,8 @@ Files created: ---------------------------------------- {output_directory} .ytdl-sub-music_video_playlist_test-download-archive.json + fanart.jpg + poster.jpg season01-poster.jpg tvshow.nfo NFO tags: diff --git a/tests/unit/cli/test_entrypoint.py b/tests/unit/cli/test_entrypoint.py index 7a4c66e7..31c4c443 100644 --- a/tests/unit/cli/test_entrypoint.py +++ b/tests/unit/cli/test_entrypoint.py @@ -2,6 +2,7 @@ import re import sys from pathlib import Path from typing import Callable +from typing import List from unittest.mock import patch import pytest @@ -22,6 +23,7 @@ from ytdl_sub.utils.exceptions import ExperimentalFeatureNotEnabled @pytest.mark.parametrize("dry_run", [True, False]) @pytest.mark.parametrize("mock_success_output", [True, False]) @pytest.mark.parametrize("keep_successful_logs", [True, False]) +@pytest.mark.parametrize("match", [[], ["Rick", "Michael"]]) def test_subscription_logs_write_to_file( persist_logs_directory: str, persist_logs_config_factory: Callable, @@ -30,8 +32,11 @@ def test_subscription_logs_write_to_file( dry_run: bool, mock_success_output: bool, keep_successful_logs: bool, + match: List[str], ): - subscripton_names = ["Rick Astley", "Michael Jackson", "Eric Clapton"] + subscription_names = ["Rick Astley", "Michael Jackson", "Eric Clapton"] + if match: + subscription_names = ["Rick Astley", "Michael Jackson"] num_runs = 2 config = persist_logs_config_factory(keep_successful_logs=keep_successful_logs) @@ -47,6 +52,8 @@ def test_subscription_logs_write_to_file( _download_subscriptions_from_yaml_files( config=config, subscription_paths=subscription_paths, + subscription_matches=match, + subscription_override_dict={}, update_with_info_json=False, dry_run=dry_run, ) @@ -61,8 +68,8 @@ def test_subscription_logs_write_to_file( return # If not success, expect 2 log files for both sub errors elif not mock_success_output: - assert len(log_directory_files) == (num_runs * len(subscripton_names)) - for log_path, subscription_name in zip(log_directory_files, subscripton_names): + assert len(log_directory_files) == (num_runs * len(subscription_names)) + for log_path, subscription_name in zip(log_directory_files, subscription_names): subscription_log_file_name = subscription_name.lower().replace(" ", "_") assert bool(re.match(rf"\d\.{subscription_log_file_name}\.error\.log", log_path.name)) @@ -74,9 +81,9 @@ def test_subscription_logs_write_to_file( ) # If success and success logging, expect 3 log files else: - assert len(log_directory_files) == (num_runs * len(subscripton_names)) + assert len(log_directory_files) == (num_runs * len(subscription_names)) for log_file_path, subscription_name in zip( - log_directory_files, subscripton_names * num_runs + log_directory_files, subscription_names * num_runs ): subscription_log_file_name = subscription_name.lower().replace(" ", "_") diff --git a/tests/unit/config/test_preset.py b/tests/unit/config/test_preset.py index dff51a50..de651ada 100644 --- a/tests/unit/config/test_preset.py +++ b/tests/unit/config/test_preset.py @@ -17,9 +17,6 @@ class TestPreset: "youtube.com/watch?v=123abc", # single string ["youtube.com/watch?v=123abc", "youtube.com/watch?v=123xyz"], # list of strings [{"url": "youtube.com/watch?v=123abc"}, "youtube.com/watch?v=123abc"], # dict and str - # OLD download_strategy format - {"download_strategy": "url", "url": "youtube.com/watch?v=123abc"}, - {"download_strategy": "multi-url", "urls": [{"url": "youtube.com/watch?v=123abc"}]}, ], ) def test_bare_minimum_preset(self, config_file, output_options, download_value): @@ -129,7 +126,7 @@ class TestPreset: ): with pytest.raises( StringFormattingVariableNotFoundException, - match="Variable dne_var does not exist.", + match="contains the following variables that do not exist: dne_var", ): _ = Preset( config=config_file, @@ -145,7 +142,7 @@ class TestPreset: ): with pytest.raises( StringFormattingVariableNotFoundException, - match="Variable dne_var does not exist", + match="contains the following variables that do not exist: dne_var", ): _ = Preset( config=config_file, @@ -161,7 +158,7 @@ class TestPreset: ): with pytest.raises( StringFormattingVariableNotFoundException, - match="Variable dne_var does not exist", + match="contains the following variables that do not exist: dne_var", ): _ = Preset( config=config_file, @@ -182,7 +179,7 @@ class TestPreset: ): with pytest.raises( StringFormattingVariableNotFoundException, - match="Variable dne_var does not exist", + match="contains the following variables that do not exist: dne_var", ): _ = Preset( config=config_file, @@ -198,6 +195,26 @@ class TestPreset: }, ) + def test_preset_error__dict_override_variable_not_static( + self, config_file, output_options, youtube_video + ): + with pytest.raises( + StringFormattingVariableNotFoundException, + match="static formatters must contain variables that " + "have no dependency to entry variables", + ): + _ = Preset( + config=config_file, + name="test", + value={ + "download": youtube_video, + "output_options": { + "output_directory": "{title}", + "file_name": "{uid}", + }, + }, + ) + def test_preset_with_multi_url__contains_empty_url(self, config_file, output_options): _ = Preset( config=config_file, diff --git a/tests/unit/main/test_main.py b/tests/unit/main/test_main.py index a42e21a5..a88faa13 100644 --- a/tests/unit/main/test_main.py +++ b/tests/unit/main/test_main.py @@ -47,8 +47,8 @@ def test_main_exit_code(mock_sys_exit, return_code: int): main() assert mock_logger_cleanup.call_count == 1 - assert mock_logger_cleanup.call_args.kwargs["cleanup_error_log"] == ( - True if return_code == 0 else False + assert mock_logger_cleanup.call_args.kwargs["has_error"] == ( + True if return_code != 0 else False ) @@ -107,6 +107,58 @@ def test_args_after_sub_work(mock_sys_exit, tv_show_config_path): assert mock_sub.call_count == 1 assert mock_sub.call_args.kwargs["subscription_paths"] == ["subscriptions.yaml"] assert mock_sub.call_args.kwargs["config"]._name == tv_show_config_path + assert mock_sub.call_args.kwargs["subscription_matches"] == [] + assert Logger._LOGGER_LEVEL == LoggerLevels.VERBOSE + + +def test_sub_match_arguments_before(mock_sys_exit, tv_show_config_path): + with mock_sys_exit(expected_exit_code=0), patch.object( + sys, + "argv", + [ + "ytdl-sub", + "--match", + "testA", + "testB", + "-c", + tv_show_config_path, + "sub", + "--log-level", + "verbose", + ], + ), patch("ytdl_sub.cli.entrypoint._download_subscriptions_from_yaml_files") as mock_sub: + main() + + assert mock_sub.call_count == 1 + assert mock_sub.call_args.kwargs["subscription_paths"] == ["subscriptions.yaml"] + assert mock_sub.call_args.kwargs["config"]._name == tv_show_config_path + assert mock_sub.call_args.kwargs["subscription_matches"] == ["testA", "testB"] + assert Logger._LOGGER_LEVEL == LoggerLevels.VERBOSE + + +def test_sub_match_arguments_after_many(mock_sys_exit, tv_show_config_path): + with mock_sys_exit(expected_exit_code=0), patch.object( + sys, + "argv", + [ + "ytdl-sub", + "-c", + tv_show_config_path, + "sub", + "--log-level", + "verbose", + "--match", + "testA", + "--match", + "testB", + ], + ), patch("ytdl_sub.cli.entrypoint._download_subscriptions_from_yaml_files") as mock_sub: + main() + + assert mock_sub.call_count == 1 + assert mock_sub.call_args.kwargs["subscription_paths"] == ["subscriptions.yaml"] + assert mock_sub.call_args.kwargs["config"]._name == tv_show_config_path + assert mock_sub.call_args.kwargs["subscription_matches"] == ["testA", "testB"] assert Logger._LOGGER_LEVEL == LoggerLevels.VERBOSE diff --git a/tests/unit/plugins/test_throttle_protection.py b/tests/unit/plugins/test_throttle_protection.py index 0d72e9fd..9dcdb89b 100644 --- a/tests/unit/plugins/test_throttle_protection.py +++ b/tests/unit/plugins/test_throttle_protection.py @@ -1,3 +1,4 @@ +import pytest from conftest import assert_logs from ytdl_sub.plugins.throttle_protection import logger as throttle_protection_logger @@ -57,3 +58,59 @@ class TestThrottleProtectionPlugin: expected_occurrences=1, ): _ = subscription.download(dry_run=False) + + @pytest.mark.parametrize( + "disable_value", + [ + "", + False, + "{tp_bool_string}", + "{tp_empty_string}", + ], + ) + def test_disabled( + self, + config, + subscription_name, + output_directory, + mock_download_collection_entries, + disable_value, + ): + preset_dict = { + "preset": [ + "Kodi Music Videos", + ], + "overrides": { + "url": "https://your.name.here", + "music_video_directory": output_directory, + "tp_bool_string": "{ %bool(False) }", + "tp_empty_string": "", + }, + "throttle_protection": { + "enable": disable_value, + "sleep_per_download_s": { + "min": 0.01, + "max": 0.01, + }, + "sleep_per_subscription_s": { + "min": 0.02, + "max": 0.02, + }, + }, + } + + subscription = Subscription.from_dict( + config=config, + preset_name=subscription_name, + preset_dict=preset_dict, + ) + + with mock_download_collection_entries( + is_youtube_channel=False, num_urls=1, is_extracted_audio=False + ), assert_logs( + logger=throttle_protection_logger, + expected_message="Sleeping between downloads for %0.2f seconds", + log_level="debug", + expected_occurrences=0, + ): + _ = subscription.download(dry_run=False) diff --git a/tests/unit/script/functions/test_array_functions.py b/tests/unit/script/functions/test_array_functions.py index 06a3b150..bfee05da 100644 --- a/tests/unit/script/functions/test_array_functions.py +++ b/tests/unit/script/functions/test_array_functions.py @@ -26,6 +26,14 @@ class TestArrayFunctions: output = single_variable_output("{%array_at(['a', 'b', 'c'], 1)}") assert output == "b" + def test_array_at_default(self): + output = single_variable_output("{%array_at(['a', 'b', 'c'], 30, 'd')}") + assert output == "d" + + def test_array_at_error(self): + with pytest.raises(FunctionRuntimeException): + single_variable_output("{%array_at(['a', 'b', 'c'], 30)}") + def test_array_flatten(self): output = single_variable_output("{%array_flatten(['a', ['b'], [['c']]])}") assert output == ["a", "b", "c"] diff --git a/tests/unit/script/functions/test_conditional_functions.py b/tests/unit/script/functions/test_conditional_functions.py index f7217915..5cf04a18 100644 --- a/tests/unit/script/functions/test_conditional_functions.py +++ b/tests/unit/script/functions/test_conditional_functions.py @@ -1,6 +1,10 @@ +import re + import pytest from unit.script.conftest import single_variable_output +from ytdl_sub.script.utils.exceptions import FunctionRuntimeException + class TestConditionalFunction: @pytest.mark.parametrize( @@ -33,3 +37,67 @@ class TestConditionalFunction: }""" ) assert output == "winner" + + def test_elif_function(self): + output = single_variable_output( + """{ + %elif( + False, + "nope", + False, + "still nope", + True, + "yes", + "default value" + ) + }""" + ) + assert output == "yes" + + def test_elif_function_default_value(self): + output = single_variable_output( + """{ + %elif( + False, + "nope", + False, + "still nope", + False, + "will be default", + "default value" + ) + }""" + ) + assert output == "default value" + + def test_elif_function_errors_lt3(self): + with pytest.raises( + FunctionRuntimeException, + match=re.escape("elif requires at least 3 arguments"), + ): + single_variable_output( + """ + { + %elif( + False, + "only two args" + ) + }""" + ) + + def test_elif_function_errors_odd(self): + with pytest.raises( + FunctionRuntimeException, + match=re.escape("elif must have an odd number of arguments"), + ): + single_variable_output( + """ + { + %elif( + False, + "1", + False, + "even number args bad" + ) + }""" + ) diff --git a/tests/unit/script/functions/test_string_functions.py b/tests/unit/script/functions/test_string_functions.py index 9a48079a..cfa5175e 100644 --- a/tests/unit/script/functions/test_string_functions.py +++ b/tests/unit/script/functions/test_string_functions.py @@ -1,3 +1,6 @@ +from typing import List +from typing import Optional + import pytest from unit.script.conftest import single_variable_output @@ -114,3 +117,21 @@ class TestNumericFunctions: def test_contains(self, value, expected_output): output = single_variable_output(f"{{%contains('a brown dog', '{value}')}}") assert output == expected_output + + @pytest.mark.parametrize( + "input_string, split, max_split, expected_output", + [ + ("no splits", " | ", None, ["no splits"]), + ("one | split", " | ", None, ["one", "split"]), + ("max | split | one", " | ", 1, ["max", "split | one"]), + ], + ) + def test_split( + self, input_string: str, split: str, max_split: Optional[int], expected_output: List[str] + ): + if max_split: + output = single_variable_output(f"{{%split('{input_string}', '{split}', {max_split})}}") + else: + output = single_variable_output(f"{{%split('{input_string}', '{split}')}}") + + assert output == expected_output diff --git a/tests/unit/script/types/test_function.py b/tests/unit/script/types/test_function.py index 014e4a85..e13828f4 100644 --- a/tests/unit/script/types/test_function.py +++ b/tests/unit/script/types/test_function.py @@ -49,7 +49,11 @@ class TestFunction: @pytest.mark.parametrize( "function_str, expected_types, received_types", [ - ("{%array_at({'a': 'dict?'}, 1)}", "array: Array, idx: Integer", "Map, Integer"), + ( + "{%array_at({'a': 'dict?'}, 1)}", + "array: Array, idx: Integer, default: Optional[AnyArgument]", + "Map, Integer", + ), ("{%array_extend('not', 'array')}", "arrays: Array, ...", "String, String"), ( "{%replace('hi mom', 'mom', 'dad', 1, 0)}", diff --git a/tests/unit/utils/test_logger.py b/tests/unit/utils/test_logger.py index 5a75e5c3..97ce6bbd 100644 --- a/tests/unit/utils/test_logger.py +++ b/tests/unit/utils/test_logger.py @@ -111,8 +111,8 @@ class TestLogger: Logger.cleanup() assert not os.path.isfile(Logger._DEBUG_LOGGER_FILE.name) - @pytest.mark.parametrize("clean_error_log", [True, False]) - def test_logger_can_be_cleaned_during_execution(self, clean_error_log: bool): + @pytest.mark.parametrize("has_error", [True, False]) + def test_logger_can_be_cleaned_during_execution(self, has_error: bool): Logger._LOGGER_LEVEL = LoggerLevels.INFO logger = Logger.get(name="name_test") @@ -133,11 +133,11 @@ class TestLogger: except ValueError as exc: Logger.log_exception(exception=exc) - Logger.cleanup(cleanup_error_log=clean_error_log) + Logger.cleanup(has_error=has_error) assert not os.path.isfile(Logger.debug_log_filename()) - assert clean_error_log == (not os.path.isfile(Logger.error_log_filename())) - if not clean_error_log: + assert not has_error == (not os.path.isfile(Logger.error_log_filename())) + if has_error: with open(Logger.error_log_filename(), mode="r", encoding="utf-8") as err_file: err_logs = err_file.readlines() expected = [ diff --git a/tests/unit/utils/test_script_utils.py b/tests/unit/utils/test_script_utils.py index 8b2b0e90..d3ff8921 100644 --- a/tests/unit/utils/test_script_utils.py +++ b/tests/unit/utils/test_script_utils.py @@ -1,5 +1,6 @@ import copy +import pytest from unit.script.conftest import single_variable_output from ytdl_sub.utils.script import ScriptUtils @@ -31,3 +32,22 @@ class TestScriptUtils: output = single_variable_output(ScriptUtils.to_script(json_dict)) assert output == expected_output + + @pytest.mark.parametrize( + "input_str, expected_output", + [ + ("", False), + ("true", True), + ("false", False), + ("[ ]", False), + ("{ }", False), + ("True", True), + ("False", False), + ("lol not False", True), + ("0", False), + ("-1", True), + ("1", True), + ], + ) + def test_bool_formatter_output(self, input_str: str, expected_output: bool): + assert ScriptUtils.bool_formatter_output(input_str) == expected_output