From 287ac821dcd03c5e5f84ce5d02996960367f24ed Mon Sep 17 00:00:00 2001 From: Maria Mozgunova <63199273+MariaMozgunova@users.noreply.github.com> Date: Wed, 27 Jul 2022 19:52:25 +0300 Subject: [PATCH] [FEATURE] List support for ytdl-sub dl (#125) * List support for ytdl-sub dl (#103) * Add test and amend docstrings --- src/ytdl_sub/cli/download_args_parser.py | 114 ++++++++++++++++++-- src/ytdl_sub/utils/exceptions.py | 4 + tests/unit/cli/test_download_args_parser.py | 60 ++++++++++- 3 files changed, 165 insertions(+), 13 deletions(-) diff --git a/src/ytdl_sub/cli/download_args_parser.py b/src/ytdl_sub/cli/download_args_parser.py index 5f0fa3df..3931107f 100644 --- a/src/ytdl_sub/cli/download_args_parser.py +++ b/src/ytdl_sub/cli/download_args_parser.py @@ -1,12 +1,13 @@ import hashlib +import re import shlex -from argparse import ArgumentError from typing import Dict from typing import List from mergedeep import mergedeep from ytdl_sub.config.config_file import ConfigOptions +from ytdl_sub.utils.exceptions import InvalidDlArguments class DownloadArgsParser: @@ -30,13 +31,12 @@ class DownloadArgsParser: self._config_options = config_options @property - def _argument_exception(self) -> ArgumentError: + def _argument_exception(self) -> InvalidDlArguments: """ :return: Exception to raise if a parsing error occurs. """ - return ArgumentError( - argument=None, - message="dl arguments must be in the form of --subscription.option.name 'value'", + return InvalidDlArguments( + "dl arguments must be in the form of --subscription.option.name 'value'", ) @classmethod @@ -61,6 +61,46 @@ class DownloadArgsParser: """ return arg_name[2:] + @classmethod + def _uses_list(cls, argument_name: str) -> bool: + """ + :param argument_name: The argument name which might be using list + :return: True if the argument uses list, denoted by ending with '[]'. + False otherwise. + """ + pattern = re.compile(r"\[[1-9]\d*\]$") + + return bool(pattern.search(argument_name)) + + @classmethod + def _extract_list_index(cls, argument_name: str) -> int: + """ + :param argument_name: The argument name using list + :return: Splits the argument and returns the name and the list index. + """ + prefix, _, last = argument_name.rpartition("[") + + return prefix, int(last[:-1]) - 1 + + @classmethod + def _find_largest_consecutive(cls, indices: List[int]) -> int: + """ + :param indices: List of indices + :return: Largest integer n + 1 so that all the integers from 0 up to n exist + in the indices. + 0 if indices list is empty or the smallest integer in indices is larger than 0. + """ + largest_consecutive = -1 + indices.sort() + + for idx in indices: + if idx - largest_consecutive <= 1: + largest_consecutive = idx + else: + break + + return largest_consecutive + 1 + @classmethod def _argument_name_and_value_to_dict(cls, arg_name: str, arg_value: str) -> Dict: """ @@ -98,28 +138,82 @@ class DownloadArgsParser: return applied_alias_arguments + @classmethod + def _make_lists(cls, arguments: List[str]) -> List[str]: + """ + Assemble lists + """ + list_arguments = {} + new_arguments = [] + + for idx in range(0, len(arguments), 2): + argument_name, argument_value = arguments[idx], arguments[idx + 1] + + if not cls._uses_list(argument_name=argument_name): + new_arguments.append(argument_name) + new_arguments.append(argument_value) + continue + + argument_name, list_index = cls._extract_list_index(argument_name=argument_name) + + if argument_name not in list_arguments: + list_arguments[argument_name] = ([], []) + + list_arguments[argument_name][0].append(list_index) + list_arguments[argument_name][1].append(argument_value) + + for name, (indices, values) in list_arguments.items(): + list_length = cls._find_largest_consecutive(indices=indices.copy()) + + if list_length == 0: + raise InvalidDlArguments("Incomplete list") + + list_value = [""] * list_length + + for idx, value in zip(indices, values): + if idx < list_length: + list_value[idx] = value + + new_arguments.append(name) + new_arguments.append(list_value) + + return new_arguments + def to_subscription_dict(self) -> Dict: """ Converts the extra arguments into a dict equivalent to a subscription yaml. + :raises InvalidDlArguments: when an argument tries to use both list and non-list values. :return: dict containing argument names and values """ - subscription_dict = {} arguments = self._apply_aliases( unknown_arguments=self._unknown_arguments, aliases=self._config_options.dl_aliases ) if len(arguments) % 2 != 0: raise self._argument_exception + if not all( + (self._is_argument_name(arg=arguments[idx]) for idx in range(0, len(arguments), 2)) + ): + raise self._argument_exception + + arguments = self._make_lists(arguments=arguments) + subscription_dict = {} + for idx in range(0, len(arguments), 2): argument_name, argument_value = arguments[idx], arguments[idx + 1] - if not self._is_argument_name(arg=argument_name): - raise self._argument_exception - argument_dict = self._argument_name_and_value_to_dict( arg_name=argument_name, arg_value=argument_value ) - mergedeep.merge(subscription_dict, argument_dict, strategy=mergedeep.Strategy.REPLACE) + try: + mergedeep.merge( + subscription_dict, argument_dict, strategy=mergedeep.Strategy.TYPESAFE_REPLACE + ) + except TypeError as exc: + raise InvalidDlArguments( + f"Invalid dl argument {argument_name}: " + "Cannot specify an argument to be two different types" + ) from exc return subscription_dict diff --git a/src/ytdl_sub/utils/exceptions.py b/src/ytdl_sub/utils/exceptions.py index 57153cb2..6798c763 100644 --- a/src/ytdl_sub/utils/exceptions.py +++ b/src/ytdl_sub/utils/exceptions.py @@ -28,3 +28,7 @@ class InvalidYamlException(ValidationException): class RegexNoMatchException(ValidationException): """Regex failed to match during download""" + + +class InvalidDlArguments(ValidationException): + """dl arguments that are invalid""" diff --git a/tests/unit/cli/test_download_args_parser.py b/tests/unit/cli/test_download_args_parser.py index bdb66910..ab175030 100644 --- a/tests/unit/cli/test_download_args_parser.py +++ b/tests/unit/cli/test_download_args_parser.py @@ -1,5 +1,4 @@ import shlex -from argparse import ArgumentError from typing import Callable from typing import Dict from typing import List @@ -10,6 +9,7 @@ import pytest from ytdl_sub.cli.download_args_parser import DownloadArgsParser from ytdl_sub.cli.main_args_parser import parser from ytdl_sub.config.config_file import ConfigOptions +from ytdl_sub.utils.exceptions import InvalidDlArguments @pytest.fixture @@ -66,6 +66,36 @@ class TestDownloadArgsParser: "dl --youtube.playlist_url https://youtube.com/playlist?list=123abc", {"youtube": {"playlist_url": "https://youtube.com/playlist?list=123abc"}}, ), + ( + None, + "dl --parameter.using.list[1] 'v1'", + {"parameter": {"using": {"list": ["v1"]}}}, + ), + ( + None, + "dl --parameter.using.list[1] 'v1' --parameter.using.list[2] 'v2'", + {"parameter": {"using": {"list": ["v1", "v2"]}}}, + ), + ( + None, + "dl --parameter.using.list[2] 'v2' --parameter.using.list[1] 'v1'", + {"parameter": {"using": {"list": ["v1", "v2"]}}}, + ), + ( + None, + "dl --parameter.using.list[2] 'v3' --parameter.using.list[1] 'v1' --parameter.using.list[2] 'v2'", + {"parameter": {"using": {"list": ["v1", "v2"]}}}, + ), + ( + None, + "dl --parameter.using.list[3] 'v3' --parameter.using.list[1] 'v1'", + {"parameter": {"using": {"list": ["v1"]}}}, + ), + ( + None, + "dl --parameter.not.using.list[0] 'v0'", + {"parameter": {"not": {"using": {"list[0]": "v0"}}}}, + ), ], ) def test_successful_args(self, config_options_generator, aliases, cmd, expected_sub_dict): @@ -82,7 +112,7 @@ class TestDownloadArgsParser: config_options = config_options_generator() extra_args = _get_extra_arguments(cmd_string="dl --preset") - with pytest.raises(ArgumentError, match=argument_error_msg): + with pytest.raises(InvalidDlArguments, match=argument_error_msg): DownloadArgsParser( extra_arguments=extra_args, config_options=config_options ).to_subscription_dict() @@ -91,7 +121,31 @@ class TestDownloadArgsParser: config_options = config_options_generator() extra_args = _get_extra_arguments(cmd_string="dl --preset buttered toast --test") - with pytest.raises(ArgumentError, match=argument_error_msg): + with pytest.raises(InvalidDlArguments, match=argument_error_msg): + DownloadArgsParser( + extra_arguments=extra_args, config_options=config_options + ).to_subscription_dict() + + def test_error_incomplete_list(self, config_options_generator): + config_options = config_options_generator() + extra_args = _get_extra_arguments(cmd_string="dl --parameter.using.list[3] 'v3'") + + with pytest.raises(InvalidDlArguments, match="Incomplete list"): + DownloadArgsParser( + extra_arguments=extra_args, config_options=config_options + ).to_subscription_dict() + + def test_error_two_different_types(self, config_options_generator): + config_options = config_options_generator() + extra_args = _get_extra_arguments( + cmd_string="dl --parameter.using.list 'v2' --parameter.using.list[1] 'v1'" + ) + + with pytest.raises( + InvalidDlArguments, + match=f"Invalid dl argument --parameter.using.list: " + "Cannot specify an argument to be two different types", + ): DownloadArgsParser( extra_arguments=extra_args, config_options=config_options ).to_subscription_dict()