From 79ba60021cd129d2392922690646dfda5ccda05f Mon Sep 17 00:00:00 2001 From: Jesse Bannon Date: Wed, 25 Sep 2024 23:24:49 -0700 Subject: [PATCH] [BACKEND] Proper functions with docstrings (#1061) Reimplement some of the 'hidden' functions, including `%regex_capture_many`, as standard functions with docstrings --- docs/source/config_reference/plugins.rst | 2 +- .../scripting/scripting_functions.rst | 51 ++++++++ .../entries/script/function_scripts.py | 64 ---------- src/ytdl_sub/plugins/regex.py | 2 +- .../script/functions/regex_functions.py | 112 ++++++++++++++++++ src/ytdl_sub/script/types/array.py | 3 + tests/e2e/entries/__init__.py | 0 tests/e2e/entries/script/__init__.py | 0 .../entries/script/test_custom_functions.py | 22 ---- tests/e2e/plugins/test_filter.py | 10 +- tools/docgen/scripting_functions.py | 17 ++- 11 files changed, 184 insertions(+), 99 deletions(-) delete mode 100644 tests/e2e/entries/__init__.py delete mode 100644 tests/e2e/entries/script/__init__.py delete mode 100644 tests/e2e/entries/script/test_custom_functions.py diff --git a/docs/source/config_reference/plugins.rst b/docs/source/config_reference/plugins.rst index 5cb786d3..cd0da7f9 100644 --- a/docs/source/config_reference/plugins.rst +++ b/docs/source/config_reference/plugins.rst @@ -796,7 +796,7 @@ regex overrides: description_date_capture: >- { - %regex_capture_many_with_defaults( + %regex_capture_many( description, [ "([0-9]{4})-([0-9]{2})-([0-9]{2})" ], [ upload_year, upload_month, upload_day ] diff --git a/docs/source/config_reference/scripting/scripting_functions.rst b/docs/source/config_reference/scripting/scripting_functions.rst index d04a05e4..ebf238b3 100644 --- a/docs/source/config_reference/scripting/scripting_functions.rst +++ b/docs/source/config_reference/scripting/scripting_functions.rst @@ -533,6 +533,50 @@ regex_capture_groups :description: Returns number of capture groups in regex +regex_capture_many +~~~~~~~~~~~~~~~~~~ +:spec: ``regex_capture_many(string: String, regex_array: Array, default: Optional[Array]) -> Array`` + +:description: + Returns the input string and first regex's capture groups that match to the string + in an array. If a default is not provided, then all number of regex capture groups + must be equal across all regex strings. In addition, an error will be thrown if + no matches are found. + + If the default is provided, then the number of capture groups must be less than + or equal to the length of the default value array. Any element not captured + will return the respective default value. +:usage: + +.. code-block:: python + + { + %regex_capture_many( + "2020-02-27", + [ + "No (.*) matches here", + "([0-9]+)-([0-9]+)-27" + ], + [ "01", "01" ] + ) + } + + # ["2020-02-27", "2020", "02"] + +regex_capture_many_required +~~~~~~~~~~~~~~~~~~~~~~~~~~~ +:spec: ``regex_capture_many_required(string: String, regex_array: Array) -> Array`` + +:description: + Deprecated. Use %regex_capture_many instead. + +regex_capture_many_with_defaults +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +:spec: ``regex_capture_many_with_defaults(string: String, regex_array: Array, default: Optional[Array]) -> Array`` + +:description: + Deprecated. Use %regex_capture_many instead. + regex_fullmatch ~~~~~~~~~~~~~~~ :spec: ``regex_fullmatch(regex: String, string: String) -> Array`` @@ -560,6 +604,13 @@ regex_search the string as the first element of the Array. If there are capture groups, returns each group as a subsequent element in the Array. +regex_search_any +~~~~~~~~~~~~~~~~ +:spec: ``regex_search_any(string: String, regex_array: Array) -> Boolean`` + +:description: + Returns True if any regex pattern in the regex array matches the string. False otherwise. + regex_sub ~~~~~~~~~ :spec: ``regex_sub(regex: String, replacement: String, string: String) -> String`` diff --git a/src/ytdl_sub/entries/script/function_scripts.py b/src/ytdl_sub/entries/script/function_scripts.py index 58f950bb..ab4ba121 100644 --- a/src/ytdl_sub/entries/script/function_scripts.py +++ b/src/ytdl_sub/entries/script/function_scripts.py @@ -20,68 +20,4 @@ CUSTOM_FUNCTION_SCRIPTS: Dict[str, str] = { [] ) }}""", - ############################################################################################# - # REGEX PLUGIN - # $0 - input variable - # $1 - regex array - # $2 - defaults - # $3 - error message - "%regex_capture_inner": """{ - %assert_then( - %array_reduce( - %array_apply_fixed( - %array_apply( - $1, - %regex_capture_groups - ), - %array_size($2), - %lte - ), - %and - ), - %array_overlay( - %array( - %array_first( - %array_apply_fixed( - %array($1), - %string($0), - %regex_search - ), - [] - ) - ), - %array_extend( ['using all defaults'], $2 ), - True - ), - $3 - ) - }""", - "%regex_capture_many_required": """{ - %assert_ne( - %regex_capture_inner( - $0, $1, ['', '', '', '', '', '', '', '', '', ''], - 'When using %regex_capture_many, number of regex capture groups must be less than or equal to the number of defaults' - ), - ['using all defaults', '', '', '', '', '', '', '', '', '', ''], - 'When running %regex_capture_many_required, no regex strings were captured' - ) - }""", - "%regex_capture_many_with_defaults": """{ - %regex_capture_inner( - $0, $1, $2, - 'When using %regex_capture_with_defaults, number of regex capture groups must be less than or equal to the number of defaults' - ) - }""", - "%regex_search_any": """{ - %ne( - %array_at( - %regex_capture_inner( - $0, $1, [], - 'When using %regex_search_many, all regex strings must contain no capture groups' - ), - 0 - ), - 'using all defaults' - ) - }""", } diff --git a/src/ytdl_sub/plugins/regex.py b/src/ytdl_sub/plugins/regex.py index 0874b7f7..8b2bbfc4 100644 --- a/src/ytdl_sub/plugins/regex.py +++ b/src/ytdl_sub/plugins/regex.py @@ -144,7 +144,7 @@ class RegexOptions(ToggleableOptionsDictValidator): overrides: description_date_capture: >- { - %regex_capture_many_with_defaults( + %regex_capture_many( description, [ "([0-9]{4})-([0-9]{2})-([0-9]{2})" ], [ upload_year, upload_month, upload_day ] diff --git a/src/ytdl_sub/script/functions/regex_functions.py b/src/ytdl_sub/script/functions/regex_functions.py index feb204ce..1aa73ea1 100644 --- a/src/ytdl_sub/script/functions/regex_functions.py +++ b/src/ytdl_sub/script/functions/regex_functions.py @@ -1,10 +1,16 @@ import re from typing import AnyStr +from typing import List from typing import Match +from typing import Optional +from ytdl_sub.script.functions.array_functions import ArrayFunctions from ytdl_sub.script.types.array import Array +from ytdl_sub.script.types.resolvable import Boolean +from ytdl_sub.script.types.resolvable import Float from ytdl_sub.script.types.resolvable import Integer from ytdl_sub.script.types.resolvable import String +from ytdl_sub.script.utils.exceptions import RuntimeException def _re_output_to_array(re_out: Match[AnyStr] | None) -> Array: @@ -35,6 +41,20 @@ class RegexFunctions: """ return _re_output_to_array(re.search(regex.value, string.value)) + @staticmethod + def regex_search_any(string: String, regex_array: Array) -> Boolean: + """ + :description: + Returns True if any regex pattern in the regex array matches the string. False otherwise. + """ + return Boolean( + any( + len(RegexFunctions.regex_search(regex=String(str(regex)), string=string).value) > 0 + for regex in regex_array.value + if isinstance(regex, (String, Integer, Boolean, Float)) + ) + ) + @staticmethod def regex_fullmatch(regex: String, string: String) -> Array: """ @@ -62,3 +82,95 @@ class RegexFunctions: match groups via backslash escapes. Callables as replacement argument are not supported. """ return String(re.sub(regex.value, replacement.value, string.value)) + + @staticmethod + def regex_capture_many( + string: String, regex_array: Array, default: Optional[Array] = None + ) -> Array: + """ + :description: + Returns the input string and first regex's capture groups that match to the string + in an array. If a default is not provided, then all number of regex capture groups + must be equal across all regex strings. In addition, an error will be thrown if + no matches are found. + + If the default is provided, then the number of capture groups must be less than + or equal to the length of the default value array. Any element not captured + will return the respective default value. + :usage: + + .. code-block:: python + + { + %regex_capture_many( + "2020-02-27", + [ + "No (.*) matches here", + "([0-9]+)-([0-9]+)-27" + ], + [ "01", "01" ] + ) + } + + # ["2020-02-27", "2020", "02"] + """ + if len(regex_array) == 0: + raise RuntimeException("regex_array must contain at least a single element") + + regex_list: List[String] = [] + for regex in regex_array.value: + if not isinstance(regex, String): + raise RuntimeException("All regex_array elements must be strings") + regex_list.append(regex) + + if default is None: + num_capture_groups = RegexFunctions.regex_capture_groups(regex_list[0]).value + if any( + RegexFunctions.regex_capture_groups(regex).value != num_capture_groups + for regex in regex_list[1:] + ): + raise RuntimeException( + "regex_array elements must contain the same number of capture groups" + ) + elif any( + RegexFunctions.regex_capture_groups(regex).value > len(default) for regex in regex_list + ): + raise RuntimeException( + "When using %regex_capture_array, number of regex capture groups must be less than " + "or equal to the number of defaults" + ) + + output = Array([]) + for regex in regex_list: + output = RegexFunctions.regex_search(regex=regex, string=string) + if len(output) > 0: + break + + if len(output) == 0 and not default: + raise RuntimeException( + f"no regex strings were captured for input string {string.value}" + ) + + if default is not None: + default_output = Array([string] + default.value) + return ArrayFunctions.array_overlay(output, default_output, only_missing=Boolean(True)) + + return output + + @staticmethod + def regex_capture_many_with_defaults( + string: String, regex_array: Array, default: Optional[Array] + ) -> Array: + """ + :description: + Deprecated. Use %regex_capture_many instead. + """ + return RegexFunctions.regex_capture_many(string, regex_array, default) + + @staticmethod + def regex_capture_many_required(string: String, regex_array: Array) -> Array: + """ + :description: + Deprecated. Use %regex_capture_many instead. + """ + return RegexFunctions.regex_capture_many(string, regex_array) diff --git a/src/ytdl_sub/script/types/array.py b/src/ytdl_sub/script/types/array.py index 3129ffe7..8adcfe50 100644 --- a/src/ytdl_sub/script/types/array.py +++ b/src/ytdl_sub/script/types/array.py @@ -56,3 +56,6 @@ class Array(_Array, ResolvableToJson): @property def native(self) -> Any: return [val.native for val in self.value] + + def __len__(self) -> int: + return len(self.value) diff --git a/tests/e2e/entries/__init__.py b/tests/e2e/entries/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/tests/e2e/entries/script/__init__.py b/tests/e2e/entries/script/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/tests/e2e/entries/script/test_custom_functions.py b/tests/e2e/entries/script/test_custom_functions.py deleted file mode 100644 index 61c9a6ca..00000000 --- a/tests/e2e/entries/script/test_custom_functions.py +++ /dev/null @@ -1,22 +0,0 @@ -from unit.script.conftest import single_variable_output - - -class TestCustomFunctions: - def test_is_playlist_ordered_by_newest_true(self): - assert ( - single_variable_output( - "{%is_playlist_ordered_by_newest('https://www.youtube.com/playlist?list=PL5BC0FC26BECA5A35')}" - ) - is True - ) - - def test_is_playlist_ordered_by_newest_false(self): - assert ( - single_variable_output( - "{%is_playlist_ordered_by_newest('https://www.youtube.com/playlist?list=PL2KvlCGf4yFftX466OnFS8wvuSosBfUgm')}" - ) - is False - ) - - def test_is_playlist_ordered_by_newest_defaults_false(self): - assert single_variable_output("{%is_playlist_ordered_by_newest('aaaaaa')}") is False diff --git a/tests/e2e/plugins/test_filter.py b/tests/e2e/plugins/test_filter.py index 487c3269..7b0d2519 100644 --- a/tests/e2e/plugins/test_filter.py +++ b/tests/e2e/plugins/test_filter.py @@ -7,6 +7,7 @@ import pytest from expected_transaction_log import assert_transaction_log_matches from ytdl_sub.config.config_file import ConfigFile +from ytdl_sub.script.utils.exceptions import RuntimeException from ytdl_sub.script.utils.exceptions import UserThrownRuntimeError from ytdl_sub.subscriptions.subscription import Subscription @@ -265,9 +266,9 @@ class TestFilter: def test_regex_fails_no_match(self, playlist_subscription_no_match_fails, output_directory): with pytest.raises( - UserThrownRuntimeError, + RuntimeException, match=re.escape( - "When running %regex_capture_many_required, no regex strings were captured" + "no regex strings were captured for input string Jesse's Minecraft Server [Trailer - Mar.21]" ), ): _ = playlist_subscription_no_match_fails.download(dry_run=True) @@ -290,10 +291,9 @@ class TestFilter: ) with pytest.raises( - UserThrownRuntimeError, + RuntimeException, match=re.escape( - "When using %regex_capture_with_defaults, number of regex capture groups must " - "be less than or equal to the number of defaults" + "When using %regex_capture_array, number of regex capture groups must be less than or equal to the number of defaults" ), ): _ = subscription.download(dry_run=True) diff --git a/tools/docgen/scripting_functions.py b/tools/docgen/scripting_functions.py index cc3d4eb3..9ae519f9 100644 --- a/tools/docgen/scripting_functions.py +++ b/tools/docgen/scripting_functions.py @@ -72,11 +72,16 @@ class ScriptingFunctionsDocGen(DocGen): for function_name in static_methods(parent_objs[name]): if display_function_name := maybe_get_function_name(function_name): - docs += get_function_docstring( - function_name=function_name, - display_function_name=display_function_name, - function=getattr(parent_objs[name], function_name), - level=2, - ) + try: + docs += get_function_docstring( + function_name=function_name, + display_function_name=display_function_name, + function=getattr(parent_objs[name], function_name), + level=2, + ) + except Exception as exc: + raise ValueError( + f"Invalid docs for function {display_function_name}" + ) from exc return docs