[BACKEND] Proper functions with docstrings (#1061)

Reimplement some of the 'hidden' functions, including `%regex_capture_many`, as standard functions with docstrings

docs/source/config_reference/plugins.rst

@@ -796,7 +796,7 @@ regex
       overrides:
         description_date_capture: >-
           {
-            %regex_capture_many_with_defaults(
+            %regex_capture_many(
               description,
               [ "([0-9]{4})-([0-9]{2})-([0-9]{2})" ],
               [ upload_year, upload_month, upload_day ]

docs/source/config_reference/scripting/scripting_functions.rst

@@ -533,6 +533,50 @@ regex_capture_groups
 :description:
   Returns number of capture groups in regex
 
+regex_capture_many
+~~~~~~~~~~~~~~~~~~
+:spec: ``regex_capture_many(string: String, regex_array: Array, default: Optional[Array]) -> Array``
+
+:description:
+  Returns the input string and first regex's capture groups that match to the string
+  in an array. If a default is not provided, then all number of regex capture groups
+  must be equal across all regex strings. In addition, an error will be thrown if
+  no matches are found.
+
+  If the default is provided, then the number of capture groups must be less than
+  or equal to the length of the default value array. Any element not captured
+  will return the respective default value.
+:usage:
+
+.. code-block:: python
+
+   {
+     %regex_capture_many(
+       "2020-02-27",
+       [
+         "No (.*) matches here",
+         "([0-9]+)-([0-9]+)-27"
+       ],
+       [ "01", "01" ]
+     )
+   }
+
+   # ["2020-02-27", "2020", "02"]
+
+regex_capture_many_required
+~~~~~~~~~~~~~~~~~~~~~~~~~~~
+:spec: ``regex_capture_many_required(string: String, regex_array: Array) -> Array``
+
+:description:
+  Deprecated. Use %regex_capture_many instead.
+
+regex_capture_many_with_defaults
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+:spec: ``regex_capture_many_with_defaults(string: String, regex_array: Array, default: Optional[Array]) -> Array``
+
+:description:
+  Deprecated. Use %regex_capture_many instead.
+
 regex_fullmatch
 ~~~~~~~~~~~~~~~
 :spec: ``regex_fullmatch(regex: String, string: String) -> Array``
@@ -560,6 +604,13 @@ regex_search
   the string as the first element of the Array. If there are capture groups, returns each
   group as a subsequent element in the Array.
 
+regex_search_any
+~~~~~~~~~~~~~~~~
+:spec: ``regex_search_any(string: String, regex_array: Array) -> Boolean``
+
+:description:
+  Returns True if any regex pattern in the regex array matches the string. False otherwise.
+
 regex_sub
 ~~~~~~~~~
 :spec: ``regex_sub(regex: String, replacement: String, string: String) -> String``

src/ytdl_sub/entries/script/function_scripts.py

@@ -20,68 +20,4 @@ CUSTOM_FUNCTION_SCRIPTS: Dict[str, str] = {
             []
         )
     }}""",
-    #############################################################################################
-    # REGEX PLUGIN
-    #   $0 - input variable
-    #   $1 - regex array
-    #   $2 - defaults
-    #   $3 - error message
-    "%regex_capture_inner": """{
-        %assert_then(
-            %array_reduce(
-                %array_apply_fixed(
-                    %array_apply(
-                        $1,
-                        %regex_capture_groups
-                    ),
-                    %array_size($2),
-                    %lte
-                ),
-                %and
-            ),
-            %array_overlay(
-                %array(
-                    %array_first(
-                        %array_apply_fixed(
-                            %array($1),
-                            %string($0),
-                            %regex_search
-                        ),
-                        []
-                    )
-                ),
-                %array_extend( ['using all defaults'], $2 ),
-                True
-            ),
-            $3
-        )
-    }""",
-    "%regex_capture_many_required": """{
-        %assert_ne(
-            %regex_capture_inner(
-                $0, $1, ['', '', '', '', '', '', '', '', '', ''],
-                'When using %regex_capture_many, number of regex capture groups must be less than or equal to the number of defaults'
-            ),
-            ['using all defaults', '', '', '', '', '', '', '', '', '', ''],
-            'When running %regex_capture_many_required, no regex strings were captured'
-        )
-    }""",
-    "%regex_capture_many_with_defaults": """{
-        %regex_capture_inner(
-            $0, $1, $2,
-            'When using %regex_capture_with_defaults, number of regex capture groups must be less than or equal to the number of defaults'
-        )
-    }""",
-    "%regex_search_any": """{
-        %ne(
-            %array_at(
-                %regex_capture_inner(
-                    $0, $1, [],
-                    'When using %regex_search_many, all regex strings must contain no capture groups'
-                ),
-                0
-            ),
-            'using all defaults'
-        )
-    }""",
 }

src/ytdl_sub/plugins/regex.py

@@ -144,7 +144,7 @@ class RegexOptions(ToggleableOptionsDictValidator):
           overrides:
             description_date_capture: >-
               {
-                %regex_capture_many_with_defaults(
+                %regex_capture_many(
                   description,
                   [ "([0-9]{4})-([0-9]{2})-([0-9]{2})" ],
                   [ upload_year, upload_month, upload_day ]

src/ytdl_sub/script/functions/regex_functions.py

@@ -1,10 +1,16 @@
 import re
 from typing import AnyStr
+from typing import List
 from typing import Match
+from typing import Optional
 
+from ytdl_sub.script.functions.array_functions import ArrayFunctions
 from ytdl_sub.script.types.array import Array
+from ytdl_sub.script.types.resolvable import Boolean
+from ytdl_sub.script.types.resolvable import Float
 from ytdl_sub.script.types.resolvable import Integer
 from ytdl_sub.script.types.resolvable import String
+from ytdl_sub.script.utils.exceptions import RuntimeException
 
 
 def _re_output_to_array(re_out: Match[AnyStr] | None) -> Array:
@@ -35,6 +41,20 @@ class RegexFunctions:
         """
         return _re_output_to_array(re.search(regex.value, string.value))
 
+    @staticmethod
+    def regex_search_any(string: String, regex_array: Array) -> Boolean:
+        """
+        :description:
+          Returns True if any regex pattern in the regex array matches the string. False otherwise.
+        """
+        return Boolean(
+            any(
+                len(RegexFunctions.regex_search(regex=String(str(regex)), string=string).value) > 0
+                for regex in regex_array.value
+                if isinstance(regex, (String, Integer, Boolean, Float))
+            )
+        )
+
     @staticmethod
     def regex_fullmatch(regex: String, string: String) -> Array:
         """
@@ -62,3 +82,95 @@ class RegexFunctions:
           match groups via backslash escapes. Callables as replacement argument are not supported.
         """
         return String(re.sub(regex.value, replacement.value, string.value))
+
+    @staticmethod
+    def regex_capture_many(
+        string: String, regex_array: Array, default: Optional[Array] = None
+    ) -> Array:
+        """
+        :description:
+          Returns the input string and first regex's capture groups that match to the string
+          in an array. If a default is not provided, then all number of regex capture groups
+          must be equal across all regex strings. In addition, an error will be thrown if
+          no matches are found.
+
+          If the default is provided, then the number of capture groups must be less than
+          or equal to the length of the default value array. Any element not captured
+          will return the respective default value.
+        :usage:
+
+        .. code-block:: python
+
+           {
+             %regex_capture_many(
+               "2020-02-27",
+               [
+                 "No (.*) matches here",
+                 "([0-9]+)-([0-9]+)-27"
+               ],
+               [ "01", "01" ]
+             )
+           }
+
+           # ["2020-02-27", "2020", "02"]
+        """
+        if len(regex_array) == 0:
+            raise RuntimeException("regex_array must contain at least a single element")
+
+        regex_list: List[String] = []
+        for regex in regex_array.value:
+            if not isinstance(regex, String):
+                raise RuntimeException("All regex_array elements must be strings")
+            regex_list.append(regex)
+
+        if default is None:
+            num_capture_groups = RegexFunctions.regex_capture_groups(regex_list[0]).value
+            if any(
+                RegexFunctions.regex_capture_groups(regex).value != num_capture_groups
+                for regex in regex_list[1:]
+            ):
+                raise RuntimeException(
+                    "regex_array elements must contain the same number of capture groups"
+                )
+        elif any(
+            RegexFunctions.regex_capture_groups(regex).value > len(default) for regex in regex_list
+        ):
+            raise RuntimeException(
+                "When using %regex_capture_array, number of regex capture groups must be less than "
+                "or equal to the number of defaults"
+            )
+
+        output = Array([])
+        for regex in regex_list:
+            output = RegexFunctions.regex_search(regex=regex, string=string)
+            if len(output) > 0:
+                break
+
+        if len(output) == 0 and not default:
+            raise RuntimeException(
+                f"no regex strings were captured for input string {string.value}"
+            )
+
+        if default is not None:
+            default_output = Array([string] + default.value)
+            return ArrayFunctions.array_overlay(output, default_output, only_missing=Boolean(True))
+
+        return output
+
+    @staticmethod
+    def regex_capture_many_with_defaults(
+        string: String, regex_array: Array, default: Optional[Array]
+    ) -> Array:
+        """
+        :description:
+          Deprecated. Use %regex_capture_many instead.
+        """
+        return RegexFunctions.regex_capture_many(string, regex_array, default)
+
+    @staticmethod
+    def regex_capture_many_required(string: String, regex_array: Array) -> Array:
+        """
+        :description:
+          Deprecated. Use %regex_capture_many instead.
+        """
+        return RegexFunctions.regex_capture_many(string, regex_array)

src/ytdl_sub/script/types/array.py

@@ -56,3 +56,6 @@ class Array(_Array, ResolvableToJson):
     @property
     def native(self) -> Any:
         return [val.native for val in self.value]
+
+    def __len__(self) -> int:
+        return len(self.value)

tests/e2e/entries/script/test_custom_functions.py

@@ -1,22 +0,0 @@
-from unit.script.conftest import single_variable_output
-
-
-class TestCustomFunctions:
-    def test_is_playlist_ordered_by_newest_true(self):
-        assert (
-            single_variable_output(
-                "{%is_playlist_ordered_by_newest('https://www.youtube.com/playlist?list=PL5BC0FC26BECA5A35')}"
-            )
-            is True
-        )
-
-    def test_is_playlist_ordered_by_newest_false(self):
-        assert (
-            single_variable_output(
-                "{%is_playlist_ordered_by_newest('https://www.youtube.com/playlist?list=PL2KvlCGf4yFftX466OnFS8wvuSosBfUgm')}"
-            )
-            is False
-        )
-
-    def test_is_playlist_ordered_by_newest_defaults_false(self):
-        assert single_variable_output("{%is_playlist_ordered_by_newest('aaaaaa')}") is False

tests/e2e/plugins/test_filter.py

@@ -7,6 +7,7 @@ import pytest
 from expected_transaction_log import assert_transaction_log_matches
 
 from ytdl_sub.config.config_file import ConfigFile
+from ytdl_sub.script.utils.exceptions import RuntimeException
 from ytdl_sub.script.utils.exceptions import UserThrownRuntimeError
 from ytdl_sub.subscriptions.subscription import Subscription
 
@@ -265,9 +266,9 @@ class TestFilter:
 
     def test_regex_fails_no_match(self, playlist_subscription_no_match_fails, output_directory):
         with pytest.raises(
-            UserThrownRuntimeError,
+            RuntimeException,
             match=re.escape(
-                "When running %regex_capture_many_required, no regex strings were captured"
+                "no regex strings were captured for input string Jesse's Minecraft Server [Trailer - Mar.21]"
             ),
         ):
             _ = playlist_subscription_no_match_fails.download(dry_run=True)
@@ -290,10 +291,9 @@ class TestFilter:
         )
 
         with pytest.raises(
-            UserThrownRuntimeError,
+            RuntimeException,
             match=re.escape(
-                "When using %regex_capture_with_defaults, number of regex capture groups must "
+                "When using %regex_capture_array, number of regex capture groups must be less than or equal to the number of defaults"
-                "be less than or equal to the number of defaults"
             ),
         ):
             _ = subscription.download(dry_run=True)

tools/docgen/scripting_functions.py

@@ -72,11 +72,16 @@ class ScriptingFunctionsDocGen(DocGen):
 
             for function_name in static_methods(parent_objs[name]):
                 if display_function_name := maybe_get_function_name(function_name):
-                    docs += get_function_docstring(
+                    try:
-                        function_name=function_name,
+                        docs += get_function_docstring(
-                        display_function_name=display_function_name,
+                            function_name=function_name,
-                        function=getattr(parent_objs[name], function_name),
+                            display_function_name=display_function_name,
-                        level=2,
+                            function=getattr(parent_objs[name], function_name),
-                    )
+                            level=2,
+                        )
+                    except Exception as exc:
+                        raise ValueError(
+                            f"Invalid docs for function {display_function_name}"
+                        ) from exc
 
         return docs