danvics/ytdl-sub
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

[BACKEND] Proper functions with docstrings (#1061)

Browse source 
Reimplement some of the 'hidden' functions, including `%regex_capture_many`, as standard functions with docstrings
This commit is contained in:
Jesse Bannon 2024-09-25 23:24:49 -07:00 committed by GitHub
parent f963125398
commit 79ba60021c
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
11 changed files with 184 additions and 99 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
docs/source/config_reference/plugins.rst
View file

@ -796,7 +796,7 @@ regex
overrides:
description_date_capture: >-
{
%regex_capture_many_with_defaults(
%regex_capture_many(
description,
[ "([0-9]{4})-([0-9]{2})-([0-9]{2})" ],
[ upload_year, upload_month, upload_day ]

51
docs/source/config_reference/scripting/scripting_functions.rst
View file

@ -533,6 +533,50 @@ regex_capture_groups
:description:
Returns number of capture groups in regex
regex_capture_many
~~~~~~~~~~~~~~~~~~
:spec: ``regex_capture_many(string: String, regex_array: Array, default: Optional[Array]) -> Array``
:description:
Returns the input string and first regex's capture groups that match to the string
in an array. If a default is not provided, then all number of regex capture groups
must be equal across all regex strings. In addition, an error will be thrown if
no matches are found.
If the default is provided, then the number of capture groups must be less than
or equal to the length of the default value array. Any element not captured
will return the respective default value.
:usage:
.. code-block:: python
{
%regex_capture_many(
"2020-02-27",
[
"No (.*) matches here",
"([0-9]+)-([0-9]+)-27"
],
[ "01", "01" ]
)
}
# ["2020-02-27", "2020", "02"]
regex_capture_many_required
~~~~~~~~~~~~~~~~~~~~~~~~~~~
:spec: ``regex_capture_many_required(string: String, regex_array: Array) -> Array``
:description:
Deprecated. Use %regex_capture_many instead.
regex_capture_many_with_defaults
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
:spec: ``regex_capture_many_with_defaults(string: String, regex_array: Array, default: Optional[Array]) -> Array``
:description:
Deprecated. Use %regex_capture_many instead.
regex_fullmatch
~~~~~~~~~~~~~~~
:spec: ``regex_fullmatch(regex: String, string: String) -> Array``
@ -560,6 +604,13 @@ regex_search
the string as the first element of the Array. If there are capture groups, returns each
group as a subsequent element in the Array.
regex_search_any
~~~~~~~~~~~~~~~~
:spec: ``regex_search_any(string: String, regex_array: Array) -> Boolean``
:description:
Returns True if any regex pattern in the regex array matches the string. False otherwise.
regex_sub
~~~~~~~~~
:spec: ``regex_sub(regex: String, replacement: String, string: String) -> String``

64
src/ytdl_sub/entries/script/function_scripts.py
View file

@ -20,68 +20,4 @@ CUSTOM_FUNCTION_SCRIPTS: Dict[str, str] = {
[]
)
}}""",
#############################################################################################
# REGEX PLUGIN
# $0 - input variable
# $1 - regex array
# $2 - defaults
# $3 - error message
"%regex_capture_inner": """{
%assert_then(
%array_reduce(
%array_apply_fixed(
%array_apply(
$1,
%regex_capture_groups
),
%array_size($2),
%lte
),
%and
),
%array_overlay(
%array(
%array_first(
%array_apply_fixed(
%array($1),
%string($0),
%regex_search
),
[]
)
),
%array_extend( ['using all defaults'], $2 ),
True
),
$3
)
}""",
"%regex_capture_many_required": """{
%assert_ne(
%regex_capture_inner(
$0, $1, ['', '', '', '', '', '', '', '', '', ''],
'When using %regex_capture_many, number of regex capture groups must be less than or equal to the number of defaults'
),
['using all defaults', '', '', '', '', '', '', '', '', '', ''],
'When running %regex_capture_many_required, no regex strings were captured'
)
}""",
"%regex_capture_many_with_defaults": """{
%regex_capture_inner(
$0, $1, $2,
'When using %regex_capture_with_defaults, number of regex capture groups must be less than or equal to the number of defaults'
)
}""",
"%regex_search_any": """{
%ne(
%array_at(
%regex_capture_inner(
$0, $1, [],
'When using %regex_search_many, all regex strings must contain no capture groups'
),
0
),
'using all defaults'
)
}""",
}

2
src/ytdl_sub/plugins/regex.py
View file

@ -144,7 +144,7 @@ class RegexOptions(ToggleableOptionsDictValidator):
overrides:
description_date_capture: >-
{
%regex_capture_many_with_defaults(
%regex_capture_many(
description,
[ "([0-9]{4})-([0-9]{2})-([0-9]{2})" ],
[ upload_year, upload_month, upload_day ]

112
src/ytdl_sub/script/functions/regex_functions.py
View file

@ -1,10 +1,16 @@
import re
from typing import AnyStr
from typing import List
from typing import Match
from typing import Optional
from ytdl_sub.script.functions.array_functions import ArrayFunctions
from ytdl_sub.script.types.array import Array
from ytdl_sub.script.types.resolvable import Boolean
from ytdl_sub.script.types.resolvable import Float
from ytdl_sub.script.types.resolvable import Integer
from ytdl_sub.script.types.resolvable import String
from ytdl_sub.script.utils.exceptions import RuntimeException
def _re_output_to_array(re_out: Match[AnyStr] | None) -> Array:
@ -35,6 +41,20 @@ class RegexFunctions:
"""
return _re_output_to_array(re.search(regex.value, string.value))
@staticmethod
def regex_search_any(string: String, regex_array: Array) -> Boolean:
"""
:description:
Returns True if any regex pattern in the regex array matches the string. False otherwise.
"""
return Boolean(
any(
len(RegexFunctions.regex_search(regex=String(str(regex)), string=string).value) > 0
for regex in regex_array.value
if isinstance(regex, (String, Integer, Boolean, Float))
)
)
@staticmethod
def regex_fullmatch(regex: String, string: String) -> Array:
"""
@ -62,3 +82,95 @@ class RegexFunctions:
match groups via backslash escapes. Callables as replacement argument are not supported.
"""
return String(re.sub(regex.value, replacement.value, string.value))
@staticmethod
def regex_capture_many(
string: String, regex_array: Array, default: Optional[Array] = None
) -> Array:
"""
:description:
Returns the input string and first regex's capture groups that match to the string
in an array. If a default is not provided, then all number of regex capture groups
must be equal across all regex strings. In addition, an error will be thrown if
no matches are found.
If the default is provided, then the number of capture groups must be less than
or equal to the length of the default value array. Any element not captured
will return the respective default value.
:usage:
.. code-block:: python
{
%regex_capture_many(
"2020-02-27",
[
"No (.*) matches here",
"([0-9]+)-([0-9]+)-27"
],
[ "01", "01" ]
)
}
# ["2020-02-27", "2020", "02"]
"""
if len(regex_array) == 0:
raise RuntimeException("regex_array must contain at least a single element")
regex_list: List[String] = []
for regex in regex_array.value:
if not isinstance(regex, String):
raise RuntimeException("All regex_array elements must be strings")
regex_list.append(regex)
if default is None:
num_capture_groups = RegexFunctions.regex_capture_groups(regex_list[0]).value
if any(
RegexFunctions.regex_capture_groups(regex).value != num_capture_groups
for regex in regex_list[1:]
):
raise RuntimeException(
"regex_array elements must contain the same number of capture groups"
)
elif any(
RegexFunctions.regex_capture_groups(regex).value > len(default) for regex in regex_list
):
raise RuntimeException(
"When using %regex_capture_array, number of regex capture groups must be less than "
"or equal to the number of defaults"
)
output = Array([])
for regex in regex_list:
output = RegexFunctions.regex_search(regex=regex, string=string)
if len(output) > 0:
break
if len(output) == 0 and not default:
raise RuntimeException(
f"no regex strings were captured for input string {string.value}"
)
if default is not None:
default_output = Array([string] + default.value)
return ArrayFunctions.array_overlay(output, default_output, only_missing=Boolean(True))
return output
@staticmethod
def regex_capture_many_with_defaults(
string: String, regex_array: Array, default: Optional[Array]
) -> Array:
"""
:description:
Deprecated. Use %regex_capture_many instead.
"""
return RegexFunctions.regex_capture_many(string, regex_array, default)
@staticmethod
def regex_capture_many_required(string: String, regex_array: Array) -> Array:
"""
:description:
Deprecated. Use %regex_capture_many instead.
"""
return RegexFunctions.regex_capture_many(string, regex_array)

3
src/ytdl_sub/script/types/array.py
View file

@ -56,3 +56,6 @@ class Array(_Array, ResolvableToJson):
@property
def native(self) -> Any:
return [val.native for val in self.value]
def __len__(self) -> int:
return len(self.value)

0
tests/e2e/entries/__init__.py
View file

0
tests/e2e/entries/script/__init__.py
View file

22
tests/e2e/entries/script/test_custom_functions.py
View file

@ -1,22 +0,0 @@
from unit.script.conftest import single_variable_output
class TestCustomFunctions:
def test_is_playlist_ordered_by_newest_true(self):
assert (
single_variable_output(
"{%is_playlist_ordered_by_newest('https://www.youtube.com/playlist?list=PL5BC0FC26BECA5A35')}"
)
is True
)
def test_is_playlist_ordered_by_newest_false(self):
assert (
single_variable_output(
"{%is_playlist_ordered_by_newest('https://www.youtube.com/playlist?list=PL2KvlCGf4yFftX466OnFS8wvuSosBfUgm')}"
)
is False
)
def test_is_playlist_ordered_by_newest_defaults_false(self):
assert single_variable_output("{%is_playlist_ordered_by_newest('aaaaaa')}") is False

10
tests/e2e/plugins/test_filter.py
View file

@ -7,6 +7,7 @@ import pytest
from expected_transaction_log import assert_transaction_log_matches
from ytdl_sub.config.config_file import ConfigFile
from ytdl_sub.script.utils.exceptions import RuntimeException
from ytdl_sub.script.utils.exceptions import UserThrownRuntimeError
from ytdl_sub.subscriptions.subscription import Subscription
@ -265,9 +266,9 @@ class TestFilter:
def test_regex_fails_no_match(self, playlist_subscription_no_match_fails, output_directory):
with pytest.raises(
UserThrownRuntimeError,
RuntimeException,
match=re.escape(
"When running %regex_capture_many_required, no regex strings were captured"
"no regex strings were captured for input string Jesse's Minecraft Server [Trailer - Mar.21]"
),
):
_ = playlist_subscription_no_match_fails.download(dry_run=True)
@ -290,10 +291,9 @@ class TestFilter:
)
with pytest.raises(
UserThrownRuntimeError,
RuntimeException,
match=re.escape(
"When using %regex_capture_with_defaults, number of regex capture groups must "
"be less than or equal to the number of defaults"
"When using %regex_capture_array, number of regex capture groups must be less than or equal to the number of defaults"
),
):
_ = subscription.download(dry_run=True)

17
tools/docgen/scripting_functions.py
View file

@ -72,11 +72,16 @@ class ScriptingFunctionsDocGen(DocGen):
for function_name in static_methods(parent_objs[name]):
if display_function_name := maybe_get_function_name(function_name):
docs += get_function_docstring(
function_name=function_name,
display_function_name=display_function_name,
function=getattr(parent_objs[name], function_name),
level=2,
)
try:
docs += get_function_docstring(
function_name=function_name,
display_function_name=display_function_name,
function=getattr(parent_objs[name], function_name),
level=2,
)
except Exception as exc:
raise ValueError(
f"Invalid docs for function {display_function_name}"
) from exc
return docs