diff --git a/src/ytdl_sub/script/parser.py b/src/ytdl_sub/script/parser.py index 405885b0..daf75010 100644 --- a/src/ytdl_sub/script/parser.py +++ b/src/ytdl_sub/script/parser.py @@ -1,62 +1,40 @@ -from dataclasses import dataclass -from queue import LifoQueue from typing import List from typing import Optional -from typing import Union +from ytdl_sub.script.types import ArgumentType +from ytdl_sub.script.types import Boolean +from ytdl_sub.script.types import Float +from ytdl_sub.script.types import Function +from ytdl_sub.script.types import Integer +from ytdl_sub.script.types import LiteralString +from ytdl_sub.script.types import NumericType +from ytdl_sub.script.types import String +from ytdl_sub.script.types import SyntaxTree +from ytdl_sub.script.types import Variable from ytdl_sub.utils.exceptions import StringFormattingException from ytdl_sub.validators.string_formatter_validators import is_valid_source_variable_name # pylint: disable=invalid-name -@dataclass(frozen=True) -class Integer: - value: int - - -@dataclass(frozen=True) -class Float: - value: float - - -@dataclass(frozen=True) -class Boolean: - value: bool - - -@dataclass(frozen=True) -class String: - value: str - - -@dataclass(frozen=True) -class Variable: - name: str - - -NumericType = Union[Integer, Float] -ArgumentType = Union[Integer, Float, String, Boolean, Variable, "Function"] - - -@dataclass(frozen=True) -class Function: - name: str - args: List[ArgumentType] - - -@dataclass(frozen=True) -class LiteralString: - value: str - - -class Parser: +class _Parser: def __init__(self, text: str): self._text = text self._pos = 0 - self._stack: LifoQueue[LiteralString | Variable | Function] = LifoQueue() + self._ast: List[LiteralString | Variable | Function] = [] - def read(self, increment_pos: bool = True, length: int = 1) -> Optional[str]: + self._syntax_tree = self._parse() + + @property + def ast(self) -> SyntaxTree: + """ + Returns + ------- + Abstract syntax tree of the parsed text + """ + return self._syntax_tree + + def _read(self, increment_pos: bool = True, length: int = 1) -> Optional[str]: try: ch = self._text[self._pos : (self._pos + length)] except IndexError: @@ -66,9 +44,9 @@ class Parser: self._pos += length return ch - def parse_variable(self) -> Variable: + def _parse_variable(self) -> Variable: var_name = "" - while ch := self.read(increment_pos=False): + while ch := self._read(increment_pos=False): if ch.isspace() and not var_name: self._pos += 1 continue @@ -85,12 +63,15 @@ class Parser: var_name += ch self._pos += 1 + if not var_name: + raise StringFormattingException("invalid var name") + assert is_valid_source_variable_name(var_name, raise_exception=False) return Variable(var_name) - def parse_numeric(self) -> NumericType: + def _parse_numeric(self) -> NumericType: numeric_string = "" - while ch := self.read(increment_pos=False): + while ch := self._read(increment_pos=False): if not (ch.isnumeric() or ch == "."): break @@ -107,43 +88,43 @@ class Parser: return Float(value=numeric_float) - def parse_string(self) -> String: + def _parse_string(self) -> String: """ Begin parsing a string, including the quotation value """ string_value = "" - open_quotation_char = self.read() + open_quotation_char = self._read() assert open_quotation_char in ["'", '"'] - while ch := self.read(): + while ch := self._read(): if ch == open_quotation_char: return String(value=string_value) string_value += ch raise StringFormattingException("String not closed") - def parse_function_arg(self) -> ArgumentType: - if self.read(increment_pos=False) == "%": + def _parse_function_arg(self) -> ArgumentType: + if self._read(increment_pos=False) == "%": self._pos += 1 - return self.parse_function() - if self.read(increment_pos=False).isnumeric(): - return self.parse_numeric() - if (self.read(increment_pos=False, length=4) or "").lower() == "true": + return self._parse_function() + if self._read(increment_pos=False).isnumeric(): + return self._parse_numeric() + if (self._read(increment_pos=False, length=4) or "").lower() == "true": self._pos += 4 return Boolean(value=True) - if (self.read(increment_pos=False, length=5) or "").lower() == "false": + if (self._read(increment_pos=False, length=5) or "").lower() == "false": self._pos += 5 return Boolean(value=False) - if self.read(increment_pos=False) in ["'", '"']: - return self.parse_string() - if self.read(increment_pos=False).isascii() and self.read(increment_pos=False).islower(): - return self.parse_variable() + if self._read(increment_pos=False) in ["'", '"']: + return self._parse_string() + if self._read(increment_pos=False).isascii() and self._read(increment_pos=False).islower(): + return self._parse_variable() raise StringFormattingException( "Invalid function argument, should be either a function, int, float, " "string, boolean, or variable without brackets" ) - def parse_function_args(self) -> List[ArgumentType]: + def _parse_function_args(self) -> List[ArgumentType]: """ Begin parsing function args after the first ``(``, i.e. ``function_name(`` """ @@ -151,7 +132,7 @@ class Parser: comma_count = 0 arguments: List[ArgumentType] = [] - while ch := self.read(increment_pos=False): + while ch := self._read(increment_pos=False): if ch == ")": break @@ -165,43 +146,43 @@ class Parser: self._pos += 1 else: argument_index += 1 - arguments.append(self.parse_function_arg()) + arguments.append(self._parse_function_arg()) return arguments - def parse_function(self) -> Function: + def _parse_function(self) -> Function: """ Begin parsing a function after reading the first ``%`` """ function_name: str = "" function_args: List[String | Variable | "Function"] = [] - while ch := self.read(): + while ch := self._read(): if ch == ")": return Function(name=function_name, args=function_args) if ch != "(": function_name += ch else: - function_args = self.parse_function_args() + function_args = self._parse_function_args() raise StringFormattingException("Invalid function") - def parse(self) -> "Parser": + def _parse(self) -> SyntaxTree: bracket_counter = 0 literal_str = "" - while ch := self.read(): + while ch := self._read(): if ch == "}": bracket_counter -= 1 break if ch == "{": bracket_counter += 1 if literal_str: - self._stack.put(LiteralString(literal_str)) + self._ast.append(LiteralString(literal_str)) literal_str = "" # Allow whitespace after bracket opening - while ch1 := self.read(increment_pos=False): + while ch1 := self._read(increment_pos=False): if not ch1.isspace(): break self._pos += 1 @@ -213,9 +194,9 @@ class Parser: if ch1 == "%": self._pos += 1 - self._stack.put(self.parse_function()) + self._ast.append(self._parse_function()) else: - self._stack.put(self.parse_variable()) + self._ast.append(self._parse_variable()) else: literal_str += ch @@ -223,9 +204,13 @@ class Parser: raise StringFormattingException("Bracket count mismatch") if literal_str: - self._stack.put(LiteralString(literal_str)) + self._ast.append(LiteralString(literal_str)) - return self + return SyntaxTree(ast=self._ast) + + +def parse(text: str) -> SyntaxTree: + return _Parser(text).ast # pylint: enable=invalid-name diff --git a/src/ytdl_sub/script/types.py b/src/ytdl_sub/script/types.py new file mode 100644 index 00000000..5f33409c --- /dev/null +++ b/src/ytdl_sub/script/types.py @@ -0,0 +1,76 @@ +from dataclasses import dataclass +from typing import List +from typing import Set +from typing import Union + + +@dataclass(frozen=True) +class Integer: + value: int + + +@dataclass(frozen=True) +class Float: + value: float + + +@dataclass(frozen=True) +class Boolean: + value: bool + + +@dataclass(frozen=True) +class String: + value: str + + +@dataclass(frozen=True) +class Variable: + name: str + + +NumericType = Union[Integer, Float] +ArgumentType = Union[Integer, Float, String, Boolean, Variable, "Function"] + + +@dataclass(frozen=True) +class Function: + name: str + args: List[ArgumentType] + + @property + def variables(self) -> Set[Variable]: + """ + Returns + ------- + All variables used within the function + """ + variables: Set[Variable] = set() + for arg in self.args: + if isinstance(arg, Variable): + variables.add(arg) + elif isinstance(arg, Function): + variables.union(arg.variables) + + return variables + + +@dataclass(frozen=True) +class LiteralString: + value: str + + +@dataclass(frozen=True) +class SyntaxTree: + ast: List[LiteralString | Variable | Function] + + @property + def variables(self) -> Set[Variable]: + variables: Set[Variable] = set() + for token in self.ast: + if isinstance(token, Variable): + variables.add(token) + elif isinstance(token, Function): + variables.union(token.variables) + + return variables diff --git a/src/ytdl_sub/utils/thumbnail.py b/src/ytdl_sub/utils/thumbnail.py index a999f01c..9d342f64 100644 --- a/src/ytdl_sub/utils/thumbnail.py +++ b/src/ytdl_sub/utils/thumbnail.py @@ -75,7 +75,7 @@ def download_and_convert_url_thumbnail( # timeout after 8 seconds with urlopen(thumbnail_url, timeout=1.0) as file: with tempfile.NamedTemporaryFile(delete=False) as thumbnail: - thumbnail.write(file.read()) + thumbnail.write(file._read()) try: os.makedirs(os.path.dirname(output_thumbnail_path), exist_ok=True) diff --git a/tests/unit/script/test_parser.py b/tests/unit/script/test_parser.py index 81eb0eb1..b8473557 100644 --- a/tests/unit/script/test_parser.py +++ b/tests/unit/script/test_parser.py @@ -1,60 +1,63 @@ import pytest -from ytdl_sub.script.parser import Boolean -from ytdl_sub.script.parser import Float -from ytdl_sub.script.parser import Function -from ytdl_sub.script.parser import Integer -from ytdl_sub.script.parser import LiteralString -from ytdl_sub.script.parser import Parser -from ytdl_sub.script.parser import String -from ytdl_sub.script.parser import Variable +from ytdl_sub.script.parser import parse +from ytdl_sub.script.types import Boolean +from ytdl_sub.script.types import Float +from ytdl_sub.script.types import Function +from ytdl_sub.script.types import Integer +from ytdl_sub.script.types import LiteralString +from ytdl_sub.script.types import String +from ytdl_sub.script.types import SyntaxTree +from ytdl_sub.script.types import Variable from ytdl_sub.utils.exceptions import StringFormattingException class TestParser: def test_simple(self): - parser = Parser("hello world").parse() - assert list(parser._stack.queue) == [LiteralString(value="hello world")] + assert parse("hello world") == SyntaxTree([LiteralString(value="hello world")]) def test_single_function_one_arg(self): - parser = Parser("hello {%capitalize('hi mom')}").parse() - assert list(parser._stack.queue) == [ - LiteralString("hello "), - Function(name="capitalize", args=[String(value="hi mom")]), - ] + assert parse("hello {%capitalize('hi mom')}") == SyntaxTree( + [ + LiteralString("hello "), + Function(name="capitalize", args=[String(value="hi mom")]), + ] + ) - @pytest.mark.parametrize('whitespace', ["", " ", " ", "\n", " \n "]) + @pytest.mark.parametrize("whitespace", ["", " ", " ", "\n", " \n "]) def test_single_function_multiple_args(self, whitespace: str): s = whitespace - parser = Parser( + assert parse( f"hello{s}{{{s}%concat({s}'string'{s},{s}1{s},{s}2.4{s}," f"{s}TRUE{s},{s}variable_name{s},{s}%capitalize({s}'hi'{s}){s}){s}}}" - ).parse() - assert list(parser._stack.queue) == [ - LiteralString(value=f"hello{s}"), - Function( - name="concat", - args=[ - String(value="string"), - Integer(value=1), - Float(value=2.4), - Boolean(value=True), - Variable(name="variable_name"), - Function(name="capitalize", args=[String(value="hi")]), - ], - ), - ] + ([LiteralString(value=s)] if s else []) + ) == SyntaxTree( + [ + LiteralString(value=f"hello{s}"), + Function( + name="concat", + args=[ + String(value="string"), + Integer(value=1), + Float(value=2.4), + Boolean(value=True), + Variable(name="variable_name"), + Function(name="capitalize", args=[String(value="hi")]), + ], + ), + ] + + ([LiteralString(value=s)] if s else []) + ) class TestParserBracketFailures: def test_bracket_open(self): with pytest.raises(StringFormattingException): - _ = Parser("{").parse() + parse("{") def test_bracket_close(self): with pytest.raises(StringFormattingException): - _ = Parser("}").parse() + parse("}") def test_bracket_in_function(self): with pytest.raises(StringFormattingException): - _ = Parser("hello {%capitalize({as_arg)}").parse() + parse("hello {%capitalize({as_arg)}")