Skip to content

Commit afe4feb

Browse files
mzininmzinin
committed
Unit tests for Tokenizer
1 parent 310bbb5 commit afe4feb

2 files changed

Lines changed: 198 additions & 2 deletions

File tree

‎src/s2e2/tokenizer.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -95,10 +95,10 @@ def __check_uniqueness(self, entity):
9595
raise TypeError('Attempt to add None to Tokenizer')
9696

9797
if entity in self.__functions:
98-
raise ExpressionError('Function {} is alredy added'.format(entity))
98+
raise ExpressionError('Function {} is already added'.format(entity))
9999

100100
if entity in self.__operators:
101-
raise ExpressionError('Operator {} is alredy added'.format(entity))
101+
raise ExpressionError('Operator {} is already added'.format(entity))
102102

103103

104104
def __token_type_by_value(self, value):

‎test/test_tokenizer.py‎

Lines changed: 196 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,10 @@
1+
from s2e2.error import ExpressionError
12
from s2e2.token_type import TokenType
23
from s2e2.token import Token
34
from s2e2.tokenizer import Tokenizer
45

6+
import pytest
7+
58

69
class TestTokenizer:
710

@@ -47,3 +50,196 @@ def test_positive_two_operators_with_spaces_result_value(self):
4750
Token(TokenType.ATOM, 'C')]
4851

4952
assert actual_tokens == expected_tokens
53+
54+
55+
def test_positive_two_operators_without_spaces_result_value(self):
56+
self.tokenizer.add_operator('+')
57+
self.tokenizer.add_operator('&&')
58+
59+
actual_tokens = self.tokenizer.tokenize('A+B&&C')
60+
expected_tokens = [Token(TokenType.ATOM, 'A'),
61+
Token(TokenType.OPERATOR, '+'),
62+
Token(TokenType.ATOM, 'B'),
63+
Token(TokenType.OPERATOR, '&&'),
64+
Token(TokenType.ATOM, 'C')]
65+
66+
assert actual_tokens == expected_tokens
67+
68+
69+
def test_positive_one_operator_is_substring_of_another_result_value(self):
70+
self.tokenizer.add_operator('!')
71+
self.tokenizer.add_operator('!=')
72+
73+
actual_tokens = self.tokenizer.tokenize('A != !B')
74+
expected_tokens = [Token(TokenType.ATOM, 'A'),
75+
Token(TokenType.OPERATOR, '!='),
76+
Token(TokenType.OPERATOR, '!'),
77+
Token(TokenType.ATOM, 'B')]
78+
79+
assert actual_tokens == expected_tokens
80+
81+
82+
def test_positive_one_function_without_arguments_result_value(self):
83+
self.tokenizer.add_function('FUN')
84+
85+
actual_tokens = self.tokenizer.tokenize('FUN()')
86+
expected_tokens = [Token(TokenType.FUNCTION, 'FUN'),
87+
Token(TokenType.LEFT_BRACKET, '('),
88+
Token(TokenType.RIGHT_BRACKET, ')')]
89+
90+
assert actual_tokens == expected_tokens
91+
92+
93+
def test_positive_one_function_one_argument_result_value(self):
94+
self.tokenizer.add_function('FUN')
95+
96+
actual_tokens = self.tokenizer.tokenize('FUN(Arg)')
97+
expected_tokens = [Token(TokenType.FUNCTION, 'FUN'),
98+
Token(TokenType.LEFT_BRACKET, '('),
99+
Token(TokenType.ATOM, 'Arg'),
100+
Token(TokenType.RIGHT_BRACKET, ')')]
101+
102+
assert actual_tokens == expected_tokens
103+
104+
105+
def test_positive_one_function_three_arguments_result_value(self):
106+
self.tokenizer.add_function('FUN')
107+
108+
actual_tokens = self.tokenizer.tokenize('FUN(Arg1, Arg2,Arg3)')
109+
expected_tokens = [Token(TokenType.FUNCTION, 'FUN'),
110+
Token(TokenType.LEFT_BRACKET, '('),
111+
Token(TokenType.ATOM, 'Arg1'),
112+
Token(TokenType.COMMA, ','),
113+
Token(TokenType.ATOM, 'Arg2'),
114+
Token(TokenType.COMMA, ','),
115+
Token(TokenType.ATOM, 'Arg3'),
116+
Token(TokenType.RIGHT_BRACKET, ')')]
117+
118+
assert actual_tokens == expected_tokens
119+
120+
121+
def test_positive_two_functions_one_operator_result_value(self):
122+
self.tokenizer.add_function('FUN1')
123+
self.tokenizer.add_function('FUN2')
124+
self.tokenizer.add_operator('+')
125+
126+
actual_tokens = self.tokenizer.tokenize('FUN1(Arg1) + FUN2(Arg2)')
127+
expected_tokens = [Token(TokenType.FUNCTION, 'FUN1'),
128+
Token(TokenType.LEFT_BRACKET, '('),
129+
Token(TokenType.ATOM, 'Arg1'),
130+
Token(TokenType.RIGHT_BRACKET, ')'),
131+
Token(TokenType.OPERATOR, '+'),
132+
Token(TokenType.FUNCTION, 'FUN2'),
133+
Token(TokenType.LEFT_BRACKET, '('),
134+
Token(TokenType.ATOM, 'Arg2'),
135+
Token(TokenType.RIGHT_BRACKET, ')')]
136+
137+
assert actual_tokens == expected_tokens
138+
139+
140+
def test_positive_nested_functions_result_value(self):
141+
self.tokenizer.add_function('FUN1')
142+
self.tokenizer.add_function('FUN2')
143+
self.tokenizer.add_function('FUN3')
144+
145+
actual_tokens = self.tokenizer.tokenize('FUN1(FUN2(), FUN3())')
146+
expected_tokens = [Token(TokenType.FUNCTION, 'FUN1'),
147+
Token(TokenType.LEFT_BRACKET, '('),
148+
Token(TokenType.FUNCTION, 'FUN2'),
149+
Token(TokenType.LEFT_BRACKET, '('),
150+
Token(TokenType.RIGHT_BRACKET, ')'),
151+
Token(TokenType.COMMA, ','),
152+
Token(TokenType.FUNCTION, 'FUN3'),
153+
Token(TokenType.LEFT_BRACKET, '('),
154+
Token(TokenType.RIGHT_BRACKET, ')'),
155+
Token(TokenType.RIGHT_BRACKET, ')')]
156+
157+
assert actual_tokens == expected_tokens
158+
159+
160+
def test_positive_nested_brackets_result_value(self):
161+
self.tokenizer.add_operator('+')
162+
163+
actual_tokens = self.tokenizer.tokenize('(((A + B)))')
164+
expected_tokens = [Token(TokenType.LEFT_BRACKET, '('),
165+
Token(TokenType.LEFT_BRACKET, '('),
166+
Token(TokenType.LEFT_BRACKET, '('),
167+
Token(TokenType.ATOM, 'A'),
168+
Token(TokenType.OPERATOR, '+'),
169+
Token(TokenType.ATOM, 'B'),
170+
Token(TokenType.RIGHT_BRACKET, ')'),
171+
Token(TokenType.RIGHT_BRACKET, ')'),
172+
Token(TokenType.RIGHT_BRACKET, ')')]
173+
174+
assert actual_tokens == expected_tokens
175+
176+
177+
def test_positive_operators_without_arguments_result_value(self):
178+
self.tokenizer.add_operator('+')
179+
180+
actual_tokens = self.tokenizer.tokenize('+ + +')
181+
expected_tokens = [Token(TokenType.OPERATOR, '+'),
182+
Token(TokenType.OPERATOR, '+'),
183+
Token(TokenType.OPERATOR, '+')]
184+
185+
assert actual_tokens == expected_tokens
186+
187+
188+
def test_positive_unpaired_brackets_result_value(self):
189+
actual_tokens = self.tokenizer.tokenize('((()')
190+
expected_tokens = [Token(TokenType.LEFT_BRACKET, '('),
191+
Token(TokenType.LEFT_BRACKET, '('),
192+
Token(TokenType.LEFT_BRACKET, '('),
193+
Token(TokenType.RIGHT_BRACKET, ')')]
194+
195+
assert actual_tokens == expected_tokens
196+
197+
198+
def test_negative_two_operators_with_the_same_name(self):
199+
self.tokenizer.add_operator('+')
200+
201+
with pytest.raises(ExpressionError) as ex:
202+
self.tokenizer.add_operator('+')
203+
204+
assert 'Operator + is already added' in str(ex.value)
205+
206+
207+
def test_negative_two_functions_with_the_same_name(self):
208+
self.tokenizer.add_function('FUN')
209+
210+
with pytest.raises(ExpressionError) as ex:
211+
self.tokenizer.add_function('FUN')
212+
213+
assert 'Function FUN is already added' in str(ex.value)
214+
215+
216+
def test_negative_function_and_operator_with_the_same_name(self):
217+
self.tokenizer.add_function('FF')
218+
219+
with pytest.raises(ExpressionError) as ex:
220+
self.tokenizer.add_operator('FF')
221+
222+
assert 'Function FF is already added' in str(ex.value)
223+
224+
225+
def test_negative_operator_and_function_with_the_same_name(self):
226+
self.tokenizer.add_operator('FF')
227+
228+
with pytest.raises(ExpressionError) as ex:
229+
self.tokenizer.add_function('FF')
230+
231+
assert 'Operator FF is already added' in str(ex.value)
232+
233+
234+
def test_negative_empty_function(self):
235+
with pytest.raises(TypeError) as ex:
236+
self.tokenizer.add_function(None)
237+
238+
assert 'Attempt to add None' in str(ex.value)
239+
240+
241+
def test_negative_empty_operator(self):
242+
with pytest.raises(TypeError) as ex:
243+
self.tokenizer.add_operator(None)
244+
245+
assert 'Attempt to add None' in str(ex.value)

0 commit comments

Comments
 (0)