Skip to content

Commit 5b76d7f

Browse files
bastibebastibe
authored
Merge pull request #11 from stdout-se/10-quotes-fix
Fix double quotation marks causing error
2 parents fc03e9a + c73588e commit 5b76d7f

3 files changed

Lines changed: 139 additions & 20 deletions

File tree

‎+UnitTest/+tokernizer/TokenizeTests.m‎

Lines changed: 73 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,73 @@
1+
classdef TokenizeTests < matlab.unittest.TestCase
2+
%TOKENIZETESTS Tests for tokenize_code
3+
4+
methods(Test)
5+
function testDoubleQuote(obj)
6+
%TESTDOUBLEQUOTE Tests a double quoted string
7+
8+
% Input data for the test
9+
input_str = '"test"'; % String: "test"
10+
11+
% Construct expected output for comparison
12+
expected = Token('string', input_str, 1, 1);
13+
14+
% Get actual output
15+
actual = tokenize_code(input_str);
16+
17+
% Compare actual output with expected output
18+
obj.verifyEqual(actual, expected);
19+
end
20+
21+
function testSoloDoubleQuote(obj)
22+
%TESTSOLODOUBLEQUOTE Tests a string with only a double quoted
23+
24+
% Input data for the test
25+
input_str = 'output = "test"'; % String: output = 'test'
26+
27+
% Construct expected output for comparison
28+
expected(1) = Token('identifier', 'output', 1, 1);
29+
expected(2) = Token('space', ' ', 1, 7);
30+
expected(3) = Token('punctuation', '=', 1, 8);
31+
expected(4) = Token('space', ' ', 1, 9);
32+
expected(5) = Token('string', '"test"', 1, 10);
33+
34+
% Get actual output
35+
actual = tokenize_code(input_str);
36+
37+
% Compare actual output with expected output
38+
obj.verifyEqual(actual, expected);
39+
end
40+
41+
function testNestedQuote(obj)
42+
%TESTNESTEDQUOTE Tests a double quote inside single quote
43+
44+
% Input data for the test
45+
input_str = '"let''s go"'; % String: "let's go"
46+
47+
% Construct expected output for comparison
48+
expected = Token('string', input_str, 1, 1);
49+
50+
% Get actual output
51+
actual = tokenize_code(input_str);
52+
53+
% Compare actual output with expected output
54+
obj.verifyEqual(actual, expected);
55+
end
56+
57+
function testNestedQuote2(obj)
58+
%TESTNESTEDQUOTE2 Tests a double quote inside single quote
59+
60+
% Input data for the test
61+
input_str = '''He said, "hi"'''; % String: 'He said, "hi"'
62+
63+
% Construct expected output for comparison
64+
expected = Token('string', input_str, 1, 1);
65+
66+
% Get actual output
67+
actual = tokenize_code(input_str);
68+
69+
% Compare actual output with expected output
70+
obj.verifyEqual(actual, expected);
71+
end
72+
end
73+
end

‎run_unittests.m‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
function run_unittests()
2+
%RUN_UNITTESTS Runs all unit tests
3+
4+
import matlab.unittest.TestSuite
5+
import matlab.unittest.TestRunner
6+
7+
try
8+
% Create a test suite
9+
suite = ...
10+
TestSuite.fromPackage('UnitTest', ...
11+
'IncludingSubpackages', true);
12+
13+
% Run all tests
14+
runner = TestRunner.withTextOutput;
15+
result = runner.run(suite);
16+
17+
% Display results
18+
disp(table(result));
19+
disp(result);
20+
21+
% Throw an error if any test failed
22+
if sum([result(:).Failed]) + sum([result(:).Incomplete]) > 0
23+
error('There are failing unittests!')
24+
end
25+
catch err
26+
disp(err.getReport)
27+
end
28+
end

‎tokenize_code.m‎

Lines changed: 38 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -126,22 +126,35 @@
126126
% strings and transpose begin with `'`. The `.'` operator has
127127
% already been handled above:
128128
elseif letter == ''''
129-
is_first_symbol = false;
130-
previous = tokenlist(end);
131-
% transpose operator:
132-
% To differentiate the start of a string from the transpose
133-
% operator, we need to check whether the previous token was a
134-
% value or an operator. If a value, `'` means transpose. If an
135-
% operator, `'` marks the start of a string.
136-
if previous.isEqual('pair', {'}' ']' ')'}) || ...
137-
previous.hasType({'identifier' 'number' 'property'})
138-
pos = pos + 1;
139-
add_token('punctuation', letter);
140-
% strings:
141-
else
142-
string = skip_string();
129+
% the first symbol cannot be transpose, so must be string
130+
if is_first_symbol
131+
string = skip_string('''');
143132
add_token('string', string);
133+
else
134+
previous = tokenlist(end);
135+
136+
% transpose operator:
137+
% To differentiate the start of a string from the
138+
% transpose operator, we need to check whether the
139+
% previous token was a value or an operator. If a value,
140+
% `'` means transpose. If an operator, `'` marks the start
141+
% of a string.
142+
if previous.isEqual('pair', {'}' ']' ')'}) || ...
143+
previous.hasType({'identifier' 'number' 'property'})
144+
pos = pos + 1;
145+
add_token('punctuation', letter);
146+
% strings:
147+
else
148+
string = skip_string('''');
149+
add_token('string', string);
150+
end
144151
end
152+
is_first_symbol = false;
153+
% string that starts with double quotes (")
154+
elseif letter == '"'
155+
is_first_symbol = false;
156+
string = skip_string('"');
157+
add_token('string', string);
145158
% we don't make any distinction between different kinds of parens:
146159
elseif any(letter == open_pairs)
147160
is_first_symbol = false;
@@ -246,18 +259,20 @@ function add_token(token_type, token_text)
246259
string = source_code(string_start:pos-1);
247260
end
248261

249-
function string = skip_string()
262+
function string = skip_string(quote_type)
250263
%SKIP_STRING skips to the end of the string and returns the STRING
251-
% the STRING includes both quotation marks.
264+
% the STRING includes both quotation marks. QUOTE_TYPE is the
265+
% type of quote character to look for (' or ").
252266
% this modifies POS!
253267

254268
string_start = pos;
255269
while true
256-
if source_code(pos) ~= '''' || pos == string_start
270+
if source_code(pos) ~= quote_type || pos == string_start
257271
pos = pos + 1;
258-
elseif length(source_code) > pos && source_code(pos+1) == ''''
272+
elseif length(source_code) > pos ...
273+
&& source_code(pos+1) == quote_type
259274
pos = pos + 2;
260-
else % source_code(pos) == ''''
275+
else % source_code(pos) == quote_type
261276
pos = pos + 1;
262277
break;
263278
end
@@ -302,7 +317,10 @@ function parse_command()
302317
letter = source_code(pos);
303318
% commands can contain literal strings:
304319
if letter == ''''
305-
string_literal = skip_string();
320+
string_literal = skip_string('''');
321+
add_token('string', string_literal);
322+
elseif letter == '"'
323+
string_literal = skip_string('"');
306324
add_token('string', string_literal);
307325
% commands can contain spaces:
308326
elseif any(letter == spaces)

0 commit comments

Comments
 (0)