Skip to content

Commit 8d18ebd

Browse files
sebthomsebthom
committed
test: add DocumentInputStreamTest test case
1 parent d5f020e commit 8d18ebd

7 files changed

Lines changed: 177 additions & 7 deletions

File tree

‎.github/workflows/validate_pr.yml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,7 @@ jobs:
3333
revert
3434
perf
3535
style
36+
test
3637
scopes: |
3738
core
3839
deps

org.eclipse.tm4e.ui.tests/src/main/java/org/eclipse/tm4e/ui/internal/model/TMDocumentModelTest.java renamed to org.eclipse.tm4e.ui.tests/src/main/java/org/eclipse/tm4e/ui/tests/internal/model/TMDocumentModelTest.java

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@
1111
* - Mickael Istria (Red Hat Inc.)
1212
* - Sebastian Thomschke (Vegard IT GmbH) - add test cases
1313
*/
14-
package org.eclipse.tm4e.ui.internal.model;
14+
package org.eclipse.tm4e.ui.tests.internal.model;
1515

1616
import static org.junit.jupiter.api.Assertions.*;
1717

@@ -29,6 +29,7 @@
2929
import org.eclipse.tm4e.core.model.Range;
3030
import org.eclipse.tm4e.core.registry.IGrammarSource;
3131
import org.eclipse.tm4e.core.registry.Registry;
32+
import org.eclipse.tm4e.ui.internal.model.TMDocumentModel;
3233
import org.eclipse.tm4e.ui.tests.support.TestUtils;
3334
import org.junit.function.ThrowingRunnable;
3435
import org.junit.jupiter.api.AfterEach;

org.eclipse.tm4e.ui.tests/src/main/java/org/eclipse/tm4e/ui/internal/themes/TMTokenProviderTest.java renamed to org.eclipse.tm4e.ui.tests/src/main/java/org/eclipse/tm4e/ui/tests/internal/themes/TMTokenProviderTest.java

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@
99
* Contributors:
1010
* Sebastian Thomschke (Vegard IT) - initial implementation
1111
*******************************************************************************/
12-
package org.eclipse.tm4e.ui.internal.themes;
12+
package org.eclipse.tm4e.ui.tests.internal.themes;
1313

1414
import static org.junit.jupiter.api.Assertions.*;
1515

@@ -21,6 +21,7 @@
2121
import org.eclipse.swt.SWT;
2222
import org.eclipse.tm4e.core.registry.IThemeSource.ContentType;
2323
import org.eclipse.tm4e.core.theme.RGB;
24+
import org.eclipse.tm4e.ui.internal.themes.TMThemeTokenProvider;
2425
import org.eclipse.tm4e.ui.themes.ColorManager;
2526
import org.junit.jupiter.api.Test;
2627

‎org.eclipse.tm4e.ui.tests/src/main/java/org/eclipse/tm4e/ui/tests/internal/utils/DocumentInputStreamTest.java‎

Lines changed: 163 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,163 @@
1+
/*******************************************************************************
2+
* Copyright (c) 2024 Sebastian Thomschke and others.
3+
* This program and the accompanying materials are made
4+
* available under the terms of the Eclipse Public License 2.0
5+
* which is available at https://www.eclipse.org/legal/epl-2.0/
6+
*
7+
* SPDX-License-Identifier: EPL-2.0
8+
*
9+
* Contributors:
10+
* Sebastian Thomschke - initial implementation
11+
*******************************************************************************/
12+
package org.eclipse.tm4e.ui.tests.internal.utils;
13+
14+
import static java.nio.charset.StandardCharsets.UTF_8;
15+
import static org.junit.Assert.*;
16+
17+
import java.io.ByteArrayInputStream;
18+
import java.io.IOException;
19+
import java.util.ArrayList;
20+
21+
import org.eclipse.core.resources.IFile;
22+
import org.eclipse.core.resources.IProject;
23+
import org.eclipse.core.resources.ResourcesPlugin;
24+
import org.eclipse.core.runtime.CoreException;
25+
import org.eclipse.jface.text.IDocument;
26+
import org.eclipse.tm4e.ui.internal.utils.DocumentInputStream;
27+
import org.eclipse.ui.editors.text.FileDocumentProvider;
28+
import org.eclipse.ui.part.FileEditorInput;
29+
import org.eclipse.ui.texteditor.IDocumentProvider;
30+
import org.junit.Before;
31+
import org.junit.Test;
32+
33+
class DocumentInputStreamTest {
34+
35+
private static final String TEST_ASCII = "Hello, World!";
36+
37+
private static final String EMOJI = "😊";
38+
private static final int EMOJI_BYTES_LEN = EMOJI.getBytes(UTF_8).length;
39+
private static final String JAPANESE = "こんにちは";
40+
private static final String TEST_UNICODE = EMOJI + JAPANESE;
41+
private static final int TEST_UNICODE_BYTES_LEN = TEST_UNICODE.getBytes(UTF_8).length;
42+
43+
private final IDocumentProvider documentProvider = new FileDocumentProvider();
44+
private IDocument document;
45+
46+
@Before
47+
public void setUp() throws CoreException {
48+
final IProject p = ResourcesPlugin.getWorkspace().getRoot().getProject(getClass().getName() + System.currentTimeMillis());
49+
p.create(null);
50+
p.open(null);
51+
52+
IFile testFile = p.getFile("testfile");
53+
testFile.create(new ByteArrayInputStream(TEST_UNICODE.getBytes()), true, null);
54+
final var editorInput = new FileEditorInput(testFile);
55+
documentProvider.connect(editorInput);
56+
document = documentProvider.getDocument(editorInput);
57+
}
58+
59+
@Test
60+
public void testAvailable() throws IOException {
61+
document.set(TEST_ASCII);
62+
try (var is = new DocumentInputStream(document)) {
63+
assertEquals(UTF_8, is.getCharset());
64+
assertEquals(TEST_ASCII.length(), is.available());
65+
final byte[] buffer = new byte[4];
66+
is.read(buffer);
67+
assertEquals(TEST_ASCII.length() - 4, is.available());
68+
is.readAllBytes();
69+
assertEquals(0, is.available());
70+
}
71+
72+
document.set(TEST_UNICODE);
73+
try (var is = new DocumentInputStream(document)) {
74+
assertEquals(UTF_8, is.getCharset());
75+
assertTrue(is.available() > 0);
76+
is.read(new byte[10]);
77+
assertTrue(is.available() > 0);
78+
is.readAllBytes();
79+
assertEquals(0, is.available());
80+
}
81+
}
82+
83+
@Test
84+
public void testEndOfStream() throws IOException {
85+
try (var is = new DocumentInputStream(document)) {
86+
assertEquals(UTF_8, is.getCharset());
87+
is.skip(Long.MAX_VALUE);
88+
assertEquals(-1, is.read());
89+
}
90+
}
91+
92+
@Test
93+
public void testReadEachByte() throws IOException {
94+
try (var is = new DocumentInputStream(document)) {
95+
assertEquals(UTF_8, is.getCharset());
96+
final var bytesRead = new ArrayList<Byte>();
97+
int b;
98+
while ((b = is.read()) != -1) {
99+
bytesRead.add((byte) b);
100+
}
101+
102+
final byte[] byteArray = new byte[bytesRead.size()];
103+
for (int i = 0; i < bytesRead.size(); i++) {
104+
byteArray[i] = bytesRead.get(i);
105+
}
106+
assertEquals(TEST_UNICODE, new String(byteArray, UTF_8));
107+
}
108+
}
109+
110+
@Test
111+
public void testReadIntoByteArray() throws IOException {
112+
final byte[] buffer = new byte[1024]; // Buffer to read a portion of the text
113+
114+
try (var is = new DocumentInputStream(document)) {
115+
assertEquals(UTF_8, is.getCharset());
116+
final int bytesRead = is.read(buffer, 0, buffer.length);
117+
118+
assertEquals(TEST_UNICODE, new String(buffer, 0, bytesRead, UTF_8));
119+
}
120+
}
121+
122+
@Test
123+
public void testSkip() throws IOException {
124+
try (var is = new DocumentInputStream(document)) {
125+
assertEquals(UTF_8, is.getCharset());
126+
// skip emoji
127+
final long skipped = is.skip(EMOJI_BYTES_LEN);
128+
assertEquals(EMOJI_BYTES_LEN, skipped);
129+
130+
final byte[] japanese = new byte[TEST_UNICODE_BYTES_LEN];
131+
final int bytesRead = is.read(japanese);
132+
133+
assertEquals(JAPANESE, new String(japanese, 0, bytesRead, UTF_8));
134+
}
135+
}
136+
137+
@Test
138+
public void testHighSurrogateAtEndOfInput() throws IOException {
139+
document.set(new String(new char[] { 'A', '\uD800' })); // valid char followed by an isolated high surrogate
140+
try (var is = new DocumentInputStream(document)) {
141+
assertEquals(UTF_8, is.getCharset());
142+
final byte[] result = is.readAllBytes();
143+
final String output = new String(result, UTF_8);
144+
145+
// the high surrogate at the end should be replaced by the
146+
// Unicode replacement char
147+
assertEquals("A\uFFFD", output);
148+
}
149+
}
150+
151+
@Test
152+
public void testHighSurrogateWithoutLowSurrogate() throws IOException {
153+
document.set(new String(new char[] { '\uD800', 'A' })); // \uD800 is a high surrogate, followed by 'A'
154+
try (var is = new DocumentInputStream(document)) {
155+
assertEquals(UTF_8, is.getCharset());
156+
final byte[] result = is.readAllBytes();
157+
final String output = new String(result, UTF_8);
158+
159+
// the invalid surrogate pair should be replaced by the Unicode replacement char
160+
assertEquals("\uFFFD" + "A", output);
161+
}
162+
}
163+
}

‎org.eclipse.tm4e.ui/src/main/java/org/eclipse/tm4e/ui/internal/utils/CharsInputStream.java‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -26,11 +26,11 @@
2626

2727
class CharsInputStream extends InputStream {
2828
@FunctionalInterface
29-
interface CharsSupplier {
29+
public interface CharsSupplier {
3030
char charAt(int index) throws Exception;
3131
}
3232

33-
enum EncoderState {
33+
private enum EncoderState {
3434
ENCODING,
3535
FLUSHING,
3636
DONE
@@ -100,6 +100,10 @@ public int available() {
100100
return remaining == 0 ? charsLength.getAsInt() - charIndex : remaining;
101101
}
102102

103+
public Charset getCharset() {
104+
return encoder.charset();
105+
}
106+
103107
private boolean flushEncoder() throws IOException {
104108
if (encoderState == EncoderState.DONE)
105109
return false;

‎org.eclipse.tm4e.ui/src/main/java/org/eclipse/tm4e/ui/internal/utils/DocumentInputStream.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@
2020
import org.eclipse.jface.text.IDocument;
2121
import org.eclipse.tm4e.ui.TMUIPlugin;
2222

23-
final class DocumentInputStream extends CharsInputStream {
23+
public final class DocumentInputStream extends CharsInputStream {
2424

2525
private static @Nullable Charset getCharset(final IDocument document) {
2626
final ITextFileBufferManager bufferManager = FileBuffers.getTextFileBufferManager();
@@ -39,7 +39,7 @@ final class DocumentInputStream extends CharsInputStream {
3939
return null;
4040
}
4141

42-
DocumentInputStream(final IDocument doc) {
42+
public DocumentInputStream(final IDocument doc) {
4343
super(doc::getChar, doc::getLength, getCharset(doc));
4444
}
4545
}

‎pom.xml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -305,7 +305,7 @@
305305
<!-- https://wiki.eclipse.org/Tycho/FAQ#How_to_use_SWTBot_or_some_UI_tool_for_testing.3F -->
306306
<useUIHarness>true</useUIHarness>
307307
<useUIThread>true</useUIThread>
308-
<argLine>-Xms512m -Xmx512m ${tycho-surefire-plugin.vmargs}</argLine>
308+
<argLine>-Xms512m -Xmx512m -Dfile.encoding=${project.build.sourceEncoding} ${tycho-surefire-plugin.vmargs}</argLine>
309309
</configuration>
310310
</plugin>
311311
<plugin>

0 commit comments

Comments
 (0)