fix: Parsing negative variable-length failed with joni SyntaxException

org.eclipse.tm4e.core.TMException: Parsing regex pattern "(?<=\s*\.)\w+"
failed with org.joni.exception.SyntaxException: invalid pattern in
look-behind

Author: sebthom

org.eclipse.tm4e.core/src/main/java/org/eclipse/tm4e/core/internal/oniguruma/OnigRegExp.java

@@ -106,11 +106,17 @@ private String rewritePatternIfRequired(final String pattern) {
 			return "(?<!\\.)\\s*" + pattern.substring(lookbehind1.length());
 		}
 
-		// e.g. used in markdown.math.inline.tmLanguage.json
+		// e.g. used in markdown.math.block.tmLanguage.json and tex.tmLanguage.json
 		final var lookbehind2 = "(?<=^\\s*)";
 		if (pattern.startsWith(lookbehind2)) {
 			return "(?<=^)\\s*" + pattern.substring(lookbehind2.length());
 		}
+
+		// e.g. used in carbon.tmLanguage.json
+		final var lookbehind3 = "(?<=\\s*\\.)";
+		if (pattern.startsWith(lookbehind3)) {
+			return "\\s*\\." + pattern.substring(lookbehind3.length());
+		}
 		return pattern;
 	}
 

org.eclipse.tm4e.core/src/test/java/org/eclipse/tm4e/core/internal/oniguruma/OnigRegExpTest.java

@@ -1,5 +1,5 @@
 /**
- * Copyright (c) 2022 Sebastian Thomschke and others.
+ * Copyright (c) 2022,2024 Sebastian Thomschke and others.
  *
  * This program and the accompanying materials are made
  * available under the terms of the Eclipse Public License 2.0
@@ -11,23 +11,46 @@
 
 import static org.junit.jupiter.api.Assertions.*;
 
+import org.eclipse.jdt.annotation.Nullable;
 import org.junit.jupiter.api.Test;
 
 class OnigRegExpTest {
 
-	@Test
-	void testOnigRegExp() throws Exception {
-		final var regexp = new OnigRegExp(
-				"\\G(MAKEFILES|VPATH|SHELL|MAKESHELL|MAKE|MAKELEVEL|MAKEFLAGS|MAKECMDGOALS|CURDIR|SUFFIXES|\\.LIBPATTERNS)(?=\\s*\\))");
+	private void assertOnigRegExpSearch(final String input, final @Nullable OnigResult result, final int startPosition,
+			final boolean shouldMatch, final String... expectedGroups) {
+		if (shouldMatch) {
+			assertNotNull(result, "Expected a match in input: \"" + input + "\" starting at position " + startPosition);
+			assertEquals(expectedGroups.length, result.count(),
+					"Expected " + expectedGroups.length + " groups, but found " + result.count() + " in input: \"" + input + "\"");
+			for (int i = 0; i < expectedGroups.length; i++) {
+				final String expectedGroup = expectedGroups[i];
+				final int start = result.locationAt(i);
+				final int end = start + result.lengthAt(i);
+				final String actualGroup = input.substring(start, end);
+				assertEquals(expectedGroup, actualGroup, "Expected group " + i + " to be \"" + expectedGroup + "\" but found \""
+						+ actualGroup + "\" in input: \"" + input + "\"");
+			}
+		} else {
+			assertNull(result, "Did not expect a match in input: \"" + input + "\" starting at position " + startPosition);
+		}
+	}
 
-		final var line = "ifeq (version,$(firstword $(MAKECMDGOALS))\n";
-		final var onigLine = OnigString.of(line);
+	private void assertOnigRegExpSearch(final String pattern, final String input, final int startPosition, final boolean shouldMatch,
+			final String... expectedGroups) {
+		final OnigRegExp regexp = new OnigRegExp(pattern);
+		final OnigResult result = regexp.search(OnigString.of(input), startPosition);
+		assertOnigRegExpSearch(input, result, startPosition, shouldMatch, expectedGroups);
+	}
 
-		final var result = regexp.search(onigLine, 28);
-		assertNotNull(result);
-		assertEquals(2, result.count());
-		assertEquals("MAKECMDGOALS", line.substring(result.locationAt(0), result.locationAt(0) + result.lengthAt(0)));
-		assertEquals("MAKECMDGOALS", line.substring(result.locationAt(1), result.locationAt(1) + result.lengthAt(1)));
+	@Test
+	void testOnigRegExp() throws Exception {
+		assertOnigRegExpSearch(
+				"\\G(MAKEFILES|VPATH|SHELL|MAKESHELL|MAKE|MAKELEVEL|MAKEFLAGS|MAKECMDGOALS|CURDIR|SUFFIXES|\\.LIBPATTERNS)(?=\\s*\\))",
+				"ifeq (version,$(firstword $(MAKECMDGOALS))\n",
+				28,
+				true,
+				"MAKECMDGOALS",
+				"MAKECMDGOALS");
 	}
 
 	/**
@@ -45,9 +68,23 @@ void testOnigRegExpCaching() {
 		assertNull(result);
 
 		result = regexp.search(onigLine, 28);
-		assertNotNull(result);
-		assertEquals(2, result.count());
-		assertEquals("MAKECMDGOALS", line.substring(result.locationAt(0), result.locationAt(0) + result.lengthAt(0)));
-		assertEquals("MAKECMDGOALS", line.substring(result.locationAt(1), result.locationAt(1) + result.lengthAt(1)));
+		assertOnigRegExpSearch(line, result, 28, true, "MAKECMDGOALS", "MAKECMDGOALS");
+	}
+
+	@Test
+	void testNegativeLookBehinds() {
+		// test of OnigRegExp.rewritePatternIfRequired (lookbehind1)
+		assertOnigRegExpSearch("(?<!\\.\\s*)\\b(await)\\b", "await", 0, true, "await", "await");
+		assertOnigRegExpSearch("(?<!\\.\\s*)\\b(await)\\b", "  await", 0, true, "  await", "await");
+		assertOnigRegExpSearch("(?<!\\.\\s*)\\b(await)\\b", ".await", 0, false);
+		assertOnigRegExpSearch("(?<!\\.\\s*)\\b(await)\\b", "  .await", 0, false);
+
+		// test of OnigRegExp.rewritePatternIfRequired (lookbehind2)
+		assertOnigRegExpSearch("(?<=^\\s*)\\\\fi", "\\fi", 0, true, "\\fi");
+		assertOnigRegExpSearch("(?<=^\\s*)\\\\fi", "  \\fi", 0, true, "  \\fi");
+
+		// test of OnigRegExp.rewritePatternIfRequired (lookbehind3)
+		assertOnigRegExpSearch("(?<=\\s*\\.)\\w+", ".foo", 0, true, ".foo");
+		assertOnigRegExpSearch("(?<=\\s*\\.)\\w+", "  .foo", 0, true, "  .foo");
 	}
 }