feat: add TextMate document partitioner (#952)

Author: sebthom

documentation/DOCUMENT_PARTITIONER.md

@@ -0,0 +1,94 @@
+# TM Partitioner (IDocumentPartitioner)
+
+## Overview
+
+- Partitioning ID: `tm4e.partitioning`
+- Partition type naming: `tm4e:<root-scope>` derived from the active TextMate grammar's base root scope.
+  - Examples: `tm4e:text.html`, `tm4e:source.js`, `tm4e:source.css`.
+- Base partition type:
+  - Before a grammar is known, the synthetic base type `tm4e:base` is used.
+  - After activation with a grammar, the base becomes `tm4e:<root-scope>` (for example, HTML base `tm4e:text.html`).
+- Installed automatically by `org.eclipse.tm4e.ui` for any document that resolves a TextMate grammar. It is added as a secondary partitioning and does not replace an editor's default partitioner.
+
+## Partition Semantics
+
+- `computePartitioning(offset, length)` returns regions that cover the entire requested range contiguously. Gaps between known embedded regions are filled with the current base type.
+- `getPartition(offset)` clamps the offset to the valid range `[0, docLength-1]`. Negative offsets resolve to the first region; EOF resolves to the last region.
+- Regions implement `ITMPartitionRegion` (extends `ITypedRegion`) and carry language metadata:
+  - `getGrammarScope()` returns the effective grammar scope such as `source.js` or `text.html`.
+
+## How to Consume from Code
+
+Get partitions from the TM4E partitioner via `IDocumentExtension3`:
+```java
+var ext3 = (IDocumentExtension3) doc;
+var partitioner = ext3.getDocumentPartitioner(org.eclipse.tm4e.ui.text.TMPartitions.TM_PARTITIONING /* or "tm4e.partitioning" */);
+ITypedRegion[] regions = partitioner.computePartitioning(offset, length);
+```
+
+### Example: Content Assist / Feature Code
+
+```java
+import org.eclipse.tm4e.ui.text.ITMPartitionRegion;
+import org.eclipse.tm4e.ui.text.ITMPartitioner;
+import org.eclipse.tm4e.ui.text.TMPartitions;
+
+// ...
+
+IDocument doc = viewer.getDocument();
+if (doc instanceof IDocumentExtension3 ext3) {
+  if (ext3.getDocumentPartitioner(TMPartitions.TM_PARTITIONING) instanceof ITMPartitioner tmPartitioner) {
+    ITypedRegion r = tmPartitioner.getPartition(caretOffset);
+    // Prefer matching by normalized language scope (handles embedded variants)
+    switch(r.getGrammarScope()) {
+      case "source.js":   // offer JavaScript proposals
+      case "source.css":  // offer CSS proposals
+    }
+  }
+}
+```
+
+## Matching by Scope (Recommended)
+
+- Partition type strings are base-root only (for example, `tm4e:source.js`, `tm4e:text.html`). If you need to distinguish embedded variants (for example, JavaScript-in-Markdown), use `ITMPartitionRegion.getGrammarScope()` which carries the normalized full scope (such as `source.js`).
+- For most feature switches, detect the language via the normalized scope name from `getGrammarScope()` as shown in the example. This is stable for families like `source.css`, `source.js`, or `text.html`.
+
+## Mapping to Content Types (Alternative)
+
+- If your feature keys off `IContentType`, translate the partition at the caret offset:
+
+```java
+IContentType[] cts = org.eclipse.tm4e.ui.text.TMPartitions.getContentTypesForOffset(doc, caretOffset);
+```
+
+## Generic Editor Contributions
+
+- Register your feature with the Generic Editor as usual (e.g., content assist, hovers). Inside your implementation, use the code above to query TM4E’s partitioning and branch behavior based on the normalized scope (recommended) or the partition type.
+- You do not need to install a partitioner yourself; `org.eclipse.tm4e.ui` installs it automatically when a grammar exists.
+
+Example plugin.xml (content assist):
+
+```xml
+<extension point="org.eclipse.ui.genericeditor.contentAssistProcessors">
+  <contentAssistProcessor
+      class="com.example.MyTM4EAwareAssist"
+      contentType="org.eclipse.core.runtime.text"
+      targetId="org.eclipse.ui.genericeditor.GenericEditor"/>
+  <!-- partitionType here refers to the editor's default partitioning;
+       to use TM4E partitions, query tm4e.partitioning from your code. -->
+  </extension>
+```
+
+## Utilities
+
+- Check if a document has a TM4E partitioner installed:
+
+```java
+boolean installed = org.eclipse.tm4e.ui.text.TMPartitions.hasPartitioning(doc);
+```
+
+- Convenience to fetch the TM4E partition at an offset (null if none):
+
+```java
+ITypedRegion part = org.eclipse.tm4e.ui.text.TMPartitions.getPartition(doc, caretOffset);
+```

org.eclipse.tm4e.core/src/main/java/org/eclipse/tm4e/core/internal/grammar/Grammar.java

@@ -318,7 +318,8 @@ private synchronized <T> TokenizeLineResult<T> _tokenize(
 					false,
 					null,
 					scopeList,
-					scopeList);
+					scopeList,
+					null);
 		} else {
 			isFirstLine = false;
 			prevState.reset();

org.eclipse.tm4e.core/src/main/java/org/eclipse/tm4e/core/internal/grammar/LineTokenizer.java

@@ -190,15 +190,24 @@ private void scanNext() {
 			final StateStack beforePush = stack;
 			// push it on the stack rule
 			final var scopeName = rule.getName(lineText.content, captureIndices);
-			final var nameScopesList = castNonNull(stack.contentNameScopesList).pushAttributed(scopeName, grammar);
+
+			// custom tm4e code - not from upstream (for TMPartitioner)
+			// make sure origin grammar scope is present when external
+			AttributedScopeStack baseNames = castNonNull(stack.contentNameScopesList);
+			final var nameScopesList = baseNames.pushAttributed(scopeName, grammar);
+			final String frameGrammarScope = (rule.grammarScope != null && !rule.grammarScope.equals(grammar.getScopeName()))
+					? rule.grammarScope
+					: stack.grammarScope;
+
 			stack = stack.push(
 					matchedRuleId,
 					linePos,
 					anchorPosition,
 					captureIndices[0].end == lineLength,
 					null,
 					nameScopesList,
-					nameScopesList);
+					nameScopesList,
+					frameGrammarScope);
 
 			if (rule instanceof final BeginEndRule pushedRule) {
 				/*if(LOGGER.isLoggable(DEBUG)) {
@@ -218,7 +227,19 @@ private void scanNext() {
 
 				final var contentName = pushedRule.getContentName(lineText.content, captureIndices);
 				final var contentNameScopesList = nameScopesList.pushAttributed(contentName, grammar);
-				stack = stack.withContentNameScopesList(contentNameScopesList);
+
+				// custom tm4e code - not from upstream (for TMPartitioner)
+				// If contentName indicates an embedded root (e.g., source.* or text.*) distinct from the grammar base,
+				// propagate that as the effective grammar scope for this frame so that non-retokenized tokens (e.g., whitespace)
+				// still carry the embedded grammar.
+				String derivedGrammarScope = stack.grammarScope;
+				if (contentName != null) {
+					final String candidate = pickRootScopeFromContentName(contentName);
+					if (candidate != null && !candidate.equals(grammar.getScopeName())) {
+						derivedGrammarScope = candidate;
+					}
+				}
+				stack = stack.withContentNameScopesListAndGrammarScope(contentNameScopesList, derivedGrammarScope);
 
 				if (pushedRule.endHasBackReferences) {
 					stack = stack.withEndRule(
@@ -307,6 +328,31 @@ private void scanNext() {
 		}
 	}
 
+	/**
+	 * Extract a root scope candidate from a contentName string, which may contain multiple scopes separated by spaces.
+	 * Returns the first token that starts with "source." or "text.", or null if none found.
+	 */
+	private @Nullable String pickRootScopeFromContentName(final String contentName) {
+		// custom tm4e code - not from upstream (for TMPartitioner)
+		final int len = contentName.length();
+		int i = 0;
+		while (i < len) {
+			while (i < len && contentName.charAt(i) == ' ')
+				i++;
+			int j = i;
+			while (j < len && contentName.charAt(j) != ' ')
+				j++;
+			if (j > i) {
+				final String part = contentName.substring(i, j);
+				if (part.startsWith("source.") || part.startsWith("text.")) {
+					return part;
+				}
+			}
+			i = j + 1;
+		}
+		return null;
+	}
+
 	private @Nullable MatchResult matchRule(final Grammar grammar, final OnigString lineText, final boolean isFirstLine, final int linePos,
 			final StateStack stack, final int anchorPosition) {
 		final var rule = stack.getRule(grammar);
@@ -460,13 +506,22 @@ private void handleCaptures(final Grammar grammar, final OnigString lineText, fi
 			if (retokenizeCapturedWithRuleId.notEquals(RuleId.NO_RULE)) {
 				// the capture requires additional matching
 				final var scopeName = captureRule.getName(lineTextContent, captureIndices);
-				final var nameScopesList = castNonNull(stack.contentNameScopesList).pushAttributed(scopeName, grammar);
+
+				// custom tm4e code - not from upstream (for TMPartitioner)
+				AttributedScopeStack baseNames = castNonNull(stack.contentNameScopesList);
+				final var retokenizeRule = grammar.getRule(retokenizeCapturedWithRuleId);
+				final var grammarScope = retokenizeRule.grammarScope;
+				final var nameScopesList = baseNames.pushAttributed(scopeName, grammar);
+				final String frameGrammarScope = (grammarScope != null && !grammarScope.equals(grammar.getScopeName()))
+						? grammarScope
+						: stack.grammarScope;
+
 				final var contentName = captureRule.getContentName(lineTextContent, captureIndices);
 				final var contentNameScopesList = nameScopesList.pushAttributed(contentName, grammar);
 
 				// the capture requires additional matching
 				final var stackClone = stack.push(retokenizeCapturedWithRuleId, captureIndex.start, -1, false, null, nameScopesList,
-						contentNameScopesList);
+						contentNameScopesList, frameGrammarScope);
 				final var onigSubStr = OnigString.of(lineTextContent.substring(0, captureIndex.end));
 				tokenizeString(grammar, onigSubStr, isFirstLine && captureIndex.start == 0, captureIndex.start, stackClone, lineTokens,
 						false, Duration.ZERO /* no time limit */);

org.eclipse.tm4e.core/src/main/java/org/eclipse/tm4e/core/internal/grammar/LineTokens.java

@@ -37,17 +37,24 @@
  *      "https://github.com/microsoft/vscode-textmate/blob/167bbbd509356cc4617f250c0d754aef670ab14a/src/grammar/grammar.ts#L945">
  *      github.com/microsoft/vscode-textmate/blob/main/src/grammar/grammar.ts</a>
  */
-final class LineTokens {
+public final class LineTokens {
 
-	private static final class Token implements IToken {
-		private int startIndex;
-		private final int endIndex;
-		private final List<String> scopes;
+	public static final class Token implements IToken {
+		public int startIndex;
+		public final int endIndex;
+		public final List<String> scopes;
 
-		Token(final int startIndex, final int endIndex, final List<String> scopes) {
+		/**
+		 * Optional grammar root scope for this token (e.g., "source.js" or "text.html.basic").
+		 * Implementations may return null when not available.
+		 */
+		public final @Nullable String grammarScope; // custom tm4e code - not from upstream (for TMPartitioner)
+
+		Token(final int startIndex, final int endIndex, final List<String> scopes, final @Nullable String grammarScope) {
 			this.startIndex = startIndex;
 			this.endIndex = endIndex;
 			this.scopes = scopes;
+			this.grammarScope = grammarScope;
 		}
 
 		@Override
@@ -104,7 +111,7 @@ public String toString() {
 
 	private static final Logger LOGGER = System.getLogger(LineTokens.class.getName());
 
-	private static final Deque<IToken> EMPTY_DEQUE = new ArrayDeque<>(0);
+	private static final Deque<Token> EMPTY_DEQUE = new ArrayDeque<>(0);
 
 	private final boolean _emitBinaryTokens;
 
@@ -116,14 +123,15 @@ public String toString() {
 	/**
 	 * used only if `emitBinaryTokens` is false.
 	 */
-	private final Deque<IToken> _tokens;
+	private final Deque<Token> _tokens;
 
 	/**
 	 * used only if `emitBinaryTokens` is true.
 	 */
 	private final List<Integer> _binaryTokens;
 
 	private int _lastTokenEndIndex = 0;
+	private @Nullable String _currentGrammarScope; // custom tm4e code - not from upstream (for TMPartitioner)
 
 	private final List<TokenTypeMatcher> _tokenTypeOverrides;
 
@@ -148,6 +156,7 @@ public String toString() {
 	}
 
 	void produce(final StateStack stack, final int endIndex) {
+		this._currentGrammarScope = stack.grammarScope;
 		this.produceFromScopes(stack.contentNameScopesList, endIndex);
 	}
 
@@ -233,24 +242,24 @@ void produceFromScopes(final @Nullable AttributedScopeStack scopesList, final in
 			}
 		}
 
-		this._tokens.add(new Token(_lastTokenEndIndex, endIndex, scopes));
+		this._tokens.add(new Token(_lastTokenEndIndex, endIndex, scopes, _currentGrammarScope));
 
 		this._lastTokenEndIndex = endIndex;
 	}
 
 	IToken[] getResult(final StateStack stack, final int lineLength) {
-		if (!this._tokens.isEmpty() && this._tokens.getLast().getStartIndex() == lineLength - 1) {
+		if (!this._tokens.isEmpty() && this._tokens.getLast().startIndex == lineLength - 1) {
 			// pop produced token for newline
 			this._tokens.removeLast();
 		}
 
 		if (this._tokens.isEmpty()) {
 			this._lastTokenEndIndex = -1;
 			this.produce(stack, lineLength);
-			this._tokens.getLast().setStartIndex(0);
+			this._tokens.getLast().startIndex = 0;
 		}
 
-		return this._tokens.toArray(IToken[]::new);
+		return this._tokens.toArray(Token[]::new);
 	}
 
 	int[] getBinaryResult(final StateStack stack, final int lineLength) {