From 156d4a5022132de0c07c5fd4734dba51294f2ff8 Mon Sep 17 00:00:00 2001
From: Patrick Klaeren <patrick.klaeren@bspokesoftware.io>
Date: Sat, 20 Jun 2026 09:31:04 +0100
Subject: [PATCH] Fixes #32 Greedy tokenization fix

---
 Remora.Commands/Tokenization/TokenizingEnumerator.cs       | 7 ++++++-
 .../Tokenization/TokenizingEnumeratorTests.cs              | 5 +++++
 2 files changed, 11 insertions(+), 1 deletion(-)

diff --git a/Remora.Commands/Tokenization/TokenizingEnumerator.cs b/Remora.Commands/Tokenization/TokenizingEnumerator.cs
index 681e0a0..0b7fa50 100644
--- a/Remora.Commands/Tokenization/TokenizingEnumerator.cs
+++ b/Remora.Commands/Tokenization/TokenizingEnumerator.cs
@@ -33,6 +33,7 @@ public ref struct TokenizingEnumerator
 {
     private readonly TokenizerOptions _tokenizerOptions;
     private bool _isInCombinedShortNameSegment;
+    private bool _isInAssignmentValueSegment;
     private ReadOnlySpan<char> _segment;
     private SpanSplitEnumerator _splitEnumerator;
 
@@ -51,6 +52,7 @@ public TokenizingEnumerator(ReadOnlySpan<char> value, TokenizerOptions? tokenize
         _tokenizerOptions = tokenizerOptions ?? new TokenizerOptions();
 
         _isInCombinedShortNameSegment = default;
+        _isInAssignmentValueSegment = default;
         _segment = default;
         _splitEnumerator = new SpanSplitEnumerator(value, _tokenizerOptions);
         this.Current = default;
@@ -76,6 +78,7 @@ public bool MoveNext()
             }
 
             _isInCombinedShortNameSegment = false;
+            _isInAssignmentValueSegment = false;
             _segment = _splitEnumerator.Current;
         }
 
@@ -97,9 +100,10 @@ public bool MoveNext()
                 span = span[1..];
             }
         }
-        else if (span.StartsWith("="))
+        else if (_isInAssignmentValueSegment && span.StartsWith("="))
         {
             span = span[1..];
+            _isInAssignmentValueSegment = false;
         }
         else if (_isInCombinedShortNameSegment)
         {
@@ -123,6 +127,7 @@ public bool MoveNext()
             if (assignmentIndex > 0)
             {
                 remainder = span[assignmentIndex..];
+                _isInAssignmentValueSegment = true;
                 span = span[..assignmentIndex];
             }
         }
diff --git a/Tests/Remora.Commands.Tests/Tokenization/TokenizingEnumeratorTests.cs b/Tests/Remora.Commands.Tests/Tokenization/TokenizingEnumeratorTests.cs
index a58a457..9139e1d 100644
--- a/Tests/Remora.Commands.Tests/Tokenization/TokenizingEnumeratorTests.cs
+++ b/Tests/Remora.Commands.Tests/Tokenization/TokenizingEnumeratorTests.cs
@@ -56,6 +56,9 @@ public class TokenizingEnumeratorTests
     [InlineData("-xvf=value", new[] { ShortName, ShortName, ShortName, Value }, new[] { "x", "v", "f", "value" })]
     [InlineData("-xvf=\"booga wooga\"", new[] { ShortName, ShortName, ShortName, Value }, new[] { "x", "v", "f", "booga wooga" })]
     [InlineData("--a -10", new[] { LongName, Value }, new[] { "a", "-10" })]
+    [InlineData("1 == 1", new[] { Value, Value, Value }, new[] { "1", "==", "1" })]
+    [InlineData("1==1", new[] { Value }, new[] { "1==1" })]
+    [InlineData("--a==", new[] { LongName, Value }, new[] { "a", "=" })]
     internal void TokenizesStringCorrectly
     (
         string value,
@@ -99,6 +102,8 @@ IEnumerable<string> expectedTokenValues
     [InlineData("-xvf=value", new[] { ShortName, ShortName, ShortName, Value }, new[] { "x", "v", "f", "value" })]
     [InlineData("-xvf=\"booga wooga\"", new[] { ShortName, ShortName, ShortName, Value }, new[] { "x", "v", "f", "\"booga wooga\"" })]
     [InlineData("--a -10", new[] { LongName, Value }, new[] { "a", "-10" })]
+    [InlineData("1 == 1", new[] { Value, Value, Value }, new[] { "1", "==", "1" })]
+    [InlineData("--a==", new[] { LongName, Value }, new[] { "a", "=" })]
     [InlineData("-a \"\"", new[] { ShortName, Value }, new[] { "a", "\"\"" })]
     internal void RetainsQuotationMarksCorrectly
     (