fix(web): improve prediction whitespace-input, backspace-input edge case handling 🚂

web/src/engine/predictive-text/worker-thread/src/main/correction/context-tokenization.ts

@@ -673,9 +673,6 @@ export class ContextTokenization {
         tailTokenization.splice(tokenIndex, 1, affectedToken);
       }
 
-      affectedToken.isPartial = true;
-      delete affectedToken.appliedTransitionId;
-
       // If we are completely replacing a token via delete left, erase the deleteLeft;
       // that part applied to a _previous_ token that no longer exists.
       // We start at index 0 in the insert string for the "new" token.
@@ -699,6 +696,11 @@ export class ContextTokenization {
       affectedToken = new ContextToken(affectedToken);
       affectedToken.addInput(inputSource, distribution);
 
+      // Do not adjust the original token, as it may be used by other transitions.
+      // Only adjust the new, extended token.
+      affectedToken.isPartial = true;
+      delete affectedToken.appliedTransitionId;
+
       const tokenize = determineModelTokenizer(lexicalModel);
       affectedToken.isWhitespace = tokenize({left: affectedToken.exampleInput, startOfBuffer: false, endOfBuffer: false}).left[0]?.isWhitespace ?? false;
       // Do not re-use the previous token; the mutation may have unexpected
@@ -708,8 +710,19 @@ export class ContextTokenization {
       affectedToken = null;
     }
 
+    // Backspace handling - emptying context via backspace or erasing _part_ of
+    // a whitespace token can erase the tokenization-final empty token usually
+    // used for word-initial suggestions.
+    //
+    // We re-add it here so that suggestions can be presented to the user as
+    // normal.
+    const tokenSequence = this.tokens.slice(0, sliceIndex).concat(tailTokenization);
+    if(tokenSequence.length == 0 || tokenSequence[tokenSequence.length - 1]?.isWhitespace) {
+      tokenSequence.push(new ContextToken(new LegacyQuotientRoot(lexicalModel)));
+    }
+
     return new ContextTokenization(
-      this.tokens.slice(0, sliceIndex).concat(tailTokenization),
+      tokenSequence,
       null,
       determineTaillessTrueKeystroke(transitionEdge)
     );

web/src/engine/predictive-text/worker-thread/src/main/correction/context-transition.ts

@@ -19,6 +19,15 @@ import Reversion = LexicalModelTypes.Reversion;
 import Suggestion = LexicalModelTypes.Suggestion;
 import Transform = LexicalModelTypes.Transform;
 
+export interface TransitionReversionView extends Pick<ContextTransition, 'reversion'> {
+  /**
+   * Gets the context state resulting from the context transition event,
+   * including any generated suggestions and data regarding potential
+   * application thereof.
+   */
+  final: Pick<ContextState, 'suggestions'>
+}
+
 /**
  * Represents the transition between two context states as triggered
  * by input keystrokes or applied suggestions.

web/src/engine/predictive-text/worker-thread/src/main/correction/tokenization-subsets.ts

@@ -135,7 +135,8 @@ export function legacySubsetKeyer(tokenizationEdits: TokenizationTransitionEdits
   // Now, based on the transform tokenization. We want to force uniqueness for
   // all variations of result length on each tokenized transform resulting from
   // the precomputation's represented keystroke.
-  for(const {0: relativeIndex} of tokenizedTransform.entries()) {
+  for(const {0: relativeIndex, 1: transform} of tokenizedTransform.entries()) {
+    const insertLen = KMWString.length(transform.insert);
     if(relativeIndex > 0) {
       // The true boundary lie before the insert if the value is non-zero;
       // don't differentiate here!
@@ -148,10 +149,10 @@ export function legacySubsetKeyer(tokenizationEdits: TokenizationTransitionEdits
       //
       // IMPORTANT:  update unit tests manually if the BI marker here changes
       // or the use of SENTINEL_CODE_UNIT as a key component separator changes.
-      components.push(`BI@${relativeIndex}`);
+      components.push(`BI@${relativeIndex}-${boundaryTextLen + insertLen}`);
       boundaryTextLen = 0;
     } else {
-      components.push(`I@${relativeIndex}`);
+      components.push(`I@${relativeIndex}-${boundaryTextLen + insertLen}`);
     }
   }
 

web/src/engine/predictive-text/worker-thread/src/main/model-compositor.ts

@@ -2,7 +2,7 @@ import * as models from '@keymanapp/models-templates';
 import { LexicalModelTypes } from '@keymanapp/common-types';
 
 import { TransformUtils } from './transformUtils.js';
-import { applySuggestionCasing, correctAndEnumerate, createDefaultKeep, dedupeSuggestions, finalizeSuggestions, predictionAutoSelect, processSimilarity, toAnnotatedSuggestion, tupleDisplayOrderSort } from './predict-helpers.js';
+import { applySuggestionCasing, correctAndEnumerate, createDefaultKeep, dedupeSuggestions, finalizeSuggestions, predictionAutoSelect, prependReversion, processSimilarity, toAnnotatedSuggestion, tupleDisplayOrderSort } from './predict-helpers.js';
 import { detectCurrentCasing, determineModelTokenizer, determineModelWordbreaker, determinePunctuationFromModel } from './model-helpers.js';
 
 import { ContextTracker } from './correction/context-tracker.js';
@@ -206,18 +206,8 @@ export class ModelCompositor {
       this.SUGGESTION_ID_SEED++;
     });
 
-    if(revertableTransitionId) {
-      const reversion = this.contextTracker.peek(revertableTransitionId)?.reversion;
-      if(reversion) {
-        if(suggestions[0]?.tag == 'keep') {
-          const keep = suggestions.shift();
-          suggestions.unshift(reversion);
-          suggestions.unshift(keep);
-        } else {
-          suggestions.unshift(reversion)
-        }
-      }
-    }
+    const transitionToRevert = this.contextTracker?.peek(revertableTransitionId);
+    prependReversion(suggestions, transitionToRevert);
 
     // Store the suggestions on the final token of the current context state (if it exists).
     // Or, once phrase-level suggestions are possible, on whichever token serves as each prediction's root.
@@ -239,8 +229,8 @@ export class ModelCompositor {
     // Step 1:  re-use the original input Transform as the reversion's Transform.
     // The Web engine will restore the original state of the context before accepting
     // and before reverting; all we need to do is put the original keystroke back in place.
-    let reversionTransform: Transform = originalInput 
-      ? { ...originalInput } 
+    let reversionTransform: Transform = originalInput
+      ? { ...originalInput }
       : { insert: '', deleteLeft: 0, id: suggestion.transform.id };
 
     // Step 2:  building the proper 'displayAs' string for the Reversion

web/src/engine/predictive-text/worker-thread/src/main/predict-helpers.ts

@@ -9,11 +9,10 @@ import { ContextTokenization } from './correction/context-tokenization.js';
 import { ContextTracker } from './correction/context-tracker.js';
 import { ContextToken } from './correction/context-token.js';
 import { ContextState, determineContextSlideTransform } from './correction/context-state.js';
-import { ContextTransition } from './correction/context-transition.js';
+import { ContextTransition, TransitionReversionView } from './correction/context-transition.js';
 import { ExecutionTimer } from './correction/execution-timer.js';
 import { ModelCompositor } from './model-compositor.js';
 import { getBestTokenMatches } from './correction/distance-modeler.js';
-import { TokenResultMapping } from './correction/token-result-mapping.js';
 
 import CasingForm = LexicalModelTypes.CasingForm;
 import Context = LexicalModelTypes.Context;
@@ -459,7 +458,8 @@ export function determineSuggestionRange(
 export function buildAndMapPredictions(
   transition: ContextTransition,
   tokenization: ContextTokenization,
-  match: Readonly<TokenResultMapping>,
+  // Originally, Readonly<TokenResultMapping> - but we only need these two components here.
+  match: Readonly<{matchString: string, totalCost: number}>,
   costFactor: number
 ): CorrectionPredictionTuple[] {
   const model = transition.final.model;
@@ -493,6 +493,13 @@ export function buildAndMapPredictions(
     // entry.baseTokenization = transition.final.tokenizationSourceMap.get(tokenization);
   });
 
+  // Backspaces that shorten a multi-codepoint whitespace token are not handled well by default.
+  // As a new empty token is placed at the end for such cases, we can detect and handle such cases.
+  const inputTransform = transition.inputDistribution?.[0].sample ?? { insert: '', deleteLeft: 0 };
+  if(tokenization.tokens.length > 1 && tokenization.tail.searchModule.codepointLength == 0 && inputTransform.deleteLeft > 0) {
+    predictions.forEach((p) => p.prediction.sample.transform.deleteLeft += inputTransform.deleteLeft);
+  }
+
   return predictions;
 }
 
@@ -578,11 +585,6 @@ export async function correctAndEnumerate(
     // Corrections obtained:  now to predict from them!
     const tokenization = tokenizations.find(t => t.spaceId == match.spaceId);
 
-    // If our 'match' results in fully deleting the new token, reject it and try again.
-    if(match.matchSequence.length == 0 && match.inputSequence.length != 0) {
-      continue;
-    }
-
     // If our 'match' fully replaces the token, reject it and try again.
     if(match.matchSequence.length != 0 && match.matchSequence.length == match.knownCost) {
       continue;
@@ -1077,6 +1079,9 @@ export function finalizeSuggestions(
       if(presDL > 0) {
         mergedTransform.deleteLeft -= presDL;
       }
+      if(prediction.sample.transform.id !== undefined) {
+        mergedTransform.id = prediction.sample.transform.id;
+      }
 
       // Temporarily and locally drops 'readonly' semantics so that we can reassign the transform.
       // See https://www.typescriptlang.org/docs/handbook/release-notes/typescript-2-8.html#improved-control-over-mapped-type-modifiers
@@ -1191,4 +1196,35 @@ export function toAnnotatedSuggestion(
   }
 
   return result;
+}
+
+/**
+ * For applicable scenarios, this mutates the passed-in suggestion array by
+ * prepending a predictive-text reversion that restores the context to a prior
+ * state.  Otherwise, it leaves the suggestion array unaltered.
+ * @param suggestions
+ * @param transitionToRevert
+ * @returns
+ */
+export function prependReversion(suggestions: Suggestion[], transitionToRevert: TransitionReversionView) {
+  if(transitionToRevert) {
+    const reversion = transitionToRevert.reversion;
+    if(reversion) {
+      if(suggestions[0]?.tag == 'keep') {
+        const appliedId = -reversion.id;
+        const appliedSuggestion = transitionToRevert.final.suggestions.find((s) => s.id == appliedId);
+        // If the selected suggestion was itself a `keep`, we don't need a
+        // reversion.  They'd do the same thing.
+        if(appliedSuggestion.tag != 'keep') {
+          const keep = suggestions.shift();
+          suggestions.unshift(reversion);
+          suggestions.unshift(keep);
+        }
+      } else {
+        suggestions.unshift(reversion);
+      }
+    }
+  }
+
+  return suggestions;
 }

web/src/engine/predictive-text/worker-thread/src/main/test-index.ts

@@ -3,7 +3,7 @@ export * from './correction/context-state.js';
 export * from './correction/context-token.js';
 export * from './correction/context-tokenization.js';
 export { ContextTracker } from './correction/context-tracker.js';
-export { ContextTransition } from './correction/context-transition.js';
+export * from './correction/context-transition.js';
 export * from './correction/correction-searchable.js';
 export * from './correction/correction-result-mapping.js';
 export * from './correction/distance-modeler.js';

web/src/engine/src/main/headless/transcriptionCache.ts

@@ -1,7 +1,7 @@
 import { Transcription } from "keyman/engine/keyboard";
 import { RewindableCache } from "keyman/common/web-utils";
 
-const TRANSCRIPTION_BUFFER_SIZE = 10;
+const TRANSCRIPTION_BUFFER_SIZE = 20;
 
 export class TranscriptionCache extends RewindableCache<Transcription> {
   constructor() {