From 404bdad0aee2ed7ab7a0644f8dfd1f9f8990524a Mon Sep 17 00:00:00 2001
From: marshall <marshall@developerz.ai>
Date: Tue, 14 Jul 2026 02:01:04 +0000
Subject: [PATCH] =?UTF-8?q?fix(compat):=20Read=20fingerprints=20the=20UTF-?=
 =?UTF-8?q?8=20decode,=20not=20raw=20bytes=20=E2=80=94=20non-UTF-8=20stale?=
 =?UTF-8?q?-read=20loop=20(#177)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Read recorded hashFile(path) (raw bytes) while Edit/MultiEdit compare
hashContent(readFile(utf8)). For any file whose bytes are not UTF-8-roundtrippable
(Latin-1 remnants, mixed encodings), the two hashes never agree: Edit rejects with
'modified since you read it', re-reading records the same mismatching byte-hash,
and the pair locks into a permanent stale loop that burns the whole step budget.

Read now fingerprints hashContent(readFile(utf8)) — the same normalization Edit
compares — so a Read->Edit round-trip agrees on non-UTF-8 files while a genuine
external modification is still detected. hashFile stays in file-state.ts for
callers wanting byte identity.

Tests (edit-tools.test.ts, where the Read+Edit interaction lives): a 0xE9-byte
fixture survives Read->Edit (lands, no stale rejection); an external modification
after Read on the same non-UTF-8 file is still rejected.

Part of #196.
---
 .../ai-claude-compat/src/edit-tools.test.ts   | 41 +++++++++++++++++++
 packages/ai-claude-compat/src/fs-tools.ts     | 12 +++---
 2 files changed, 48 insertions(+), 5 deletions(-)

diff --git a/packages/ai-claude-compat/src/edit-tools.test.ts b/packages/ai-claude-compat/src/edit-tools.test.ts
index ba678f4..3bae284 100644
--- a/packages/ai-claude-compat/src/edit-tools.test.ts
+++ b/packages/ai-claude-compat/src/edit-tools.test.ts
@@ -142,6 +142,47 @@ test('editFileTool: rejects an edit against content changed on disk since the re
   }
 });
 
+test('editFileTool: a non-UTF-8-byte file survives Read → Edit — no permanent stale-read loop (issue #177)', async () => {
+  const dir = await tempDir();
+  try {
+    // 0xE9 (Latin-1 'é') is not valid standalone UTF-8: readFile('utf8') decodes it to U+FFFD. Read
+    // must fingerprint the same decode Edit compares, or the two hashes never agree and the pair locks
+    // into a permanent "modified since you read it" loop.
+    const file = join(dir.path, 'latin1.txt');
+    await writeFile(file, Buffer.from([0x68, 0x69, 0x20, 0xe9, 0x20, 0x62, 0x79, 0x65, 0x0a])); // "hi é bye\n"
+    const t = tools(dir.path);
+    await run(t.read, { path: 'latin1.txt' });
+    const out = await run<
+      { path: string; oldString: string; newString: string },
+      { replacements: number }
+    >(t.edit, { path: 'latin1.txt', oldString: 'hi', newString: 'yo' });
+    assert.equal(
+      out.replacements,
+      1,
+      'edit lands — read-before-edit hash matched despite non-UTF-8 bytes',
+    );
+  } finally {
+    await dir.cleanup();
+  }
+});
+
+test('editFileTool: a genuine external modification after Read is still rejected on a non-UTF-8 file (issue #177)', async () => {
+  const dir = await tempDir();
+  try {
+    const file = join(dir.path, 'latin1.txt');
+    await writeFile(file, Buffer.from([0x68, 0x69, 0x20, 0xe9, 0x0a])); // "hi é\n"
+    const t = tools(dir.path);
+    await run(t.read, { path: 'latin1.txt' });
+    await writeFile(file, Buffer.from([0x62, 0x79, 0x65, 0x20, 0xe9, 0x0a])); // externally → "bye é\n"
+    await assert.rejects(
+      () => run(t.edit, { path: 'latin1.txt', oldString: 'bye', newString: 'x' }),
+      /modified since you read it/,
+    );
+  } finally {
+    await dir.cleanup();
+  }
+});
+
 test('editFileTool: an immediate follow-up edit needs no re-read (issue #104)', async () => {
   const dir = await tempDir();
   try {
diff --git a/packages/ai-claude-compat/src/fs-tools.ts b/packages/ai-claude-compat/src/fs-tools.ts
index 30ea5b6..01832dd 100644
--- a/packages/ai-claude-compat/src/fs-tools.ts
+++ b/packages/ai-claude-compat/src/fs-tools.ts
@@ -7,13 +7,13 @@
 // content. Output is `cat -n` numbered, windowed to a default of 2000 lines.
 
 import { createReadStream } from 'node:fs';
-import { mkdir, stat } from 'node:fs/promises';
+import { mkdir, readFile, stat } from 'node:fs/promises';
 import { dirname } from 'node:path';
 import { createInterface } from 'node:readline';
 import { type Tool, tool } from 'ai';
 import { z } from 'zod';
 import { atomicWriteFile } from './atomic-write.ts';
-import { type FileStateTracker, hashContent, hashFile } from './file-state.ts';
+import { type FileStateTracker, hashContent } from './file-state.ts';
 import { resolveInside } from './safe-path.ts';
 
 export type ToolInit = {
@@ -78,9 +78,11 @@ export function readFileTool(init: FileToolInit): Tool<ReadFileInput, ReadFileOu
       const offset = input.offset ?? 1;
       const limit = input.limit ?? DEFAULT_READ_LINES;
       const window = await readNumberedWindow(safe, offset, limit);
-      // Fingerprint the WHOLE file (hashFile streams it) even for a windowed read, so a later Edit
-      // can tell whether the on-disk content changed since this read.
-      init.fileState.record(safe, await hashFile(safe), 'read');
+      // Fingerprint the WHOLE file even for a windowed read, so a later Edit can tell whether the
+      // on-disk content changed since this read. Hash the UTF-8 DECODE (not raw bytes) so it matches
+      // what Edit compares (`hashContent(readFile(utf8))`) — otherwise a non-UTF-8-roundtrippable file
+      // never agrees between Read and Edit and locks into a permanent stale-read loop (issue #177).
+      init.fileState.record(safe, hashContent(await readFile(safe, 'utf8')), 'read');
       return { content: renderReadBody(input.path, window, offset) };
     },
     // The model sees the file content as raw text, not a JSON-escaped `{"content":"…"}` envelope