From 6d0cf9cd13956960a676e974761827e840d87dcf Mon Sep 17 00:00:00 2001 From: gubin-dev Date: Sat, 18 Jul 2026 16:16:03 +0300 Subject: [PATCH 1/5] fix(code-index): index plain text files --- .../processors/__tests__/parser.txt.spec.ts | 42 +++++++++++++++++++ .../code-index/shared/supported-extensions.ts | 1 + src/services/tree-sitter/index.ts | 2 + 3 files changed, 45 insertions(+) create mode 100644 src/services/code-index/processors/__tests__/parser.txt.spec.ts diff --git a/src/services/code-index/processors/__tests__/parser.txt.spec.ts b/src/services/code-index/processors/__tests__/parser.txt.spec.ts new file mode 100644 index 0000000000..29adda8633 --- /dev/null +++ b/src/services/code-index/processors/__tests__/parser.txt.spec.ts @@ -0,0 +1,42 @@ +import { CodeParser } from "../parser" +import { scannerExtensions, shouldUseFallbackChunking } from "../../shared/supported-extensions" + +vi.mock("../../../../../packages/telemetry/src/TelemetryService", () => ({ + TelemetryService: { + instance: { + captureEvent: vi.fn(), + }, + }, +})) + +describe("CodeParser - plain text support", () => { + it("supports .txt files through fallback chunking", async () => { + expect(scannerExtensions).toContain(".txt") + expect(shouldUseFallbackChunking(".txt")).toBe(true) + + const content = [ + "Zoo Code plain text indexing regression test.", + "This sentence contains searchable content that only exists in the text file.", + "The fallback parser should preserve every line while creating an indexable chunk.", + ].join("\n") + + const blocks = await new CodeParser().parseFile("manual.txt", { + content, + fileHash: "txt-file-hash", + }) + + expect(blocks).toHaveLength(1) + expect(blocks[0]).toMatchObject({ + file_path: "manual.txt", + type: "fallback_chunk", + start_line: 1, + end_line: 3, + content, + fileHash: "txt-file-hash", + }) + }) + + it("matches uppercase .TXT extensions", () => { + expect(shouldUseFallbackChunking(".TXT")).toBe(true) + }) +}) diff --git a/src/services/code-index/shared/supported-extensions.ts b/src/services/code-index/shared/supported-extensions.ts index 80dd7102ff..9c62abcee1 100644 --- a/src/services/code-index/shared/supported-extensions.ts +++ b/src/services/code-index/shared/supported-extensions.ts @@ -19,6 +19,7 @@ export const scannerExtensions = allExtensions * Note: Do NOT remove parser cases from languageParser.ts as they may be used elsewhere */ export const fallbackExtensions = [ + ".txt", // Plain text - no tree-sitter parser needed ".vb", // Visual Basic .NET - no dedicated WASM parser ".scala", // Scala - uses fallback chunking instead of Lua query workaround ".swift", // Swift - uses fallback chunking due to parser instability diff --git a/src/services/tree-sitter/index.ts b/src/services/tree-sitter/index.ts index 3c124e5b74..bd5996ff7e 100644 --- a/src/services/tree-sitter/index.ts +++ b/src/services/tree-sitter/index.ts @@ -66,6 +66,8 @@ const extensions = [ // Markdown "md", "markdown", + // Plain text + "txt", // JSON "json", // CSS From f34dd54773fe359726d4a0579d31feb960c16a3e Mon Sep 17 00:00:00 2001 From: gubin-dev Date: Sat, 18 Jul 2026 16:46:35 +0300 Subject: [PATCH 2/5] fix(tree-sitter): skip plain text definition parsing --- .../__tests__/plainTextIntegration.spec.ts | 31 +++++++++++++++++++ src/services/tree-sitter/index.ts | 5 +++ 2 files changed, 36 insertions(+) create mode 100644 src/services/tree-sitter/__tests__/plainTextIntegration.spec.ts diff --git a/src/services/tree-sitter/__tests__/plainTextIntegration.spec.ts b/src/services/tree-sitter/__tests__/plainTextIntegration.spec.ts new file mode 100644 index 0000000000..b23fe87b01 --- /dev/null +++ b/src/services/tree-sitter/__tests__/plainTextIntegration.spec.ts @@ -0,0 +1,31 @@ +// Mocks must come first, before imports + +vi.mock("fs/promises", () => ({ + readFile: vi.fn(), +})) + +vi.mock("../../../utils/fs", () => ({ + fileExistsAtPath: vi.fn().mockResolvedValue(true), +})) + +vi.mock("../languageParser", () => ({ + loadRequiredLanguageParsers: vi.fn(), +})) + +import * as fs from "fs/promises" +import { loadRequiredLanguageParsers } from "../languageParser" +import { parseSourceCodeDefinitionsForFile } from "../index" + +describe("Plain Text Integration Tests", () => { + beforeEach(() => { + vi.clearAllMocks() + }) + + it("returns undefined without loading a tree-sitter parser", async () => { + const result = await parseSourceCodeDefinitionsForFile("manual.txt") + + expect(result).toBeUndefined() + expect(loadRequiredLanguageParsers).not.toHaveBeenCalled() + expect(fs.readFile).not.toHaveBeenCalled() + }) +}) diff --git a/src/services/tree-sitter/index.ts b/src/services/tree-sitter/index.ts index bd5996ff7e..900f3437ad 100644 --- a/src/services/tree-sitter/index.ts +++ b/src/services/tree-sitter/index.ts @@ -113,6 +113,11 @@ export async function parseSourceCodeDefinitionsForFile( return undefined } + // Plain text files have no structural definitions to extract + if (ext === ".txt") { + return undefined + } + // Special case for markdown files if (ext === ".md" || ext === ".markdown") { // Check if we have permission to access this file From c8b5491c88b5e131ecc5afa54b8eba84f0a2e922 Mon Sep 17 00:00:00 2001 From: gubin-dev Date: Mon, 20 Jul 2026 00:47:40 +0300 Subject: [PATCH 3/5] fix(code-index): address plain text review feedback --- .../processors/__tests__/parser.spec.ts | 25 +++++++++++++++++++ .../processors/__tests__/parser.txt.spec.ts | 18 ++++++++++++- src/services/code-index/processors/parser.ts | 2 +- .../code-index/shared/supported-extensions.ts | 12 +++------ src/services/shared/fallback-extensions.ts | 14 +++++++++++ .../__tests__/plainTextIntegration.spec.ts | 14 ++++++++--- src/services/tree-sitter/index.ts | 5 ++-- 7 files changed, 74 insertions(+), 16 deletions(-) create mode 100644 src/services/shared/fallback-extensions.ts diff --git a/src/services/code-index/processors/__tests__/parser.spec.ts b/src/services/code-index/processors/__tests__/parser.spec.ts index 68b523c7df..ccf96c64a7 100644 --- a/src/services/code-index/processors/__tests__/parser.spec.ts +++ b/src/services/code-index/processors/__tests__/parser.spec.ts @@ -267,6 +267,31 @@ describe("CodeParser", () => { }) describe("_chunkTextByLines", () => { + it("should not emit a chunk whose segment hash has already been seen", async () => { + const lines = ["Fallback content long enough to produce a chunk without being filtered out."] + const seenSegmentHashes = new Set() + + const firstResult = await parser["_chunkTextByLines"]( + lines, + "manual.txt", + "hash", + "fallback_chunk", + seenSegmentHashes, + ) + const duplicateResult = await parser["_chunkTextByLines"]( + lines, + "manual.txt", + "hash", + "fallback_chunk", + seenSegmentHashes, + ) + + expect(firstResult).toHaveLength(1) + expect(firstResult[0].segmentHash).toMatch(/^[a-f0-9]{64}$/) + expect(seenSegmentHashes).toEqual(new Set([firstResult[0].segmentHash])) + expect(duplicateResult).toEqual([]) + }) + it("should handle oversized lines by splitting them", async () => { const longLine = "a".repeat(2000) const lines = ["normal", longLine, "normal"] diff --git a/src/services/code-index/processors/__tests__/parser.txt.spec.ts b/src/services/code-index/processors/__tests__/parser.txt.spec.ts index 29adda8633..0fa9526916 100644 --- a/src/services/code-index/processors/__tests__/parser.txt.spec.ts +++ b/src/services/code-index/processors/__tests__/parser.txt.spec.ts @@ -33,10 +33,26 @@ describe("CodeParser - plain text support", () => { end_line: 3, content, fileHash: "txt-file-hash", + segmentHash: expect.any(String), }) }) - it("matches uppercase .TXT extensions", () => { + it("parses uppercase .TXT extensions", async () => { expect(shouldUseFallbackChunking(".TXT")).toBe(true) + + const content = "Uppercase plain-text extension content long enough to produce a fallback chunk." + const blocks = await new CodeParser().parseFile("manual.TXT", { + content, + fileHash: "uppercase-txt-file-hash", + }) + + expect(blocks).toHaveLength(1) + expect(blocks[0]).toMatchObject({ + file_path: "manual.TXT", + type: "fallback_chunk", + content, + fileHash: "uppercase-txt-file-hash", + segmentHash: expect.any(String), + }) }) }) diff --git a/src/services/code-index/processors/parser.ts b/src/services/code-index/processors/parser.ts index 8611884ade..8445ea1971 100644 --- a/src/services/code-index/processors/parser.ts +++ b/src/services/code-index/processors/parser.ts @@ -45,7 +45,7 @@ export class CodeParser implements ICodeParser { let content: string let fileHash: string - if (options?.content) { + if (options?.content !== undefined) { content = options.content fileHash = options.fileHash || this.createFileHash(content) } else { diff --git a/src/services/code-index/shared/supported-extensions.ts b/src/services/code-index/shared/supported-extensions.ts index 9c62abcee1..a5f93e161f 100644 --- a/src/services/code-index/shared/supported-extensions.ts +++ b/src/services/code-index/shared/supported-extensions.ts @@ -1,4 +1,7 @@ import { extensions as allExtensions } from "../../tree-sitter" +import { fallbackExtensions, isFallbackExtension } from "../../shared/fallback-extensions" + +export { fallbackExtensions } // Include all extensions including markdown for the scanner export const scannerExtensions = allExtensions @@ -18,18 +21,11 @@ export const scannerExtensions = allExtensions * * Note: Do NOT remove parser cases from languageParser.ts as they may be used elsewhere */ -export const fallbackExtensions = [ - ".txt", // Plain text - no tree-sitter parser needed - ".vb", // Visual Basic .NET - no dedicated WASM parser - ".scala", // Scala - uses fallback chunking instead of Lua query workaround - ".swift", // Swift - uses fallback chunking due to parser instability -] - /** * Check if a file extension should use fallback chunking * @param extension File extension (including the dot) * @returns true if the extension should use fallback chunking */ export function shouldUseFallbackChunking(extension: string): boolean { - return fallbackExtensions.includes(extension.toLowerCase()) + return isFallbackExtension(extension) } diff --git a/src/services/shared/fallback-extensions.ts b/src/services/shared/fallback-extensions.ts new file mode 100644 index 0000000000..25390540d6 --- /dev/null +++ b/src/services/shared/fallback-extensions.ts @@ -0,0 +1,14 @@ +/** + * Extensions that should not be parsed for structural definitions and should + * instead use line-based fallback chunking where indexing is supported. + */ +export const fallbackExtensions = [".txt", ".vb", ".scala", ".swift"] as const + +/** + * Check whether a file extension should bypass structural parsing. + * + * @param extension File extension, including the leading dot + */ +export function isFallbackExtension(extension: string): boolean { + return (fallbackExtensions as readonly string[]).includes(extension.toLowerCase()) +} diff --git a/src/services/tree-sitter/__tests__/plainTextIntegration.spec.ts b/src/services/tree-sitter/__tests__/plainTextIntegration.spec.ts index b23fe87b01..8894e81d6c 100644 --- a/src/services/tree-sitter/__tests__/plainTextIntegration.spec.ts +++ b/src/services/tree-sitter/__tests__/plainTextIntegration.spec.ts @@ -16,15 +16,21 @@ import * as fs from "fs/promises" import { loadRequiredLanguageParsers } from "../languageParser" import { parseSourceCodeDefinitionsForFile } from "../index" -describe("Plain Text Integration Tests", () => { +describe("Fallback Extension Integration Tests", () => { beforeEach(() => { vi.clearAllMocks() }) - it("returns undefined without loading a tree-sitter parser", async () => { - const result = await parseSourceCodeDefinitionsForFile("manual.txt") + it.each(["manual.txt", "legacy.vb", "service.scala", "client.swift"])( + "returns undefined for %s without loading a tree-sitter parser", + async (filePath) => { + const result = await parseSourceCodeDefinitionsForFile(filePath) - expect(result).toBeUndefined() + expect(result).toBeUndefined() + }, + ) + + afterEach(() => { expect(loadRequiredLanguageParsers).not.toHaveBeenCalled() expect(fs.readFile).not.toHaveBeenCalled() }) diff --git a/src/services/tree-sitter/index.ts b/src/services/tree-sitter/index.ts index 900f3437ad..768d53e416 100644 --- a/src/services/tree-sitter/index.ts +++ b/src/services/tree-sitter/index.ts @@ -5,6 +5,7 @@ import { fileExistsAtPath } from "../../utils/fs" import { parseMarkdown } from "./markdownParser" import { RooIgnoreController } from "../../core/ignore/RooIgnoreController" import { QueryCapture } from "web-tree-sitter" +import { isFallbackExtension } from "../shared/fallback-extensions" // Private constant const DEFAULT_MIN_COMPONENT_LINES_VALUE = 4 @@ -113,8 +114,8 @@ export async function parseSourceCodeDefinitionsForFile( return undefined } - // Plain text files have no structural definitions to extract - if (ext === ".txt") { + // Fallback files have no structural definitions to extract + if (isFallbackExtension(ext)) { return undefined } From 7446f3375854096d1b43d45780a8970ce9aabb6e Mon Sep 17 00:00:00 2001 From: gubin-dev Date: Mon, 20 Jul 2026 00:57:04 +0300 Subject: [PATCH 4/5] fix(tree-sitter): preserve Scala structural parsing --- src/services/shared/fallback-extensions.ts | 15 +++++++++++++++ .../__tests__/plainTextIntegration.spec.ts | 4 ++-- src/services/tree-sitter/index.ts | 6 +++--- 3 files changed, 20 insertions(+), 5 deletions(-) diff --git a/src/services/shared/fallback-extensions.ts b/src/services/shared/fallback-extensions.ts index 25390540d6..ee78b61b0f 100644 --- a/src/services/shared/fallback-extensions.ts +++ b/src/services/shared/fallback-extensions.ts @@ -4,6 +4,12 @@ */ export const fallbackExtensions = [".txt", ".vb", ".scala", ".swift"] as const +/** + * Fallback extensions that do not have a structural parser. Scala and Swift + * still support structural parsing outside code indexing. + */ +export const nonStructuralExtensions = [".txt", ".vb"] as const + /** * Check whether a file extension should bypass structural parsing. * @@ -12,3 +18,12 @@ export const fallbackExtensions = [".txt", ".vb", ".scala", ".swift"] as const export function isFallbackExtension(extension: string): boolean { return (fallbackExtensions as readonly string[]).includes(extension.toLowerCase()) } + +/** + * Check whether a file extension should bypass structural parsing entirely. + * + * @param extension File extension, including the leading dot + */ +export function isNonStructuralExtension(extension: string): boolean { + return (nonStructuralExtensions as readonly string[]).includes(extension.toLowerCase()) +} diff --git a/src/services/tree-sitter/__tests__/plainTextIntegration.spec.ts b/src/services/tree-sitter/__tests__/plainTextIntegration.spec.ts index 8894e81d6c..1635a40d04 100644 --- a/src/services/tree-sitter/__tests__/plainTextIntegration.spec.ts +++ b/src/services/tree-sitter/__tests__/plainTextIntegration.spec.ts @@ -16,12 +16,12 @@ import * as fs from "fs/promises" import { loadRequiredLanguageParsers } from "../languageParser" import { parseSourceCodeDefinitionsForFile } from "../index" -describe("Fallback Extension Integration Tests", () => { +describe("Non-structural Extension Integration Tests", () => { beforeEach(() => { vi.clearAllMocks() }) - it.each(["manual.txt", "legacy.vb", "service.scala", "client.swift"])( + it.each(["manual.txt", "legacy.vb"])( "returns undefined for %s without loading a tree-sitter parser", async (filePath) => { const result = await parseSourceCodeDefinitionsForFile(filePath) diff --git a/src/services/tree-sitter/index.ts b/src/services/tree-sitter/index.ts index 768d53e416..da20a07de7 100644 --- a/src/services/tree-sitter/index.ts +++ b/src/services/tree-sitter/index.ts @@ -5,7 +5,7 @@ import { fileExistsAtPath } from "../../utils/fs" import { parseMarkdown } from "./markdownParser" import { RooIgnoreController } from "../../core/ignore/RooIgnoreController" import { QueryCapture } from "web-tree-sitter" -import { isFallbackExtension } from "../shared/fallback-extensions" +import { isNonStructuralExtension } from "../shared/fallback-extensions" // Private constant const DEFAULT_MIN_COMPONENT_LINES_VALUE = 4 @@ -114,8 +114,8 @@ export async function parseSourceCodeDefinitionsForFile( return undefined } - // Fallback files have no structural definitions to extract - if (isFallbackExtension(ext)) { + // Files without a structural parser have no definitions to extract + if (isNonStructuralExtension(ext)) { return undefined } From 7dde9a117e72a082065b5318cb91ad6ba1bd22b4 Mon Sep 17 00:00:00 2001 From: gubin-dev Date: Mon, 20 Jul 2026 01:06:51 +0300 Subject: [PATCH 5/5] test(code-index): remove redundant async usage --- src/services/code-index/processors/__tests__/parser.spec.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/services/code-index/processors/__tests__/parser.spec.ts b/src/services/code-index/processors/__tests__/parser.spec.ts index ccf96c64a7..1c8154e03f 100644 --- a/src/services/code-index/processors/__tests__/parser.spec.ts +++ b/src/services/code-index/processors/__tests__/parser.spec.ts @@ -267,18 +267,18 @@ describe("CodeParser", () => { }) describe("_chunkTextByLines", () => { - it("should not emit a chunk whose segment hash has already been seen", async () => { + it("should not emit a chunk whose segment hash has already been seen", () => { const lines = ["Fallback content long enough to produce a chunk without being filtered out."] const seenSegmentHashes = new Set() - const firstResult = await parser["_chunkTextByLines"]( + const firstResult = parser["_chunkTextByLines"]( lines, "manual.txt", "hash", "fallback_chunk", seenSegmentHashes, ) - const duplicateResult = await parser["_chunkTextByLines"]( + const duplicateResult = parser["_chunkTextByLines"]( lines, "manual.txt", "hash",