diff --git a/src/services/code-index/processors/__tests__/parser.spec.ts b/src/services/code-index/processors/__tests__/parser.spec.ts index 68b523c7df..1c8154e03f 100644 --- a/src/services/code-index/processors/__tests__/parser.spec.ts +++ b/src/services/code-index/processors/__tests__/parser.spec.ts @@ -267,6 +267,31 @@ describe("CodeParser", () => { }) describe("_chunkTextByLines", () => { + it("should not emit a chunk whose segment hash has already been seen", () => { + const lines = ["Fallback content long enough to produce a chunk without being filtered out."] + const seenSegmentHashes = new Set() + + const firstResult = parser["_chunkTextByLines"]( + lines, + "manual.txt", + "hash", + "fallback_chunk", + seenSegmentHashes, + ) + const duplicateResult = parser["_chunkTextByLines"]( + lines, + "manual.txt", + "hash", + "fallback_chunk", + seenSegmentHashes, + ) + + expect(firstResult).toHaveLength(1) + expect(firstResult[0].segmentHash).toMatch(/^[a-f0-9]{64}$/) + expect(seenSegmentHashes).toEqual(new Set([firstResult[0].segmentHash])) + expect(duplicateResult).toEqual([]) + }) + it("should handle oversized lines by splitting them", async () => { const longLine = "a".repeat(2000) const lines = ["normal", longLine, "normal"] diff --git a/src/services/code-index/processors/__tests__/parser.txt.spec.ts b/src/services/code-index/processors/__tests__/parser.txt.spec.ts new file mode 100644 index 0000000000..0fa9526916 --- /dev/null +++ b/src/services/code-index/processors/__tests__/parser.txt.spec.ts @@ -0,0 +1,58 @@ +import { CodeParser } from "../parser" +import { scannerExtensions, shouldUseFallbackChunking } from "../../shared/supported-extensions" + +vi.mock("../../../../../packages/telemetry/src/TelemetryService", () => ({ + TelemetryService: { + instance: { + captureEvent: vi.fn(), + }, + }, +})) + +describe("CodeParser - plain text support", () => { + it("supports .txt files through fallback chunking", async () => { + expect(scannerExtensions).toContain(".txt") + expect(shouldUseFallbackChunking(".txt")).toBe(true) + + const content = [ + "Zoo Code plain text indexing regression test.", + "This sentence contains searchable content that only exists in the text file.", + "The fallback parser should preserve every line while creating an indexable chunk.", + ].join("\n") + + const blocks = await new CodeParser().parseFile("manual.txt", { + content, + fileHash: "txt-file-hash", + }) + + expect(blocks).toHaveLength(1) + expect(blocks[0]).toMatchObject({ + file_path: "manual.txt", + type: "fallback_chunk", + start_line: 1, + end_line: 3, + content, + fileHash: "txt-file-hash", + segmentHash: expect.any(String), + }) + }) + + it("parses uppercase .TXT extensions", async () => { + expect(shouldUseFallbackChunking(".TXT")).toBe(true) + + const content = "Uppercase plain-text extension content long enough to produce a fallback chunk." + const blocks = await new CodeParser().parseFile("manual.TXT", { + content, + fileHash: "uppercase-txt-file-hash", + }) + + expect(blocks).toHaveLength(1) + expect(blocks[0]).toMatchObject({ + file_path: "manual.TXT", + type: "fallback_chunk", + content, + fileHash: "uppercase-txt-file-hash", + segmentHash: expect.any(String), + }) + }) +}) diff --git a/src/services/code-index/processors/parser.ts b/src/services/code-index/processors/parser.ts index 8611884ade..8445ea1971 100644 --- a/src/services/code-index/processors/parser.ts +++ b/src/services/code-index/processors/parser.ts @@ -45,7 +45,7 @@ export class CodeParser implements ICodeParser { let content: string let fileHash: string - if (options?.content) { + if (options?.content !== undefined) { content = options.content fileHash = options.fileHash || this.createFileHash(content) } else { diff --git a/src/services/code-index/shared/supported-extensions.ts b/src/services/code-index/shared/supported-extensions.ts index 80dd7102ff..a5f93e161f 100644 --- a/src/services/code-index/shared/supported-extensions.ts +++ b/src/services/code-index/shared/supported-extensions.ts @@ -1,4 +1,7 @@ import { extensions as allExtensions } from "../../tree-sitter" +import { fallbackExtensions, isFallbackExtension } from "../../shared/fallback-extensions" + +export { fallbackExtensions } // Include all extensions including markdown for the scanner export const scannerExtensions = allExtensions @@ -18,17 +21,11 @@ export const scannerExtensions = allExtensions * * Note: Do NOT remove parser cases from languageParser.ts as they may be used elsewhere */ -export const fallbackExtensions = [ - ".vb", // Visual Basic .NET - no dedicated WASM parser - ".scala", // Scala - uses fallback chunking instead of Lua query workaround - ".swift", // Swift - uses fallback chunking due to parser instability -] - /** * Check if a file extension should use fallback chunking * @param extension File extension (including the dot) * @returns true if the extension should use fallback chunking */ export function shouldUseFallbackChunking(extension: string): boolean { - return fallbackExtensions.includes(extension.toLowerCase()) + return isFallbackExtension(extension) } diff --git a/src/services/shared/fallback-extensions.ts b/src/services/shared/fallback-extensions.ts new file mode 100644 index 0000000000..ee78b61b0f --- /dev/null +++ b/src/services/shared/fallback-extensions.ts @@ -0,0 +1,29 @@ +/** + * Extensions that should not be parsed for structural definitions and should + * instead use line-based fallback chunking where indexing is supported. + */ +export const fallbackExtensions = [".txt", ".vb", ".scala", ".swift"] as const + +/** + * Fallback extensions that do not have a structural parser. Scala and Swift + * still support structural parsing outside code indexing. + */ +export const nonStructuralExtensions = [".txt", ".vb"] as const + +/** + * Check whether a file extension should bypass structural parsing. + * + * @param extension File extension, including the leading dot + */ +export function isFallbackExtension(extension: string): boolean { + return (fallbackExtensions as readonly string[]).includes(extension.toLowerCase()) +} + +/** + * Check whether a file extension should bypass structural parsing entirely. + * + * @param extension File extension, including the leading dot + */ +export function isNonStructuralExtension(extension: string): boolean { + return (nonStructuralExtensions as readonly string[]).includes(extension.toLowerCase()) +} diff --git a/src/services/tree-sitter/__tests__/plainTextIntegration.spec.ts b/src/services/tree-sitter/__tests__/plainTextIntegration.spec.ts new file mode 100644 index 0000000000..1635a40d04 --- /dev/null +++ b/src/services/tree-sitter/__tests__/plainTextIntegration.spec.ts @@ -0,0 +1,37 @@ +// Mocks must come first, before imports + +vi.mock("fs/promises", () => ({ + readFile: vi.fn(), +})) + +vi.mock("../../../utils/fs", () => ({ + fileExistsAtPath: vi.fn().mockResolvedValue(true), +})) + +vi.mock("../languageParser", () => ({ + loadRequiredLanguageParsers: vi.fn(), +})) + +import * as fs from "fs/promises" +import { loadRequiredLanguageParsers } from "../languageParser" +import { parseSourceCodeDefinitionsForFile } from "../index" + +describe("Non-structural Extension Integration Tests", () => { + beforeEach(() => { + vi.clearAllMocks() + }) + + it.each(["manual.txt", "legacy.vb"])( + "returns undefined for %s without loading a tree-sitter parser", + async (filePath) => { + const result = await parseSourceCodeDefinitionsForFile(filePath) + + expect(result).toBeUndefined() + }, + ) + + afterEach(() => { + expect(loadRequiredLanguageParsers).not.toHaveBeenCalled() + expect(fs.readFile).not.toHaveBeenCalled() + }) +}) diff --git a/src/services/tree-sitter/index.ts b/src/services/tree-sitter/index.ts index 3c124e5b74..da20a07de7 100644 --- a/src/services/tree-sitter/index.ts +++ b/src/services/tree-sitter/index.ts @@ -5,6 +5,7 @@ import { fileExistsAtPath } from "../../utils/fs" import { parseMarkdown } from "./markdownParser" import { RooIgnoreController } from "../../core/ignore/RooIgnoreController" import { QueryCapture } from "web-tree-sitter" +import { isNonStructuralExtension } from "../shared/fallback-extensions" // Private constant const DEFAULT_MIN_COMPONENT_LINES_VALUE = 4 @@ -66,6 +67,8 @@ const extensions = [ // Markdown "md", "markdown", + // Plain text + "txt", // JSON "json", // CSS @@ -111,6 +114,11 @@ export async function parseSourceCodeDefinitionsForFile( return undefined } + // Files without a structural parser have no definitions to extract + if (isNonStructuralExtension(ext)) { + return undefined + } + // Special case for markdown files if (ext === ".md" || ext === ".markdown") { // Check if we have permission to access this file